@goodandready/dsh-model-sync 0.2.7 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -0
- package/lib/adapter-registry.js +147 -27
- package/lib/client.js +87 -14
- package/lib/credentials.js +86 -5
- package/lib/generic-adapter.js +37 -5
- package/lib/history.js +138 -0
- package/lib/http.js +121 -4
- package/lib/index.js +62 -1
- package/lib/lifecycle.js +130 -0
- package/lib/models.js +203 -11
- package/lib/policy.js +71 -0
- package/lib/reconcile.js +8 -2
- package/lib/reliability.js +112 -0
- package/lib/reporting.js +155 -0
- package/lib/scheduler.js +127 -21
- package/lib/synchronizer.js +427 -36
- package/package.json +1 -1
package/lib/synchronizer.js
CHANGED
|
@@ -3,8 +3,45 @@ import { createProviderAdapterRegistry } from './adapter-registry.js'
|
|
|
3
3
|
import { createCredentialResolver } from './credentials.js'
|
|
4
4
|
import { reconcileModels, catalogPatch } from './reconcile.js'
|
|
5
5
|
import { normalizeModels } from './models.js'
|
|
6
|
+
import { filterModels, normalizePolicy, validatePolicy } from './policy.js'
|
|
7
|
+
import { createTimeoutSignal, isRetryableError, mapWithConcurrency, retryWithBackoff, statusOfError } from './reliability.js'
|
|
8
|
+
import { appendHistory, createHistoryEntry, historyEntry, listHistory } from './history.js'
|
|
9
|
+
import { lifecycleOptions, lifecycleStatus, selectableModels, updateLifecycle } from './lifecycle.js'
|
|
10
|
+
import { appendNotification, listNotifications, createSyncReport, updateNotification } from './reporting.js'
|
|
6
11
|
|
|
7
12
|
const PIAI_NS = 'llm-pi-ai'
|
|
13
|
+
const DEFAULT_RELIABILITY = Object.freeze({
|
|
14
|
+
timeoutMs: 15000,
|
|
15
|
+
retryAttempts: 3,
|
|
16
|
+
retryBaseDelayMs: 250,
|
|
17
|
+
retryMaxDelayMs: 4000,
|
|
18
|
+
concurrency: 4,
|
|
19
|
+
circuitBreakerFailures: 3,
|
|
20
|
+
circuitBreakerCooldownMs: 300000,
|
|
21
|
+
})
|
|
22
|
+
|
|
23
|
+
function boundedInteger(value, fallback, minimum, maximum) {
|
|
24
|
+
const number = Number(value)
|
|
25
|
+
if (!Number.isFinite(number)) return fallback
|
|
26
|
+
return Math.min(maximum, Math.max(minimum, Math.floor(number)))
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function reliabilityOptions(config, options) {
|
|
30
|
+
const source = { ...config, ...options }
|
|
31
|
+
const baseDelay = boundedInteger(source.retryBaseDelayMs, DEFAULT_RELIABILITY.retryBaseDelayMs, 0, 60000)
|
|
32
|
+
const cooldownMs = source.circuitBreakerCooldownMs !== undefined
|
|
33
|
+
? boundedInteger(source.circuitBreakerCooldownMs, DEFAULT_RELIABILITY.circuitBreakerCooldownMs, 0, 3600000)
|
|
34
|
+
: boundedInteger(Number(source.circuitBreakerCooldownMinutes) * 60000, DEFAULT_RELIABILITY.circuitBreakerCooldownMs, 0, 3600000)
|
|
35
|
+
return {
|
|
36
|
+
timeoutMs: boundedInteger(source.requestTimeoutMs, DEFAULT_RELIABILITY.timeoutMs, 100, 120000),
|
|
37
|
+
retryAttempts: boundedInteger(source.retryAttempts, DEFAULT_RELIABILITY.retryAttempts, 1, 6),
|
|
38
|
+
retryBaseDelayMs: baseDelay,
|
|
39
|
+
retryMaxDelayMs: Math.max(baseDelay, boundedInteger(source.retryMaxDelayMs, DEFAULT_RELIABILITY.retryMaxDelayMs, 0, 120000)),
|
|
40
|
+
concurrency: boundedInteger(source.concurrency, DEFAULT_RELIABILITY.concurrency, 1, 32),
|
|
41
|
+
circuitBreakerFailures: boundedInteger(source.circuitBreakerFailures, DEFAULT_RELIABILITY.circuitBreakerFailures, 0, 20),
|
|
42
|
+
circuitBreakerCooldownMs: cooldownMs,
|
|
43
|
+
}
|
|
44
|
+
}
|
|
8
45
|
|
|
9
46
|
function settingsOf(ctx) {
|
|
10
47
|
try { return ctx.get('settings') } catch { return undefined }
|
|
@@ -24,11 +61,88 @@ function errorMessage(error) {
|
|
|
24
61
|
return error instanceof Error ? error.message : String(error)
|
|
25
62
|
}
|
|
26
63
|
|
|
27
|
-
|
|
64
|
+
function diagnosticMessage(error) {
|
|
65
|
+
return errorMessage(error).replace(/(api[_-]?key|token|secret|authorization|password)\s*[:=]\s*[^\s,;]+/gi, '$1=[redacted]').slice(0, 160)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function adapterErrorType(error) {
|
|
69
|
+
if (error?.adapterCode === 'auth') return 'auth'
|
|
70
|
+
if (error?.adapterCode === 'schema') return 'schema'
|
|
71
|
+
if (error?.adapterCode === 'endpoint') return 'endpoint'
|
|
72
|
+
if (error?.code === 'CREDENTIALS_UNAVAILABLE' || error?.code === 'AUTH') return 'credential'
|
|
73
|
+
const status = statusOfError(error)
|
|
74
|
+
if (status === 401 || status === 403) return 'auth'
|
|
75
|
+
if (status) return 'http'
|
|
76
|
+
return 'endpoint'
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function rotationRefs(settings, provider) {
|
|
80
|
+
try {
|
|
81
|
+
const rows = settings?.get?.('dsh-key-rotation')?.providers
|
|
82
|
+
if (!Array.isArray(rows)) return []
|
|
83
|
+
return rows.filter((row) => row?.provider === provider).flatMap((row) => Array.isArray(row.keys) ? row.keys : [])
|
|
84
|
+
} catch {
|
|
85
|
+
return []
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function createModelSynchronizer(ctx, { getConfig = () => ({}), saveConfig = async () => {}, fetchImpl = fetch, adapterImplementations = {} } = {}) {
|
|
28
90
|
const resolveCredential = createCredentialResolver(ctx)
|
|
29
|
-
const registry = createProviderAdapterRegistry({ resolveCredential, fetchImpl })
|
|
91
|
+
const registry = createProviderAdapterRegistry({ resolveCredential, fetchImpl, adapters: adapterImplementations })
|
|
30
92
|
let running = null
|
|
31
93
|
let lastRun = null
|
|
94
|
+
let lastHealth = null
|
|
95
|
+
let healthRunning = null
|
|
96
|
+
const circuits = new Map()
|
|
97
|
+
const credentialOutcomes = new Map()
|
|
98
|
+
let lastReport = null
|
|
99
|
+
let transientNotifications = []
|
|
100
|
+
|
|
101
|
+
function recordCredentialOutcome(provider, outcome) {
|
|
102
|
+
credentialOutcomes.set(provider, { ...outcome, at: Date.now() })
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function selectAdapter(profile) {
|
|
106
|
+
const config = getConfig() ?? {}
|
|
107
|
+
return registry.select(profile, config.adapterRegistry?.[profile.provider])
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function refsFor(row, settings) {
|
|
111
|
+
const profile = profileOf(settings, row.provider)
|
|
112
|
+
const refs = [
|
|
113
|
+
row.apiKeyEnv,
|
|
114
|
+
profile.credentialRef,
|
|
115
|
+
...(Array.isArray(profile.credentialRefs) ? profile.credentialRefs : []),
|
|
116
|
+
...(Array.isArray(profile.apiKeyRefs) ? profile.apiKeyRefs : []),
|
|
117
|
+
...rotationRefs(settings, row.provider),
|
|
118
|
+
]
|
|
119
|
+
return [...new Set(refs.filter((ref) => typeof ref === 'string' && ref.trim()).map((ref) => ref.trim()))].slice(0, 32)
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function circuitOpen(provider, reliability) {
|
|
123
|
+
if (reliability.circuitBreakerFailures <= 0) return null
|
|
124
|
+
const state = circuits.get(provider)
|
|
125
|
+
if (!state) return null
|
|
126
|
+
if (state.blockedUntil > Date.now()) {
|
|
127
|
+
return { retryAfterMs: state.blockedUntil - Date.now() }
|
|
128
|
+
}
|
|
129
|
+
if (state.blockedUntil) circuits.delete(provider)
|
|
130
|
+
return null
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function recordFailure(provider, reliability, error) {
|
|
134
|
+
if (reliability.circuitBreakerFailures <= 0 || !isRetryableError(error)) return
|
|
135
|
+
const state = circuits.get(provider) ?? { failures: 0, blockedUntil: 0 }
|
|
136
|
+
state.failures += 1
|
|
137
|
+
if (state.failures >= reliability.circuitBreakerFailures) {
|
|
138
|
+
state.blockedUntil = Date.now() + reliability.circuitBreakerCooldownMs
|
|
139
|
+
}
|
|
140
|
+
circuits.set(provider, state)
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function policyFor(config, provider) {
|
|
144
|
+
return normalizePolicy(config?.modelPolicies?.[provider])
|
|
145
|
+
}
|
|
32
146
|
|
|
33
147
|
function providers() {
|
|
34
148
|
const config = getConfig() ?? {}
|
|
@@ -38,76 +152,266 @@ export function createModelSynchronizer(ctx, { getConfig = () => ({}), saveConfi
|
|
|
38
152
|
? [...new Set(config.modelSelections[row.provider].filter((id) => typeof id === 'string' && id))]
|
|
39
153
|
: []
|
|
40
154
|
const latest = lastResults.get(row.provider)
|
|
155
|
+
const policy = policyFor(config, row.provider)
|
|
41
156
|
const cached = normalizeModels(row.provider, config.modelCatalogs?.[row.provider])
|
|
157
|
+
const available = latest?.status === 'ok'
|
|
158
|
+
? latest.availableModels ?? latest.models ?? row.models
|
|
159
|
+
: cached.length > 0 ? cached : row.models
|
|
160
|
+
const availableModels = filterModels(available, policy)
|
|
161
|
+
const lifecycleModels = lifecycleStatus(config.modelLifecycle, row.provider, availableModels)
|
|
42
162
|
return {
|
|
43
163
|
...row,
|
|
44
164
|
selectedModels: selection,
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
165
|
+
policy,
|
|
166
|
+
availableModels,
|
|
167
|
+
lifecycleModels,
|
|
168
|
+
selectableModels: lifecycleModels.filter((model) => model.lifecycleStatus === 'active'),
|
|
48
169
|
}
|
|
49
170
|
})
|
|
50
171
|
}
|
|
51
172
|
|
|
173
|
+
function listPolicies() {
|
|
174
|
+
const config = getConfig() ?? {}
|
|
175
|
+
return Object.fromEntries(providers()
|
|
176
|
+
.filter((row) => row.configured)
|
|
177
|
+
.map((row) => [row.provider, policyFor(config, row.provider)]))
|
|
178
|
+
}
|
|
179
|
+
|
|
52
180
|
function selectionFor(config, provider) {
|
|
53
181
|
const selection = config?.modelSelections?.[provider]
|
|
54
182
|
return Array.isArray(selection) ? [...new Set(selection.filter((id) => typeof id === 'string' && id))] : []
|
|
55
183
|
}
|
|
56
184
|
|
|
57
|
-
async function discover(row, { signal } = {}) {
|
|
185
|
+
async function discover(row, { signal, reliability = DEFAULT_RELIABILITY } = {}) {
|
|
58
186
|
const settings = settingsOf(ctx)
|
|
59
187
|
const profile = { ...profileOf(settings, row.provider), provider: row.provider }
|
|
60
|
-
|
|
188
|
+
let adapter
|
|
189
|
+
try { adapter = selectAdapter(profile) } catch (error) {
|
|
190
|
+
return { provider: row.provider, status: 'invalid-config', models: [], message: diagnosticMessage(error), code: error?.code ?? 'ADAPTER_CONFIG_INVALID' }
|
|
191
|
+
}
|
|
61
192
|
if (!adapter) return { provider: row.provider, status: 'unsupported', models: [], message: 'no supported API-key model adapter' }
|
|
62
|
-
|
|
193
|
+
const adapterConfig = getConfig()?.adapterRegistry?.[row.provider]
|
|
194
|
+
if (!profile.apiKeyEnv && !profile.baseURL && !adapterConfig?.endpoint && !profile.adapterId) {
|
|
63
195
|
return { provider: row.provider, status: 'dormant', models: [], message: 'provider has no API key route configured' }
|
|
64
196
|
}
|
|
197
|
+
const open = circuitOpen(row.provider, reliability)
|
|
198
|
+
if (open) {
|
|
199
|
+
return {
|
|
200
|
+
provider: row.provider,
|
|
201
|
+
status: 'circuit-open',
|
|
202
|
+
models: [],
|
|
203
|
+
message: 'provider circuit is open after repeated transient failures',
|
|
204
|
+
retryAfterMs: open.retryAfterMs,
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
let retries = 0
|
|
208
|
+
try {
|
|
209
|
+
const models = await retryWithBackoff(async () => {
|
|
210
|
+
const timed = createTimeoutSignal(signal, reliability.timeoutMs)
|
|
211
|
+
try {
|
|
212
|
+
return await adapter.discover(profile, { signal: timed.signal })
|
|
213
|
+
} finally {
|
|
214
|
+
timed.cleanup()
|
|
215
|
+
}
|
|
216
|
+
}, {
|
|
217
|
+
attempts: reliability.retryAttempts,
|
|
218
|
+
baseDelayMs: reliability.retryBaseDelayMs,
|
|
219
|
+
maxDelayMs: reliability.retryMaxDelayMs,
|
|
220
|
+
signal,
|
|
221
|
+
onRetry: () => { retries += 1 },
|
|
222
|
+
})
|
|
223
|
+
circuits.delete(row.provider)
|
|
224
|
+
recordCredentialOutcome(row.provider, { status: 'ok', retries })
|
|
225
|
+
return { provider: row.provider, status: 'ok', models, retries }
|
|
226
|
+
} catch (error) {
|
|
227
|
+
if (!signal?.aborted) recordFailure(row.provider, reliability, error)
|
|
228
|
+
const state = circuits.get(row.provider)
|
|
229
|
+
recordCredentialOutcome(row.provider, { status: 'error', code: statusOfError(error), errorType: adapterErrorType(error), retryable: isRetryableError(error), message: diagnosticMessage(error), retries })
|
|
230
|
+
return {
|
|
231
|
+
provider: row.provider,
|
|
232
|
+
status: 'error',
|
|
233
|
+
models: [],
|
|
234
|
+
message: diagnosticMessage(error),
|
|
235
|
+
errorType: adapterErrorType(error),
|
|
236
|
+
retries,
|
|
237
|
+
retryable: isRetryableError(error),
|
|
238
|
+
circuitOpen: Boolean(state?.blockedUntil > Date.now()),
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
async function probe(row, { signal, reliability = DEFAULT_RELIABILITY } = {}) {
|
|
244
|
+
const settings = settingsOf(ctx)
|
|
245
|
+
const profile = { ...profileOf(settings, row.provider), provider: row.provider }
|
|
246
|
+
let adapter
|
|
247
|
+
try { adapter = selectAdapter(profile) } catch (error) {
|
|
248
|
+
return { provider: row.provider, status: 'invalid-config', message: diagnosticMessage(error), code: error?.code ?? 'ADAPTER_CONFIG_INVALID' }
|
|
249
|
+
}
|
|
250
|
+
if (!adapter?.health) return { provider: row.provider, status: 'unsupported', message: 'provider has no health probe' }
|
|
251
|
+
const adapterConfig = getConfig()?.adapterRegistry?.[row.provider]
|
|
252
|
+
if (!profile.apiKeyEnv && !profile.baseURL && !adapterConfig?.endpoint && !profile.adapterId) {
|
|
253
|
+
return { provider: row.provider, status: 'dormant', message: 'provider has no API key route configured' }
|
|
254
|
+
}
|
|
255
|
+
const startedAt = Date.now()
|
|
256
|
+
let retries = 0
|
|
65
257
|
try {
|
|
66
|
-
const
|
|
67
|
-
|
|
258
|
+
const result = await retryWithBackoff(async () => {
|
|
259
|
+
const timed = createTimeoutSignal(signal, reliability.timeoutMs)
|
|
260
|
+
try {
|
|
261
|
+
return await adapter.health(profile, { signal: timed.signal })
|
|
262
|
+
} finally {
|
|
263
|
+
timed.cleanup()
|
|
264
|
+
}
|
|
265
|
+
}, {
|
|
266
|
+
attempts: reliability.retryAttempts,
|
|
267
|
+
baseDelayMs: reliability.retryBaseDelayMs,
|
|
268
|
+
maxDelayMs: reliability.retryMaxDelayMs,
|
|
269
|
+
signal,
|
|
270
|
+
onRetry: () => { retries += 1 },
|
|
271
|
+
})
|
|
272
|
+
recordCredentialOutcome(row.provider, { status: 'ok', code: result?.statusCode ?? 200, retries })
|
|
273
|
+
return {
|
|
274
|
+
provider: row.provider,
|
|
275
|
+
status: 'ok',
|
|
276
|
+
statusCode: result?.statusCode ?? 200,
|
|
277
|
+
latencyMs: result?.latencyMs ?? Date.now() - startedAt,
|
|
278
|
+
retries,
|
|
279
|
+
}
|
|
68
280
|
} catch (error) {
|
|
69
|
-
|
|
281
|
+
recordCredentialOutcome(row.provider, { status: 'error', code: statusOfError(error), errorType: adapterErrorType(error), retryable: isRetryableError(error), message: diagnosticMessage(error), retries })
|
|
282
|
+
return {
|
|
283
|
+
provider: row.provider,
|
|
284
|
+
status: 'error',
|
|
285
|
+
statusCode: statusOfError(error),
|
|
286
|
+
latencyMs: Date.now() - startedAt,
|
|
287
|
+
message: diagnosticMessage(error),
|
|
288
|
+
errorType: adapterErrorType(error),
|
|
289
|
+
retries,
|
|
290
|
+
retryable: isRetryableError(error),
|
|
291
|
+
}
|
|
70
292
|
}
|
|
71
293
|
}
|
|
72
294
|
|
|
295
|
+
async function credentialDiagnostics(options = {}) {
|
|
296
|
+
const settings = settingsOf(ctx)
|
|
297
|
+
const selected = typeof options.provider === 'string' && options.provider
|
|
298
|
+
? providers().filter((row) => row.provider === options.provider && row.configured)
|
|
299
|
+
: providers().filter((row) => row.configured)
|
|
300
|
+
const results = await Promise.all(selected.map(async (row) => {
|
|
301
|
+
const refs = refsFor(row, settings)
|
|
302
|
+
const rotation = rotationRefs(settings, row.provider)
|
|
303
|
+
const diagnostics = typeof resolveCredential.diagnostics === 'function'
|
|
304
|
+
? await resolveCredential.diagnostics(refs)
|
|
305
|
+
: []
|
|
306
|
+
return {
|
|
307
|
+
provider: row.provider,
|
|
308
|
+
displayName: row.displayName,
|
|
309
|
+
refs: diagnostics,
|
|
310
|
+
rotation: {
|
|
311
|
+
configured: rotation.length > 0,
|
|
312
|
+
keyCount: rotation.length,
|
|
313
|
+
fallbackOrder: diagnostics.filter((item) => rotation.includes(item.ref)).map((item) => item.label),
|
|
314
|
+
},
|
|
315
|
+
...(credentialOutcomes.has(row.provider) ? { lastRequest: structuredClone(credentialOutcomes.get(row.provider)) } : {}),
|
|
316
|
+
}
|
|
317
|
+
}))
|
|
318
|
+
return { checkedAt: Date.now(), results }
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
async function health(options = {}) {
|
|
322
|
+
if (healthRunning) return healthRunning
|
|
323
|
+
const config = getConfig() ?? {}
|
|
324
|
+
const reliability = reliabilityOptions(config, options)
|
|
325
|
+
const providerConfig = Array.isArray(config.providers) ? config.providers : []
|
|
326
|
+
const selected = typeof options.provider === 'string' && options.provider
|
|
327
|
+
? providers().filter((row) => row.provider === options.provider)
|
|
328
|
+
: providers().filter((row) => providerConfig.length === 0 || providerConfig.some((item) => item.provider === row.provider && item.enabled !== false))
|
|
329
|
+
const startedAt = Date.now()
|
|
330
|
+
const task = (async () => {
|
|
331
|
+
const results = await mapWithConcurrency(
|
|
332
|
+
selected,
|
|
333
|
+
(row) => probe(row, { signal: options.signal, reliability }),
|
|
334
|
+
reliability.concurrency,
|
|
335
|
+
)
|
|
336
|
+
const snapshot = { startedAt, finishedAt: Date.now(), reliability, results }
|
|
337
|
+
lastHealth = snapshot
|
|
338
|
+
return snapshot
|
|
339
|
+
})()
|
|
340
|
+
healthRunning = task.finally(() => { healthRunning = null })
|
|
341
|
+
return healthRunning
|
|
342
|
+
}
|
|
343
|
+
|
|
73
344
|
async function run(options = {}) {
|
|
74
345
|
if (running) return running
|
|
75
346
|
const config = getConfig() ?? {}
|
|
347
|
+
const reliability = reliabilityOptions(config, options)
|
|
76
348
|
const cachedCatalogs = structuredClone(config.modelCatalogs ?? {})
|
|
349
|
+
const nextLifecycle = structuredClone(config.modelLifecycle ?? {})
|
|
350
|
+
const lifecycleRun = (Number(config.lifecycleRevision) || 0) + 1
|
|
351
|
+
const lifecycleConfig = lifecycleOptions(config)
|
|
77
352
|
const dryRun = options.dryRun !== false
|
|
78
353
|
const removeMissing = options.removeMissing === true
|
|
354
|
+
const providerConfig = Array.isArray(config.providers) ? config.providers : []
|
|
79
355
|
const selected = typeof options.provider === 'string' && options.provider
|
|
80
356
|
? providers().filter((row) => row.provider === options.provider)
|
|
81
|
-
: providers().filter((row) =>
|
|
357
|
+
: providers().filter((row) => providerConfig.length === 0 || providerConfig.some((item) => item.provider === row.provider && item.enabled !== false))
|
|
82
358
|
const startedAt = Date.now()
|
|
83
359
|
const task = (async () => {
|
|
360
|
+
const discoveredRows = await mapWithConcurrency(
|
|
361
|
+
selected,
|
|
362
|
+
(row) => discover(row, { signal: options.signal, reliability }),
|
|
363
|
+
reliability.concurrency,
|
|
364
|
+
)
|
|
84
365
|
const results = []
|
|
85
|
-
for (
|
|
86
|
-
const
|
|
366
|
+
for (let index = 0; index < selected.length; index += 1) {
|
|
367
|
+
const row = selected[index]
|
|
368
|
+
const discovered = discoveredRows[index]
|
|
87
369
|
if (discovered.status !== 'ok') {
|
|
88
370
|
results.push({ ...discovered, changed: false })
|
|
89
371
|
continue
|
|
90
372
|
}
|
|
91
373
|
const settings = settingsOf(ctx)
|
|
92
374
|
const current = profileOf(settings, row.provider).models ?? row.models
|
|
375
|
+
const fullModels = normalizeModels(row.provider, discovered.models)
|
|
376
|
+
const lifecycleBefore = normalizeModels(row.provider, config.modelCatalogs?.[row.provider] ?? current)
|
|
377
|
+
const lifecycle = updateLifecycle(config.modelLifecycle?.[row.provider], lifecycleBefore, fullModels, {
|
|
378
|
+
runNumber: lifecycleRun,
|
|
379
|
+
...lifecycleConfig,
|
|
380
|
+
removeMissing,
|
|
381
|
+
})
|
|
382
|
+
nextLifecycle[row.provider] = lifecycle.records
|
|
383
|
+
const policy = policyFor(config, row.provider)
|
|
384
|
+
const policyModels = filterModels(fullModels, policy)
|
|
93
385
|
const selectedModels = selectionFor(config, row.provider)
|
|
94
386
|
const advertised = selectedModels.length
|
|
95
|
-
?
|
|
96
|
-
:
|
|
97
|
-
const
|
|
98
|
-
const reconciliation = reconcileModels(row.provider, current, advertised, { removeMissing:
|
|
387
|
+
? policyModels.filter((model) => selectedModels.includes(model.id))
|
|
388
|
+
: policyModels
|
|
389
|
+
const explicitRemoveIds = new Set(lifecycle.removedIds)
|
|
390
|
+
const reconciliation = reconcileModels(row.provider, current, advertised, { removeMissing: false, removeIds: explicitRemoveIds })
|
|
99
391
|
results.push({
|
|
100
392
|
...discovered,
|
|
101
|
-
|
|
393
|
+
discoveredModels: fullModels,
|
|
394
|
+
availableModels: policyModels,
|
|
395
|
+
policy,
|
|
102
396
|
selectedModels,
|
|
103
397
|
advertised,
|
|
398
|
+
lifecycle: lifecycle.statuses,
|
|
399
|
+
lifecycleChanges: lifecycle.changes,
|
|
104
400
|
...reconciliation,
|
|
105
|
-
removeMissing:
|
|
401
|
+
removeMissing: lifecycle.removedIds.length > 0,
|
|
106
402
|
changed: reconciliation.changed,
|
|
107
403
|
})
|
|
108
|
-
|
|
404
|
+
const freshIds = new Set(fullModels.map((model) => model.id))
|
|
405
|
+
const removedIds = new Set(lifecycle.removedIds)
|
|
406
|
+
const retainedCatalog = lifecycleBefore.filter((model) => !freshIds.has(model.id) && !removedIds.has(model.id))
|
|
407
|
+
cachedCatalogs[row.provider] = normalizeModels(row.provider, [...retainedCatalog, ...fullModels])
|
|
109
408
|
}
|
|
110
|
-
const result = { startedAt, finishedAt: Date.now(), dryRun, removeMissing, results, applied: false }
|
|
409
|
+
const result = { startedAt, finishedAt: Date.now(), dryRun, removeMissing, reliability, lifecycleRevision: lifecycleRun, results, applied: false }
|
|
410
|
+
const report = createSyncReport(result, { source: options.reportSource === 'schedule' ? 'schedule' : 'manual' })
|
|
411
|
+
lastReport = report
|
|
412
|
+
const notificationLimit = config.notificationLimit
|
|
413
|
+
transientNotifications = appendNotification(transientNotifications, report, { limit: notificationLimit })
|
|
414
|
+
result.report = report
|
|
111
415
|
if (!dryRun && results.some((row) => row.status === 'ok' && row.changed)) {
|
|
112
416
|
const settings = settingsOf(ctx)
|
|
113
417
|
const section = settings?.get?.(PIAI_NS)
|
|
@@ -124,18 +428,29 @@ export function createModelSynchronizer(ctx, { getConfig = () => ({}), saveConfi
|
|
|
124
428
|
}
|
|
125
429
|
if (!dryRun) {
|
|
126
430
|
const previousCatalogs = config.modelCatalogs ?? {}
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
431
|
+
const patch = { modelLifecycle: nextLifecycle, lifecycleRevision: lifecycleRun }
|
|
432
|
+
if (JSON.stringify(previousCatalogs) !== JSON.stringify(cachedCatalogs)) patch.modelCatalogs = cachedCatalogs
|
|
433
|
+
const previousHistory = Array.isArray(config.history) ? config.history : []
|
|
434
|
+
const nextVersion = previousHistory.reduce((max, entry) => Math.max(max, Number(entry?.version) || 0), 0) + 1
|
|
435
|
+
const entry = createHistoryEntry(result, { version: nextVersion })
|
|
436
|
+
patch.history = appendHistory(previousHistory, entry, config.historyLimit)
|
|
437
|
+
const previousNotifications = Array.isArray(config.notifications) ? config.notifications : []
|
|
438
|
+
const nextNotifications = appendNotification(previousNotifications, report, { limit: config.notificationLimit })
|
|
439
|
+
if (JSON.stringify(previousNotifications) !== JSON.stringify(nextNotifications)) patch.notifications = nextNotifications
|
|
440
|
+
result.historyId = entry.id
|
|
441
|
+
try {
|
|
442
|
+
await saveConfig(patch)
|
|
443
|
+
if (patch.modelCatalogs) result.catalogCachePersisted = true
|
|
444
|
+
result.historyPersisted = true
|
|
445
|
+
} catch (error) {
|
|
446
|
+
// The active DSH catalog has already been revision-checked. A cache
|
|
447
|
+
// or history failure is visible but must not undo that successful
|
|
448
|
+
// provider update.
|
|
449
|
+
if (patch.modelCatalogs) result.catalogCacheError = errorMessage(error)
|
|
450
|
+
result.historyError = errorMessage(error)
|
|
137
451
|
}
|
|
138
452
|
}
|
|
453
|
+
result.notifications = notificationLedger()
|
|
139
454
|
lastRun = result
|
|
140
455
|
return result
|
|
141
456
|
})()
|
|
@@ -143,6 +458,70 @@ export function createModelSynchronizer(ctx, { getConfig = () => ({}), saveConfi
|
|
|
143
458
|
return running
|
|
144
459
|
}
|
|
145
460
|
|
|
461
|
+
function history(options = {}) {
|
|
462
|
+
const config = getConfig() ?? {}
|
|
463
|
+
return listHistory(config.history, options)
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
function notificationLedger(options = {}) {
|
|
467
|
+
const config = getConfig() ?? {}
|
|
468
|
+
const byId = new Map()
|
|
469
|
+
for (const row of [...(Array.isArray(config.notifications) ? config.notifications : []), ...transientNotifications]) {
|
|
470
|
+
if (row?.id) byId.set(row.id, row)
|
|
471
|
+
}
|
|
472
|
+
return listNotifications([...byId.values()], options)
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
async function updateNotificationState(id, field) {
|
|
476
|
+
const config = getConfig() ?? {}
|
|
477
|
+
const persisted = Array.isArray(config.notifications) ? config.notifications : []
|
|
478
|
+
if (persisted.some((row) => row?.id === id)) {
|
|
479
|
+
const next = updateNotification(persisted, id, field)
|
|
480
|
+
await saveConfig({ notifications: next })
|
|
481
|
+
transientNotifications = transientNotifications.filter((row) => row?.id !== id)
|
|
482
|
+
return next.find((row) => row.id === id)
|
|
483
|
+
}
|
|
484
|
+
transientNotifications = updateNotification(transientNotifications, id, field)
|
|
485
|
+
return transientNotifications.find((row) => row.id === id)
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
async function rollbackHistory({ historyId, provider } = {}) {
|
|
489
|
+
if (typeof historyId !== 'string' || !historyId) throw Object.assign(new Error('historyId is required'), { code: 'INVALID_HISTORY' })
|
|
490
|
+
if (typeof provider !== 'string' || !provider) throw Object.assign(new Error('provider is required'), { code: 'INVALID_HISTORY' })
|
|
491
|
+
const config = getConfig() ?? {}
|
|
492
|
+
const entry = historyEntry(config.history, historyId)
|
|
493
|
+
if (!entry) throw Object.assign(new Error(`history entry not found: ${historyId}`), { code: 'HISTORY_NOT_FOUND' })
|
|
494
|
+
const snapshot = (entry.providers ?? []).find((row) => row.provider === provider)
|
|
495
|
+
if (!snapshot || !Array.isArray(snapshot.before)) throw Object.assign(new Error(`history snapshot not found: ${provider}`), { code: 'HISTORY_NOT_FOUND' })
|
|
496
|
+
if (!providers().some((row) => row.provider === provider && row.configured)) throw Object.assign(new Error(`provider is not configured: ${provider}`), { code: 'PROVIDER_NOT_CONFIGURED' })
|
|
497
|
+
const settings = settingsOf(ctx)
|
|
498
|
+
const section = settings?.get?.(PIAI_NS)
|
|
499
|
+
const descriptor = descriptorOf(settings, PIAI_NS)
|
|
500
|
+
if (!section || !descriptor || typeof settings.replace !== 'function') {
|
|
501
|
+
throw Object.assign(new Error('llm-pi-ai settings are not writable'), { code: 'SETTINGS_UNAVAILABLE' })
|
|
502
|
+
}
|
|
503
|
+
const current = normalizeModels(provider, profileOf(settings, provider).models ?? [])
|
|
504
|
+
const target = normalizeModels(provider, snapshot.before)
|
|
505
|
+
if (JSON.stringify(current) === JSON.stringify(target)) {
|
|
506
|
+
return { historyId, provider, models: target, applied: false, allowlistUntouched: true }
|
|
507
|
+
}
|
|
508
|
+
const next = catalogPatch(section, provider, target)
|
|
509
|
+
await settings.replace(PIAI_NS, next, descriptor.revision)
|
|
510
|
+
return { historyId, provider, models: target, applied: true, allowlistUntouched: true }
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
async function setModelPolicy(provider, policy) {
|
|
514
|
+
if (typeof provider !== 'string' || !provider) throw new Error('provider is required')
|
|
515
|
+
const row = providers().find((item) => item.provider === provider && item.configured)
|
|
516
|
+
if (!row) throw Object.assign(new Error(`provider is not configured: ${provider}`), { code: 'PROVIDER_NOT_CONFIGURED' })
|
|
517
|
+
const normalized = validatePolicy(policy)
|
|
518
|
+
const config = getConfig() ?? {}
|
|
519
|
+
const previous = structuredClone(config.modelPolicies ?? {})
|
|
520
|
+
const modelPolicies = { ...previous, [provider]: normalized }
|
|
521
|
+
await saveConfig({ modelPolicies })
|
|
522
|
+
return { provider, policy: normalized, applied: false }
|
|
523
|
+
}
|
|
524
|
+
|
|
146
525
|
async function setModelSelection(provider, models) {
|
|
147
526
|
if (typeof provider !== 'string' || !provider) throw new Error('provider is required')
|
|
148
527
|
if (!Array.isArray(models)) throw new Error('models must be an array')
|
|
@@ -156,7 +535,8 @@ export function createModelSynchronizer(ctx, { getConfig = () => ({}), saveConfi
|
|
|
156
535
|
const source = latestCatalog !== undefined
|
|
157
536
|
? latestCatalog
|
|
158
537
|
: cached.length > 0 ? cached : row.availableModels ?? row.models
|
|
159
|
-
const available = normalizeModels(provider, source)
|
|
538
|
+
const available = filterModels(normalizeModels(provider, source), policyFor(config, provider))
|
|
539
|
+
const safeAvailable = selectableModels(config.modelLifecycle, provider, available)
|
|
160
540
|
const availableById = new Map(available.map((model) => [model.id, model]))
|
|
161
541
|
const unknown = selectedModels.filter((id) => !availableById.has(id))
|
|
162
542
|
if (unknown.length > 0) throw Object.assign(new Error(`unknown models for ${provider}: ${unknown.join(', ')}`), { code: 'UNKNOWN_MODEL' })
|
|
@@ -171,7 +551,7 @@ export function createModelSynchronizer(ctx, { getConfig = () => ({}), saveConfi
|
|
|
171
551
|
const current = normalizeModels(provider, profileOf(settings, provider).models ?? row.models)
|
|
172
552
|
const nextModels = selectedModels.length
|
|
173
553
|
? selectedModels.map((id) => availableById.get(id))
|
|
174
|
-
: (
|
|
554
|
+
: (safeAvailable.length > 0 ? safeAvailable : current)
|
|
175
555
|
const next = catalogPatch(section, provider, nextModels)
|
|
176
556
|
const previousModelSelections = structuredClone(config.modelSelections ?? {})
|
|
177
557
|
const modelSelections = { ...previousModelSelections, [provider]: selectedModels }
|
|
@@ -189,15 +569,26 @@ export function createModelSynchronizer(ctx, { getConfig = () => ({}), saveConfi
|
|
|
189
569
|
}
|
|
190
570
|
throw error
|
|
191
571
|
}
|
|
192
|
-
return { provider, selectedModels, models: nextModels, availableModels: available, applied: true }
|
|
572
|
+
return { provider, selectedModels, models: nextModels, availableModels: available, selectableModels: safeAvailable, applied: true }
|
|
193
573
|
}
|
|
194
574
|
|
|
195
575
|
return Object.freeze({
|
|
196
576
|
listProviders: providers,
|
|
197
577
|
getProvider: (provider) => providers().find((row) => row.provider === provider) ?? null,
|
|
578
|
+
adapterRegistry: registry,
|
|
579
|
+
registerAdapter: registry.register,
|
|
198
580
|
discover,
|
|
199
581
|
run,
|
|
200
582
|
setModelSelection,
|
|
201
|
-
|
|
583
|
+
setModelPolicy,
|
|
584
|
+
listPolicies,
|
|
585
|
+
health,
|
|
586
|
+
credentialDiagnostics,
|
|
587
|
+
history,
|
|
588
|
+
report: () => structuredClone(lastReport),
|
|
589
|
+
notifications: notificationLedger,
|
|
590
|
+
updateNotification: updateNotificationState,
|
|
591
|
+
rollbackHistory,
|
|
592
|
+
status: () => ({ running: Boolean(running), lastRun, lastHealth, lastReport: structuredClone(lastReport), notifications: notificationLedger(), history: history(), policies: listPolicies(), lifecycle: structuredClone(getConfig()?.modelLifecycle ?? {}) }),
|
|
202
593
|
})
|
|
203
594
|
}
|