@goodandready/dsh-model-sync 0.5.9 → 0.5.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/adapter-registry.js +6 -0
- package/lib/alias-resolver.js +16 -2
- package/lib/client.js +2 -0
- package/lib/credentials.js +10 -1
- package/lib/diff-notifier.js +77 -15
- package/lib/generic-adapter.js +9 -1
- package/lib/history.js +2 -1
- package/lib/http.js +28 -19
- package/lib/inventory.js +16 -8
- package/lib/models.js +3 -3
- package/lib/policy.js +6 -0
- package/lib/reliability.js +86 -14
- package/lib/reporting.js +2 -4
- package/lib/synchronizer-helpers.js +2 -1
- package/lib/synchronizer-prober.js +130 -50
- package/lib/synchronizer.js +39 -6
- package/lib/updater.js +45 -11
- package/package.json +1 -1
package/lib/adapter-registry.js
CHANGED
|
@@ -219,6 +219,9 @@ async function requestSpecific(profile, descriptor, { resolveCredential, fetchIm
|
|
|
219
219
|
}
|
|
220
220
|
if (response.status === 304) return response
|
|
221
221
|
if (!response.ok) {
|
|
222
|
+
if (typeof response.body?.cancel === "function") {
|
|
223
|
+
try { await response.body.cancel() } catch (_err) { /* best-effort cancel */ }
|
|
224
|
+
}
|
|
222
225
|
const error = catalogRequestError(response.status, response)
|
|
223
226
|
error.adapterCode = [401, 403].includes(response.status) ? 'auth' : response.status === 429 ? 'rate-limit' : 'http'
|
|
224
227
|
throw error
|
|
@@ -231,6 +234,9 @@ async function fetchSpecific(profile, descriptor, options) {
|
|
|
231
234
|
try {
|
|
232
235
|
const response = await requestSpecific(profile, descriptor, options)
|
|
233
236
|
if (response.status === 304) {
|
|
237
|
+
if (typeof response.body?.cancel === "function") {
|
|
238
|
+
try { await response.body.cancel() } catch (_err) { /* best-effort cancel */ }
|
|
239
|
+
}
|
|
234
240
|
return { notModified: true, etag: response.headers?.get?.('etag') || options.etag, lastModified: response.headers?.get?.('last-modified') || options.lastModified }
|
|
235
241
|
}
|
|
236
242
|
const respEtag = response.headers?.get?.('etag') || undefined
|
package/lib/alias-resolver.js
CHANGED
|
@@ -53,6 +53,16 @@ function estimateLatency(provider, model, probes = {}) {
|
|
|
53
53
|
return Math.round(base)
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
+
function isProviderEligible(provider, { healthResults = [], isCircuitOpen = () => false } = {}) {
|
|
57
|
+
if (isCircuitOpen(provider)) return false
|
|
58
|
+
const health = healthResults.find((h) => h.provider === provider)
|
|
59
|
+
if (health) {
|
|
60
|
+
if (health.ok === false) return false
|
|
61
|
+
if (health.status && health.status !== "ok") return false
|
|
62
|
+
}
|
|
63
|
+
return true
|
|
64
|
+
}
|
|
65
|
+
|
|
56
66
|
export function resolveAlias(aliasName, {
|
|
57
67
|
aliases = {},
|
|
58
68
|
modelCatalogs = {},
|
|
@@ -72,7 +82,8 @@ export function resolveAlias(aliasName, {
|
|
|
72
82
|
for (let i = 0; i < custom.targets.length; i++) {
|
|
73
83
|
const target = custom.targets[i]
|
|
74
84
|
const circuitBlocked = isCircuitOpen(target.provider)
|
|
75
|
-
const
|
|
85
|
+
const healthEntry = healthResults.find((h) => h.provider === target.provider)
|
|
86
|
+
const healthFail = Boolean(healthEntry && (healthEntry.status === "error" || healthEntry.status === "circuit-open" || healthEntry.ok === false || (healthEntry.status && healthEntry.status !== "ok")))
|
|
76
87
|
if (circuitBlocked || healthFail) {
|
|
77
88
|
skipped.push({ ...target, reason: circuitBlocked ? 'circuit_open' : 'health_failed' })
|
|
78
89
|
continue
|
|
@@ -101,12 +112,15 @@ export function resolveAlias(aliasName, {
|
|
|
101
112
|
// 2. Static explicit alias
|
|
102
113
|
if (custom?.provider && custom?.model && !custom.strategy) {
|
|
103
114
|
const circuitBlocked = isCircuitOpen(custom.provider)
|
|
115
|
+
const healthEntry = healthResults.find((h) => h.provider === custom.provider)
|
|
116
|
+
const healthFailed = Boolean(healthEntry && (healthEntry.status === "error" || healthEntry.status === "circuit-open" || healthEntry.ok === false || (healthEntry.status && healthEntry.status !== "ok")))
|
|
104
117
|
return {
|
|
105
118
|
alias: clean,
|
|
106
119
|
provider: custom.provider,
|
|
107
120
|
model: custom.model,
|
|
108
121
|
fallbackUsed: false,
|
|
109
122
|
...(circuitBlocked ? { warning: 'circuit_open', circuitOpen: true } : {}),
|
|
123
|
+
...(healthFailed ? { warning: 'health_failed', healthFailed: true } : {}),
|
|
110
124
|
}
|
|
111
125
|
}
|
|
112
126
|
|
|
@@ -125,7 +139,7 @@ export function resolveAlias(aliasName, {
|
|
|
125
139
|
}
|
|
126
140
|
const available = []
|
|
127
141
|
for (const [provider, models] of Object.entries(modelCatalogs)) {
|
|
128
|
-
if (
|
|
142
|
+
if (!isProviderEligible(provider, { healthResults, isCircuitOpen })) continue
|
|
129
143
|
if (!Array.isArray(models)) continue
|
|
130
144
|
for (const m of models) {
|
|
131
145
|
if (!m?.id) continue
|
package/lib/client.js
CHANGED
|
@@ -600,6 +600,7 @@ window.__ModuleLoader__.load({
|
|
|
600
600
|
setBusy(true); setWebhookErr(''); setWebhookMsg('')
|
|
601
601
|
try {
|
|
602
602
|
await scope.set('diffWebhookUrl', webhookDraft.diffWebhookUrl || '')
|
|
603
|
+
await scope.set('diffWebhookSecretRef', webhookDraft.diffWebhookSecretRef || webhookDraft.diffWebhookSecret || '')
|
|
603
604
|
await scope.set('diffWebhookSecret', webhookDraft.diffWebhookSecret || '')
|
|
604
605
|
await scope.set('notifyOnModelAdded', !!webhookDraft.notifyOnModelAdded)
|
|
605
606
|
await scope.set('notifyOnModelDeprecated', !!webhookDraft.notifyOnModelDeprecated)
|
|
@@ -622,6 +623,7 @@ window.__ModuleLoader__.load({
|
|
|
622
623
|
body: JSON.stringify({
|
|
623
624
|
url: webhookDraft.diffWebhookUrl,
|
|
624
625
|
secret: webhookDraft.diffWebhookSecret || '',
|
|
626
|
+
secretRef: webhookDraft.diffWebhookSecretRef || webhookDraft.diffWebhookSecret || '',
|
|
625
627
|
}),
|
|
626
628
|
})
|
|
627
629
|
const body = await res.json()
|
package/lib/credentials.js
CHANGED
|
@@ -38,9 +38,18 @@ function safeCode(error) {
|
|
|
38
38
|
return 'credential-error'
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
+
export function sanitizeSecretString(value) {
|
|
42
|
+
if (value === null || value === undefined) return ''
|
|
43
|
+
return String(value)
|
|
44
|
+
.replace(/(authorization)\s*[:=]\s*(?:bearer|basic|token)\s+[^\s,;&]+/gi, '$1=[redacted]')
|
|
45
|
+
.replace(/(api[_-]?key|token|secret|authorization|password)\s*[:=]\s*[^\s,;&]+/gi, '$1=[redacted]')
|
|
46
|
+
.replace(/\b(bearer\s+)[A-Za-z0-9._~+/-]+=*/gi, '$1[redacted]')
|
|
47
|
+
.replace(/([?&](?:key|token|api[_-]?key|access[_-]?token|secret)=)[^&#\s]+/gi, '$1[redacted]')
|
|
48
|
+
}
|
|
49
|
+
|
|
41
50
|
function safeMessage(error) {
|
|
42
51
|
const source = error instanceof Error ? error.message : String(error ?? '')
|
|
43
|
-
return source
|
|
52
|
+
return sanitizeSecretString(source).slice(0, 160)
|
|
44
53
|
}
|
|
45
54
|
|
|
46
55
|
export function createCredentialResolver(ctx) {
|
package/lib/diff-notifier.js
CHANGED
|
@@ -104,6 +104,43 @@ export function extractDiffEvents(results, options = {}) {
|
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
|
|
107
|
+
export function isLoopbackIPv4(address) {
|
|
108
|
+
if (typeof address !== 'string' || !address.startsWith('127.')) return false
|
|
109
|
+
const parts = address.split('.')
|
|
110
|
+
if (parts.length !== 4) return false
|
|
111
|
+
return parts.every((part) => /^\d{1,3}$/.test(part) && Number(part) >= 0 && Number(part) <= 255)
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export function isForbiddenWebhookHost(rawHostname) {
|
|
115
|
+
const host = String(rawHostname || '').toLowerCase().replace(/^\[|\]$/g, '')
|
|
116
|
+
if (host === '169.254.169.254' || host.startsWith('169.254.')) return true
|
|
117
|
+
if (host.startsWith('::ffff:')) {
|
|
118
|
+
const v4 = host.slice('::ffff:'.length)
|
|
119
|
+
if (v4.startsWith('169.254.') || v4.startsWith('a9fe:')) return true
|
|
120
|
+
if (v4 === '100.100.100.200' || v4 === '6464:64c8') return true
|
|
121
|
+
}
|
|
122
|
+
if (host.startsWith('fe80:') || host.startsWith('fe80::')) return true
|
|
123
|
+
if (host === '100.100.100.200') return true
|
|
124
|
+
if (host === '0.0.0.0' || host === '255.255.255.255' || host === '::' || host === '0:0:0:0:0:0:0:0') return true
|
|
125
|
+
if (
|
|
126
|
+
host === 'metadata.google.internal' ||
|
|
127
|
+
host === 'instance-data' ||
|
|
128
|
+
host === 'metadata' ||
|
|
129
|
+
host.endsWith('.metadata.google.internal') ||
|
|
130
|
+
host.endsWith('.metadata')
|
|
131
|
+
) return true
|
|
132
|
+
return false
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export function isLoopbackHost(rawHostname) {
|
|
136
|
+
const host = String(rawHostname || '').toLowerCase().replace(/^\[|\]$/g, '')
|
|
137
|
+
if (host === '127.0.0.1' || host === 'localhost' || host === 'localhost.' || host === '::1') return true
|
|
138
|
+
if (host.startsWith('::ffff:')) {
|
|
139
|
+
return isLoopbackIPv4(host.slice('::ffff:'.length))
|
|
140
|
+
}
|
|
141
|
+
return isLoopbackIPv4(host)
|
|
142
|
+
}
|
|
143
|
+
|
|
107
144
|
export async function sendWebhookNotification(url, secret, payload, { fetchImpl = globalThis.fetch, timeoutMs = 5000 } = {}) {
|
|
108
145
|
if (typeof url !== "string" || !url.trim()) {
|
|
109
146
|
return { ok: false, error: "URL is required" }
|
|
@@ -115,16 +152,10 @@ export async function sendWebhookNotification(url, secret, payload, { fetchImpl
|
|
|
115
152
|
return { ok: false, error: "URL must use http or https protocol" }
|
|
116
153
|
}
|
|
117
154
|
const hostname = parsedUrl.hostname.toLowerCase().replace(/^\[|\]$/g, '')
|
|
118
|
-
if (
|
|
119
|
-
hostname === '169.254.169.254' ||
|
|
120
|
-
hostname.startsWith('169.254.') ||
|
|
121
|
-
hostname === 'metadata.google.internal' ||
|
|
122
|
-
hostname === '0.0.0.0' ||
|
|
123
|
-
hostname === '255.255.255.255'
|
|
124
|
-
) {
|
|
155
|
+
if (isForbiddenWebhookHost(hostname)) {
|
|
125
156
|
return { ok: false, error: "webhook url targets a forbidden metadata or link-local address" }
|
|
126
157
|
}
|
|
127
|
-
const isLoopback =
|
|
158
|
+
const isLoopback = isLoopbackHost(hostname)
|
|
128
159
|
if (parsedUrl.protocol === "http:" && !isLoopback) {
|
|
129
160
|
return { ok: false, error: "insecure http webhook url is only permitted on loopback" }
|
|
130
161
|
}
|
|
@@ -146,18 +177,29 @@ export async function sendWebhookNotification(url, secret, payload, { fetchImpl
|
|
|
146
177
|
const controller = typeof AbortController !== "undefined" ? new AbortController() : null
|
|
147
178
|
const timer = controller ? setTimeout(() => controller.abort(new Error("Webhook request timed out")), timeoutMs) : null
|
|
148
179
|
|
|
180
|
+
let res = null
|
|
149
181
|
try {
|
|
150
|
-
|
|
182
|
+
res = await fetchImpl(parsedUrl.toString(), {
|
|
151
183
|
method: "POST",
|
|
152
184
|
headers,
|
|
153
185
|
body,
|
|
154
186
|
...(controller ? { signal: controller.signal } : {}),
|
|
155
187
|
})
|
|
188
|
+
if (typeof res?.body?.cancel === "function") {
|
|
189
|
+
try { await res.body.cancel() } catch (_err) { /* best-effort cancel */ }
|
|
190
|
+
} else if (typeof res?.arrayBuffer === "function") {
|
|
191
|
+
try { await res.arrayBuffer() } catch (_err) { /* best-effort cancel */ }
|
|
192
|
+
}
|
|
156
193
|
if (timer) clearTimeout(timer)
|
|
157
194
|
return { ok: res.ok, status: res.status }
|
|
158
195
|
} catch (err) {
|
|
196
|
+
if (res && typeof res.body?.cancel === "function") {
|
|
197
|
+
try { await res.body.cancel() } catch (_err) { /* best-effort cancel */ }
|
|
198
|
+
}
|
|
159
199
|
if (timer) clearTimeout(timer)
|
|
160
200
|
return { ok: false, error: err?.message || String(err) }
|
|
201
|
+
} finally {
|
|
202
|
+
if (timer) clearTimeout(timer)
|
|
161
203
|
}
|
|
162
204
|
}
|
|
163
205
|
|
|
@@ -195,17 +237,37 @@ export async function dispatchDiffNotifications(ctx, config, results, { fetchImp
|
|
|
195
237
|
|
|
196
238
|
let webhookResult = null
|
|
197
239
|
if (config?.diffWebhookUrl && typeof config.diffWebhookUrl === "string" && config.diffWebhookUrl.trim()) {
|
|
198
|
-
let secret =
|
|
199
|
-
const secretRef = config.diffWebhookSecretRef ||
|
|
240
|
+
let secret = ''
|
|
241
|
+
const secretRef = (config.diffWebhookSecretRef || '').trim()
|
|
200
242
|
if (secretRef) {
|
|
243
|
+
let resolvedSecret = ''
|
|
201
244
|
try {
|
|
202
|
-
const
|
|
203
|
-
if (
|
|
245
|
+
const creds = ctx?.credentials || ctx?.get?.('credentials')
|
|
246
|
+
if (creds?.resolve) {
|
|
247
|
+
const res = await creds.resolve(secretRef)
|
|
248
|
+
resolvedSecret = typeof res === 'string' ? res : res?.value || ''
|
|
249
|
+
}
|
|
204
250
|
} catch (err) {
|
|
205
|
-
if (typeof ctx?.logger?.
|
|
206
|
-
ctx.logger.
|
|
251
|
+
if (typeof ctx?.logger?.warn === 'function') {
|
|
252
|
+
ctx.logger.warn('[dsh-model-sync] webhook secret resolution failed:', err)
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
if (!resolvedSecret) {
|
|
256
|
+
// FAIL-CLOSED: do not dispatch unsigned webhook when secret was expected (#204)
|
|
257
|
+
if (typeof ctx?.logger?.warn === 'function') {
|
|
258
|
+
ctx.logger.warn(`[dsh-model-sync] webhook secret resolution failed for '${secretRef}', skipping webhook dispatch (fail-closed)`)
|
|
259
|
+
}
|
|
260
|
+
return {
|
|
261
|
+
ok: false,
|
|
262
|
+
error: 'credential_resolution_failed',
|
|
263
|
+
message: `Failed to resolve webhook secret credential reference '${secretRef}'`,
|
|
264
|
+
events,
|
|
265
|
+
dispatched: false,
|
|
207
266
|
}
|
|
208
267
|
}
|
|
268
|
+
secret = resolvedSecret
|
|
269
|
+
} else if (config.diffWebhookSecret) {
|
|
270
|
+
secret = String(config.diffWebhookSecret).trim()
|
|
209
271
|
}
|
|
210
272
|
webhookResult = await sendWebhookNotification(
|
|
211
273
|
config.diffWebhookUrl,
|
package/lib/generic-adapter.js
CHANGED
|
@@ -109,7 +109,12 @@ async function resolveGenericRequest(profile, { resolveCredential, fetchImpl, si
|
|
|
109
109
|
const apiKey = keyRef ? await resolveCredential(keyRef) : ''
|
|
110
110
|
const response = await fetchImpl(endpointFor(profile.baseURL), { headers: requestHeaders(apiKey, etag, lastModified), signal })
|
|
111
111
|
if (response.status === 304) return response
|
|
112
|
-
if (!response.ok)
|
|
112
|
+
if (!response.ok) {
|
|
113
|
+
if (typeof response.body?.cancel === "function") {
|
|
114
|
+
try { await response.body.cancel() } catch (_err) { /* best-effort cancel */ }
|
|
115
|
+
}
|
|
116
|
+
throw catalogRequestError(response.status, response)
|
|
117
|
+
}
|
|
113
118
|
return response
|
|
114
119
|
}
|
|
115
120
|
|
|
@@ -125,6 +130,9 @@ export async function discoverOpenAIModels(profile, {
|
|
|
125
130
|
}
|
|
126
131
|
const response = await resolveGenericRequest(profile, { resolveCredential, fetchImpl, signal, etag, lastModified })
|
|
127
132
|
if (response.status === 304) {
|
|
133
|
+
if (typeof response.body?.cancel === "function") {
|
|
134
|
+
try { await response.body.cancel() } catch (_err) { /* best-effort cancel */ }
|
|
135
|
+
}
|
|
128
136
|
return { notModified: true, etag: response.headers?.get?.('etag') || etag, lastModified: response.headers?.get?.('last-modified') || lastModified }
|
|
129
137
|
}
|
|
130
138
|
const respEtag = response.headers?.get?.('etag') || undefined
|
package/lib/history.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { sanitizeSecretString } from './credentials.js'
|
|
1
2
|
import { normalizeModels } from './models.js'
|
|
2
3
|
|
|
3
4
|
export const DEFAULT_HISTORY_LIMIT = 50
|
|
@@ -41,7 +42,7 @@ function catalog(provider, models) {
|
|
|
41
42
|
|
|
42
43
|
function message(value) {
|
|
43
44
|
if (value === undefined || value === null) return undefined
|
|
44
|
-
return
|
|
45
|
+
return sanitizeSecretString(value).slice(0, MAX_MESSAGE_LENGTH)
|
|
45
46
|
}
|
|
46
47
|
|
|
47
48
|
function diffFor(provider, diff) {
|
package/lib/http.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { isForbiddenWebhookHost, isLoopbackHost } from './diff-notifier.js'
|
|
1
2
|
import crypto from 'node:crypto'
|
|
2
3
|
import { validateAliasName } from './alias-resolver.js'
|
|
3
4
|
import { validatePolicy } from './policy.js'
|
|
@@ -15,22 +16,17 @@ function normalizeWebhookTestRequest(body) {
|
|
|
15
16
|
throw Object.assign(new Error("URL must use http or https protocol"), { code: "INVALID_REQUEST" })
|
|
16
17
|
}
|
|
17
18
|
const hostname = parsed.hostname.toLowerCase().replace(/^\[|\]$/g, '')
|
|
18
|
-
|
|
19
|
+
if (isForbiddenWebhookHost(hostname)) {
|
|
20
|
+
throw Object.assign(new Error("webhook url targets a forbidden metadata or link-local address"), { code: "INVALID_REQUEST" })
|
|
21
|
+
}
|
|
22
|
+
const isLoopback = isLoopbackHost(hostname)
|
|
19
23
|
if (parsed.protocol === "http:" && !isLoopback) {
|
|
20
24
|
throw Object.assign(new Error("insecure http webhook url is only permitted on loopback"), { code: "INVALID_REQUEST" })
|
|
21
25
|
}
|
|
22
|
-
if (
|
|
23
|
-
hostname === '169.254.169.254' ||
|
|
24
|
-
hostname.startsWith('169.254.') ||
|
|
25
|
-
hostname === 'metadata.google.internal' ||
|
|
26
|
-
hostname === '0.0.0.0' ||
|
|
27
|
-
hostname === '255.255.255.255'
|
|
28
|
-
) {
|
|
29
|
-
throw Object.assign(new Error("webhook url targets a forbidden metadata or link-local address"), { code: "INVALID_REQUEST" })
|
|
30
|
-
}
|
|
31
26
|
return {
|
|
32
27
|
url: parsed.toString(),
|
|
33
28
|
secret: typeof body.secret === "string" ? body.secret.trim() : "",
|
|
29
|
+
...(body.secretRef && typeof body.secretRef === "string" ? { secretRef: body.secretRef.trim() } : {}),
|
|
34
30
|
}
|
|
35
31
|
}
|
|
36
32
|
|
|
@@ -42,14 +38,17 @@ function json(res, status, body) {
|
|
|
42
38
|
}
|
|
43
39
|
|
|
44
40
|
export function isLoopbackAddress(ip) {
|
|
45
|
-
|
|
46
|
-
const clean = ip.toLowerCase().replace(/^\[|\]$/g, '')
|
|
47
|
-
if (clean === '127.0.0.1' || clean === '::1' || clean === '::ffff:127.0.0.1' || clean === 'localhost') return true
|
|
48
|
-
if (clean.startsWith('127.') || clean.startsWith('::ffff:127.')) return true
|
|
49
|
-
return false
|
|
41
|
+
return isLoopbackHost(ip)
|
|
50
42
|
}
|
|
51
43
|
|
|
52
|
-
export function trusted(req, { requireLoopback = false } = {}) {
|
|
44
|
+
export function trusted(req, { requireLoopback = false, ctx = null } = {}) {
|
|
45
|
+
// 1. Core connection rejection check (#237)
|
|
46
|
+
const conn = ctx?.connection || ctx?.get?.('connection') || req?.connection
|
|
47
|
+
if (conn && typeof conn.requestRejection === 'function') {
|
|
48
|
+
const rejection = conn.requestRejection(req)
|
|
49
|
+
if (rejection !== undefined) return false
|
|
50
|
+
}
|
|
51
|
+
|
|
53
52
|
const remoteAddress = req?.socket?.remoteAddress
|
|
54
53
|
if (requireLoopback || req?.method === 'POST') {
|
|
55
54
|
if (remoteAddress && !isLoopbackAddress(remoteAddress)) return false
|
|
@@ -66,7 +65,7 @@ export function trusted(req, { requireLoopback = false } = {}) {
|
|
|
66
65
|
const url = new URL(origin)
|
|
67
66
|
if (url.host !== host) return false
|
|
68
67
|
if (requireLoopback || req?.method === 'POST') {
|
|
69
|
-
if (!isLoopbackAddress(url.hostname)
|
|
68
|
+
if (!isLoopbackAddress(url.hostname)) return false
|
|
70
69
|
}
|
|
71
70
|
} catch { return false }
|
|
72
71
|
}
|
|
@@ -77,7 +76,7 @@ export function trusted(req, { requireLoopback = false } = {}) {
|
|
|
77
76
|
const url = new URL(referer)
|
|
78
77
|
if (url.host !== host) return false
|
|
79
78
|
if (requireLoopback || req?.method === 'POST') {
|
|
80
|
-
if (!isLoopbackAddress(url.hostname)
|
|
79
|
+
if (!isLoopbackAddress(url.hostname)) return false
|
|
81
80
|
}
|
|
82
81
|
} catch { return false }
|
|
83
82
|
}
|
|
@@ -273,8 +272,18 @@ export function registerHttpApi(ctx, synchronizer, { getSchedulerStatus = () =>
|
|
|
273
272
|
return []
|
|
274
273
|
}
|
|
275
274
|
const makeHandler = (route) => async (req, res) => {
|
|
275
|
+
// 1. Core connection check on all routes (#237)
|
|
276
|
+
const conn = ctx?.connection || ctx?.get?.('connection') || req?.connection
|
|
277
|
+
if (conn && typeof conn.requestRejection === 'function') {
|
|
278
|
+
const rejection = conn.requestRejection(req)
|
|
279
|
+
if (rejection !== undefined) {
|
|
280
|
+
const statusCode = typeof rejection === 'number' ? rejection : 401
|
|
281
|
+
return json(res, statusCode, { error: { code: 'unauthorized', message: 'dsh-model-sync: core connection rejected request' } })
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
276
285
|
const isPublicStatus = route === 'status' && req.method === 'GET'
|
|
277
|
-
if (!trusted(req, { requireLoopback: !isPublicStatus })) return json(res, 403, { error: { code: 'forbidden', message: 'dsh-model-sync: cross-origin or non-loopback request rejected' } })
|
|
286
|
+
if (!trusted(req, { requireLoopback: !isPublicStatus, ctx })) return json(res, 403, { error: { code: 'forbidden', message: 'dsh-model-sync: cross-origin or non-loopback request rejected' } })
|
|
278
287
|
if (route === 'status') {
|
|
279
288
|
if (req.method !== 'GET') return json(res, 405, { error: { code: 'method', message: 'GET only' } })
|
|
280
289
|
const payload = { ...synchronizer.status(), scheduler: getSchedulerStatus(), providers: synchronizer.listProviders() }
|
package/lib/inventory.js
CHANGED
|
@@ -66,17 +66,25 @@ export function listApiKeyProviders(llm, settings) {
|
|
|
66
66
|
const seen = new Set()
|
|
67
67
|
for (const entry of directory) {
|
|
68
68
|
let profile = profiles[entry.provider]
|
|
69
|
+
let configured = Object.prototype.hasOwnProperty.call(profiles, entry.provider)
|
|
69
70
|
if (!profile && (entry.provider === 'commandcode' || entry.settingsNs === 'llm-commandcode')) {
|
|
70
71
|
const cc = sectionOf(settings, 'llm-commandcode')
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
72
|
+
if (cc && typeof cc === 'object' && Object.keys(cc).length > 0) {
|
|
73
|
+
const hasCredentials = Boolean(cc.apiKey || cc.apiKeyEnv || cc.credentialRef || cc.apiKeyRef)
|
|
74
|
+
const hasModels = Array.isArray(cc.visibleModels) && cc.visibleModels.length > 0
|
|
75
|
+
const hasBase = Boolean(cc.apiBase)
|
|
76
|
+
if (hasCredentials || hasModels || hasBase) {
|
|
77
|
+
configured = true
|
|
78
|
+
profile = {
|
|
79
|
+
apiKeyEnv: cc.apiKeyEnv ?? (hasCredentials ? 'COMMANDCODE_API_KEY' : ''),
|
|
80
|
+
credentialRef: cc.apiKeyEnv ?? cc.credentialRef ?? '',
|
|
81
|
+
baseURL: cc.apiBase ?? 'https://api.commandcode.ai',
|
|
82
|
+
models: cc.visibleModels ?? [],
|
|
83
|
+
}
|
|
84
|
+
}
|
|
76
85
|
}
|
|
77
|
-
} else {
|
|
78
|
-
profile = profile ?? {}
|
|
79
86
|
}
|
|
87
|
+
profile = profile ?? {}
|
|
80
88
|
if (!isApiKeyProvider(entry, profile) || seen.has(entry.provider)) continue
|
|
81
89
|
seen.add(entry.provider)
|
|
82
90
|
out.push({
|
|
@@ -85,7 +93,7 @@ export function listApiKeyProviders(llm, settings) {
|
|
|
85
93
|
settingsNs: entry.settingsNs,
|
|
86
94
|
settingsPath: [...(entry.settingsPath ?? [])],
|
|
87
95
|
declared: entry.declared === true,
|
|
88
|
-
configured
|
|
96
|
+
configured,
|
|
89
97
|
live: live.has(entry.provider),
|
|
90
98
|
apiKeyEnv: profile.apiKeyEnv ?? profile.credentialRef ?? profile.apiKeyRef ?? '',
|
|
91
99
|
credentialRef: profile.credentialRef ?? profile.apiKeyRef ?? profile.apiKeyEnv ?? '',
|
package/lib/models.js
CHANGED
|
@@ -140,13 +140,13 @@ function normalizeCapabilities(row) {
|
|
|
140
140
|
|
|
141
141
|
// Model ID & name heuristic markers for modern models
|
|
142
142
|
const idStr = String(row?.id ?? row?.slug ?? row?.model ?? '').toLowerCase()
|
|
143
|
-
if (
|
|
143
|
+
if (result.reasoning === undefined && (/(^|[_-])(o1|o3|r1|qwq|deepseek-r1|thinking)($|[_-])/i.test(idStr) || idStr.includes('-r1') || idStr.includes('reasoning') || idStr.includes('thinking'))) {
|
|
144
144
|
result.reasoning = true
|
|
145
145
|
}
|
|
146
|
-
if (
|
|
146
|
+
if (result.code === undefined && (/(^|[_-])(coder|code|starcoder|codellama)($|[_-])/i.test(idStr) || idStr.includes('coder') || idStr.includes('code-'))) {
|
|
147
147
|
result.code = true
|
|
148
148
|
}
|
|
149
|
-
if (
|
|
149
|
+
if (result.vision === undefined && (/(^|[_-])(vision|vl)($|[_-])/i.test(idStr) || idStr.includes('-vl') || idStr.includes('-vision'))) {
|
|
150
150
|
result.vision = true
|
|
151
151
|
}
|
|
152
152
|
|
package/lib/policy.js
CHANGED
|
@@ -41,6 +41,12 @@ function isDangerousPattern(pattern) {
|
|
|
41
41
|
if (/\([^\)]*([+*]|\{[0-9]+,[0-9]*\})[^\)]*\)([+*]|\{[0-9]+,[0-9]*\})/.test(pattern)) return true
|
|
42
42
|
// Check for multiple consecutive open-ended wildcards (e.g. .*.*)
|
|
43
43
|
if (/(\.\*|\.\+){2,}/.test(pattern)) return true
|
|
44
|
+
// Check for quantified alternation groups like (a|aa)+, (a|b)+, (a|b)*, (a|aa){2,}
|
|
45
|
+
let simplified = pattern
|
|
46
|
+
while (/\(\(([^()]+)\)\)/.test(simplified)) {
|
|
47
|
+
simplified = simplified.replace(/\(\(([^()]+)\)\)/g, '($1)')
|
|
48
|
+
}
|
|
49
|
+
if (/\((?:[^()]*\|)+[^()]*\)\s*([+*]|\{[0-9]+,[0-9]*\})/.test(simplified)) return true
|
|
44
50
|
return false
|
|
45
51
|
}
|
|
46
52
|
|
package/lib/reliability.js
CHANGED
|
@@ -1,5 +1,39 @@
|
|
|
1
1
|
const RETRYABLE_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504, 522, 524])
|
|
2
|
-
const RETRYABLE_CODES = new Set([
|
|
2
|
+
const RETRYABLE_CODES = new Set([
|
|
3
|
+
'ECONNRESET',
|
|
4
|
+
'ECONNREFUSED',
|
|
5
|
+
'ETIMEDOUT',
|
|
6
|
+
'EAI_AGAIN',
|
|
7
|
+
'UND_ERR_CONNECT_TIMEOUT',
|
|
8
|
+
'UND_ERR_HEADERS_TIMEOUT',
|
|
9
|
+
'UND_ERR_BODY_TIMEOUT',
|
|
10
|
+
'UND_ERR_SOCKET',
|
|
11
|
+
'UND_ERR_DESTROYED',
|
|
12
|
+
'UND_ERR_CLOSED',
|
|
13
|
+
'EHOSTUNREACH',
|
|
14
|
+
'ENETUNREACH',
|
|
15
|
+
'EPIPE',
|
|
16
|
+
])
|
|
17
|
+
|
|
18
|
+
function isTimeout(err) {
|
|
19
|
+
if (!err) return false
|
|
20
|
+
if (err.name === 'TimeoutError') return true
|
|
21
|
+
if (
|
|
22
|
+
err.code === 'ETIMEDOUT' ||
|
|
23
|
+
err.code === 'UND_ERR_CONNECT_TIMEOUT' ||
|
|
24
|
+
err.code === 'UND_ERR_HEADERS_TIMEOUT' ||
|
|
25
|
+
err.code === 'UND_ERR_BODY_TIMEOUT'
|
|
26
|
+
) return true
|
|
27
|
+
const msg = String(err.message || '').toLowerCase()
|
|
28
|
+
return msg.includes('timed out') || msg.includes('timeout')
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function isAbort(err) {
|
|
32
|
+
if (!err) return false
|
|
33
|
+
if (err.name === 'AbortError') return true
|
|
34
|
+
const msg = String(err.message || '').toLowerCase()
|
|
35
|
+
return msg.includes('aborted') || msg.includes('the operation was aborted')
|
|
36
|
+
}
|
|
3
37
|
|
|
4
38
|
export function statusOfError(error) {
|
|
5
39
|
const direct = Number(error?.status ?? error?.statusCode)
|
|
@@ -10,10 +44,36 @@ export function statusOfError(error) {
|
|
|
10
44
|
|
|
11
45
|
export function isRetryableError(error) {
|
|
12
46
|
if (!error) return false
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
47
|
+
|
|
48
|
+
let curr = error
|
|
49
|
+
let depth = 0
|
|
50
|
+
let hasTimeout = false
|
|
51
|
+
let hasAbort = false
|
|
52
|
+
|
|
53
|
+
while (curr && depth < 6) {
|
|
54
|
+
if (isTimeout(curr)) {
|
|
55
|
+
hasTimeout = true
|
|
56
|
+
break
|
|
57
|
+
}
|
|
58
|
+
if (isAbort(curr)) {
|
|
59
|
+
hasAbort = true
|
|
60
|
+
}
|
|
61
|
+
if (curr.code && RETRYABLE_CODES.has(curr.code)) {
|
|
62
|
+
return true
|
|
63
|
+
}
|
|
64
|
+
const status = statusOfError(curr)
|
|
65
|
+
if (status !== undefined) {
|
|
66
|
+
if (RETRYABLE_STATUSES.has(status)) return true
|
|
67
|
+
if (status >= 400 && status < 500) return false
|
|
68
|
+
}
|
|
69
|
+
curr = curr.cause
|
|
70
|
+
depth++
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
if (hasTimeout) return true
|
|
74
|
+
if (hasAbort) return false
|
|
75
|
+
|
|
76
|
+
return false
|
|
17
77
|
}
|
|
18
78
|
|
|
19
79
|
export function parseRetryAfter(response) {
|
|
@@ -46,7 +106,10 @@ export function catalogRequestError(status, response) {
|
|
|
46
106
|
const error = new Error(`model catalog request failed with HTTP ${status}`)
|
|
47
107
|
error.status = Number(status)
|
|
48
108
|
const retryAfterMs = parseRetryAfter(response)
|
|
49
|
-
if (retryAfterMs !== undefined)
|
|
109
|
+
if (retryAfterMs !== undefined) {
|
|
110
|
+
error.retryAfterMs = retryAfterMs
|
|
111
|
+
error.retryAt = new Date(Date.now() + retryAfterMs).toISOString()
|
|
112
|
+
}
|
|
50
113
|
return error
|
|
51
114
|
}
|
|
52
115
|
|
|
@@ -105,6 +168,7 @@ export async function retryWithBackoff(operation, {
|
|
|
105
168
|
attempts = 3,
|
|
106
169
|
baseDelayMs = 250,
|
|
107
170
|
maxDelayMs = 4000,
|
|
171
|
+
maxRetryAfterMs = 300000,
|
|
108
172
|
jitter = true,
|
|
109
173
|
signal,
|
|
110
174
|
onRetry,
|
|
@@ -116,14 +180,22 @@ export async function retryWithBackoff(operation, {
|
|
|
116
180
|
return await operation({ attempt })
|
|
117
181
|
} catch (error) {
|
|
118
182
|
if (attempt >= total || !isRetryableError(error)) throw error
|
|
119
|
-
const exponential = Math.max(0, Number(baseDelayMs) || 0) * (2 ** (attempt - 1))
|
|
120
|
-
// Full jitter: randomized between [0.5, 1.0] of calculated exponential backoff
|
|
121
|
-
const jitterFactor = jitter ? (0.5 + Math.random() * 0.5) : 1
|
|
122
|
-
const calculatedDelayMs = Math.min(Math.max(0, Number(maxDelayMs) || 0), Math.round(exponential * jitterFactor))
|
|
123
183
|
const retryAfterMs = Number(error?.retryAfterMs)
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
184
|
+
let delayMs
|
|
185
|
+
if (Number.isFinite(retryAfterMs) && retryAfterMs > 0) {
|
|
186
|
+
if (retryAfterMs > maxRetryAfterMs) {
|
|
187
|
+
if (!error.retryAt) {
|
|
188
|
+
error.retryAt = new Date(Date.now() + retryAfterMs).toISOString()
|
|
189
|
+
}
|
|
190
|
+
throw error
|
|
191
|
+
}
|
|
192
|
+
delayMs = retryAfterMs
|
|
193
|
+
} else {
|
|
194
|
+
const exponential = Math.max(0, Number(baseDelayMs) || 0) * (2 ** (attempt - 1))
|
|
195
|
+
// Full jitter: randomized between [0.5, 1.0] of calculated exponential backoff
|
|
196
|
+
const jitterFactor = jitter ? (0.5 + Math.random() * 0.5) : 1
|
|
197
|
+
delayMs = Math.min(Math.max(0, Number(maxDelayMs) || 0), Math.round(exponential * jitterFactor))
|
|
198
|
+
}
|
|
127
199
|
onRetry?.({ attempt, nextAttempt: attempt + 1, delayMs, error })
|
|
128
200
|
await delay(delayMs, signal)
|
|
129
201
|
}
|
|
@@ -142,4 +214,4 @@ export async function mapWithConcurrency(items, worker, concurrency = 4) {
|
|
|
142
214
|
}
|
|
143
215
|
}))
|
|
144
216
|
return values
|
|
145
|
-
}
|
|
217
|
+
}
|
package/lib/reporting.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { sanitizeSecretString } from './credentials.js'
|
|
1
2
|
import { createHash } from 'node:crypto'
|
|
2
3
|
|
|
3
4
|
export const DEFAULT_NOTIFICATION_LIMIT = 50
|
|
@@ -14,10 +15,7 @@ export function notificationLimit(value) {
|
|
|
14
15
|
}
|
|
15
16
|
|
|
16
17
|
function safeMessage(value) {
|
|
17
|
-
return
|
|
18
|
-
.replace(/(api[_-]?key|token|secret|authorization|password)\s*[:=]\s*[^\s,;]+/gi, '$1=[redacted]')
|
|
19
|
-
.replace(/([?&](?:key|token|api[_-]?key|access[_-]?token)=)[^&#\s]+/gi, '$1[redacted]')
|
|
20
|
-
.slice(0, 240)
|
|
18
|
+
return sanitizeSecretString(value).slice(0, 240)
|
|
21
19
|
}
|
|
22
20
|
|
|
23
21
|
function stable(value) {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { sanitizeSecretString } from './credentials.js'
|
|
1
2
|
import { statusOfError } from './reliability.js'
|
|
2
3
|
import { historyEntry } from './history.js'
|
|
3
4
|
import { normalizeModels } from './models.js'
|
|
@@ -89,7 +90,7 @@ export function errorMessage(error) {
|
|
|
89
90
|
}
|
|
90
91
|
|
|
91
92
|
export function diagnosticMessage(error) {
|
|
92
|
-
return errorMessage(error)
|
|
93
|
+
return sanitizeSecretString(errorMessage(error)).slice(0, 160)
|
|
93
94
|
}
|
|
94
95
|
|
|
95
96
|
export function adapterErrorType(error) {
|
|
@@ -1,30 +1,94 @@
|
|
|
1
|
-
export function createModelProber({
|
|
1
|
+
export function createModelProber({
|
|
2
|
+
health,
|
|
3
|
+
getProviderCatalog = () => [],
|
|
4
|
+
probeModel,
|
|
5
|
+
inferModel,
|
|
6
|
+
} = {}) {
|
|
2
7
|
const modelProbes = new Map()
|
|
8
|
+
const probeFn = probeModel ?? inferModel
|
|
3
9
|
|
|
4
|
-
async function tryModel({ provider, model } = {}) {
|
|
10
|
+
async function tryModel({ provider, model, signal } = {}) {
|
|
5
11
|
if (typeof provider !== 'string' || !provider) throw new Error('provider is required')
|
|
6
12
|
if (typeof model !== 'string' || !model) throw new Error('model is required')
|
|
7
|
-
const start = Date.now()
|
|
8
|
-
const result = await health({ provider })
|
|
9
|
-
const entry = result?.results?.find((r) => r.provider === provider)
|
|
10
|
-
const providerOk = entry?.status === 'ok'
|
|
11
13
|
|
|
12
14
|
// Verify model in provider catalog if catalog is available
|
|
13
15
|
const catalog = typeof getProviderCatalog === 'function' ? getProviderCatalog(provider) : []
|
|
14
16
|
const hasCatalog = Array.isArray(catalog) && catalog.length > 0
|
|
15
17
|
const inCatalog = !hasCatalog || catalog.some((m) => (typeof m === 'string' ? m : m?.id) === model)
|
|
18
|
+
if (hasCatalog && !inCatalog) {
|
|
19
|
+
const probeResult = {
|
|
20
|
+
provider,
|
|
21
|
+
model,
|
|
22
|
+
ok: false,
|
|
23
|
+
latencyMs: 0,
|
|
24
|
+
status: 'model_not_found',
|
|
25
|
+
error: `model "${model}" not found in provider catalog`,
|
|
26
|
+
}
|
|
27
|
+
modelProbes.set(`${provider}:${model}`, {
|
|
28
|
+
latencyMs: 0,
|
|
29
|
+
ok: false,
|
|
30
|
+
status: 'model_not_found',
|
|
31
|
+
timestamp: Date.now(),
|
|
32
|
+
})
|
|
33
|
+
return probeResult
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const start = Date.now()
|
|
37
|
+
if (typeof probeFn === 'function') {
|
|
38
|
+
try {
|
|
39
|
+
const probeRes = await probeFn({ provider, model, signal })
|
|
40
|
+
const latencyMs = Math.max(1, Date.now() - start)
|
|
41
|
+
const isOk = probeRes?.ok !== false
|
|
42
|
+
const status = isOk ? 'ok' : (probeRes?.status || 'error')
|
|
43
|
+
const errorMsg = isOk ? undefined : (probeRes?.error || 'inference probe failed')
|
|
44
|
+
const probeResult = {
|
|
45
|
+
provider,
|
|
46
|
+
model,
|
|
47
|
+
ok: isOk,
|
|
48
|
+
latencyMs,
|
|
49
|
+
status,
|
|
50
|
+
...(errorMsg ? { error: errorMsg } : {}),
|
|
51
|
+
}
|
|
52
|
+
modelProbes.set(`${provider}:${model}`, {
|
|
53
|
+
latencyMs,
|
|
54
|
+
ok: isOk,
|
|
55
|
+
status,
|
|
56
|
+
timestamp: Date.now(),
|
|
57
|
+
})
|
|
58
|
+
return probeResult
|
|
59
|
+
} catch (err) {
|
|
60
|
+
const latencyMs = Math.max(1, Date.now() - start)
|
|
61
|
+
const status = err?.status ? `http_${err.status}` : (err?.code || 'error')
|
|
62
|
+
const probeResult = {
|
|
63
|
+
provider,
|
|
64
|
+
model,
|
|
65
|
+
ok: false,
|
|
66
|
+
latencyMs,
|
|
67
|
+
status,
|
|
68
|
+
error: err?.message || 'inference probe failed',
|
|
69
|
+
}
|
|
70
|
+
modelProbes.set(`${provider}:${model}`, {
|
|
71
|
+
latencyMs,
|
|
72
|
+
ok: false,
|
|
73
|
+
status,
|
|
74
|
+
timestamp: Date.now(),
|
|
75
|
+
})
|
|
76
|
+
return probeResult
|
|
77
|
+
}
|
|
78
|
+
}
|
|
16
79
|
|
|
17
|
-
|
|
18
|
-
const
|
|
19
|
-
const
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
80
|
+
// Fallback if no probeFn provided: check provider health
|
|
81
|
+
const healthResult = await health?.({ provider, model, signal })
|
|
82
|
+
const entry = healthResult?.results?.find((r) => r.provider === provider)
|
|
83
|
+
const providerOk = entry?.status === 'ok'
|
|
84
|
+
const latencyMs = Math.max(1, Date.now() - start)
|
|
85
|
+
const status = !providerOk ? (entry?.status ?? 'error') : 'ok'
|
|
86
|
+
const errorMsg = !providerOk ? (entry?.message || entry?.status || 'probe failed') : undefined
|
|
23
87
|
|
|
24
88
|
const probeResult = {
|
|
25
89
|
provider,
|
|
26
90
|
model,
|
|
27
|
-
ok,
|
|
91
|
+
ok: providerOk,
|
|
28
92
|
latencyMs,
|
|
29
93
|
status,
|
|
30
94
|
...(errorMsg ? { error: errorMsg } : {}),
|
|
@@ -32,7 +96,7 @@ export function createModelProber({ health, getProviderCatalog = () => [] } = {}
|
|
|
32
96
|
|
|
33
97
|
modelProbes.set(`${provider}:${model}`, {
|
|
34
98
|
latencyMs,
|
|
35
|
-
ok,
|
|
99
|
+
ok: providerOk,
|
|
36
100
|
status,
|
|
37
101
|
timestamp: Date.now(),
|
|
38
102
|
})
|
|
@@ -40,58 +104,74 @@ export function createModelProber({ health, getProviderCatalog = () => [] } = {}
|
|
|
40
104
|
return probeResult
|
|
41
105
|
}
|
|
42
106
|
|
|
43
|
-
async function batchTryModels({ provider, models } = {}) {
|
|
107
|
+
async function batchTryModels({ provider, models, signal, concurrency = 4 } = {}) {
|
|
44
108
|
if (typeof provider !== 'string' || !provider) throw new Error('provider is required')
|
|
45
109
|
if (!Array.isArray(models)) throw new Error('models must be an array')
|
|
46
110
|
const list = [...new Set(models.filter((m) => typeof m === 'string' && m.trim()).map((m) => m.trim()))].slice(0, 50)
|
|
47
111
|
|
|
48
|
-
|
|
112
|
+
if (typeof probeFn === 'function') {
|
|
113
|
+
// Execute true model-specific probes with bounded concurrency
|
|
114
|
+
const limit = Math.max(1, Math.min(10, Math.floor(Number(concurrency) || 4)))
|
|
115
|
+
const results = new Array(list.length)
|
|
116
|
+
let cursor = 0
|
|
117
|
+
|
|
118
|
+
await Promise.all(Array.from({ length: Math.min(limit, list.length || 1) }, async () => {
|
|
119
|
+
while (cursor < list.length) {
|
|
120
|
+
if (signal?.aborted) break
|
|
121
|
+
const index = cursor++
|
|
122
|
+
results[index] = await tryModel({ provider, model: list[index], signal })
|
|
123
|
+
}
|
|
124
|
+
}))
|
|
125
|
+
|
|
126
|
+
const reachable = results.filter((r) => r?.ok).length
|
|
127
|
+
return {
|
|
128
|
+
provider,
|
|
129
|
+
results,
|
|
130
|
+
summary: { total: results.length, reachable, unreachable: results.length - reachable },
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// If no probeFn provided, use provider health once (backwards compatibility)
|
|
49
135
|
const start = Date.now()
|
|
50
|
-
const healthResult = await health({ provider })
|
|
136
|
+
const healthResult = await health?.({ provider, signal })
|
|
51
137
|
const entry = healthResult?.results?.find((r) => r.provider === provider)
|
|
52
138
|
const providerOk = entry?.status === 'ok'
|
|
53
139
|
const catalog = typeof getProviderCatalog === 'function' ? getProviderCatalog(provider) : []
|
|
54
140
|
const hasCatalog = Array.isArray(catalog) && catalog.length > 0
|
|
55
|
-
const baseLatency = Date.now() - start
|
|
56
|
-
|
|
57
|
-
const results = []
|
|
58
|
-
const concurrency = 5
|
|
59
|
-
for (let i = 0; i < list.length; i += concurrency) {
|
|
60
|
-
const chunk = list.slice(i, i + concurrency)
|
|
61
|
-
const chunkResults = chunk.map((model) => {
|
|
62
|
-
const inCatalog = !hasCatalog || catalog.some((m) => (typeof m === 'string' ? m : m?.id) === model)
|
|
63
|
-
const ok = Boolean(providerOk && inCatalog)
|
|
64
|
-
const status = !providerOk ? (entry?.status ?? 'error') : (!inCatalog ? 'model_not_found' : 'ok')
|
|
65
|
-
const errorMsg = !providerOk
|
|
66
|
-
? (entry?.message || entry?.status || 'probe failed')
|
|
67
|
-
: (!inCatalog ? `model "${model}" not found in provider catalog` : undefined)
|
|
68
|
-
|
|
69
|
-
const itemResult = {
|
|
70
|
-
provider,
|
|
71
|
-
model,
|
|
72
|
-
ok,
|
|
73
|
-
latencyMs: baseLatency,
|
|
74
|
-
status,
|
|
75
|
-
...(errorMsg ? { error: errorMsg } : {}),
|
|
76
|
-
}
|
|
141
|
+
const baseLatency = Math.max(1, Date.now() - start)
|
|
77
142
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
143
|
+
const results = list.map((model) => {
|
|
144
|
+
const inCatalog = !hasCatalog || catalog.some((m) => (typeof m === 'string' ? m : m?.id) === model)
|
|
145
|
+
const ok = Boolean(providerOk && inCatalog)
|
|
146
|
+
const status = !providerOk ? (entry?.status ?? 'error') : (!inCatalog ? 'model_not_found' : 'ok')
|
|
147
|
+
const errorMsg = !providerOk
|
|
148
|
+
? (entry?.message || entry?.status || 'probe failed')
|
|
149
|
+
: (!inCatalog ? `model "${model}" not found in provider catalog` : undefined)
|
|
150
|
+
|
|
151
|
+
const itemResult = {
|
|
152
|
+
provider,
|
|
153
|
+
model,
|
|
154
|
+
ok,
|
|
155
|
+
latencyMs: baseLatency,
|
|
156
|
+
status,
|
|
157
|
+
...(errorMsg ? { error: errorMsg } : {}),
|
|
158
|
+
}
|
|
84
159
|
|
|
85
|
-
|
|
160
|
+
modelProbes.set(`${provider}:${model}`, {
|
|
161
|
+
latencyMs: baseLatency,
|
|
162
|
+
ok,
|
|
163
|
+
status,
|
|
164
|
+
timestamp: Date.now(),
|
|
86
165
|
})
|
|
87
|
-
|
|
88
|
-
|
|
166
|
+
|
|
167
|
+
return itemResult
|
|
168
|
+
})
|
|
89
169
|
|
|
90
170
|
const reachable = results.filter((r) => r.ok).length
|
|
91
171
|
return {
|
|
92
172
|
provider,
|
|
93
173
|
results,
|
|
94
|
-
summary: { total: results.length, reachable, unreachable: results.length - reachable }
|
|
174
|
+
summary: { total: results.length, reachable, unreachable: results.length - reachable },
|
|
95
175
|
}
|
|
96
176
|
}
|
|
97
177
|
|
|
@@ -104,4 +184,4 @@ export function createModelProber({ health, getProviderCatalog = () => [] } = {}
|
|
|
104
184
|
batchTryModels,
|
|
105
185
|
getModelProbes,
|
|
106
186
|
}
|
|
107
|
-
}
|
|
187
|
+
}
|
package/lib/synchronizer.js
CHANGED
|
@@ -31,7 +31,7 @@ import {
|
|
|
31
31
|
import { createConfigTransfer } from './synchronizer-transfer.js'
|
|
32
32
|
import { createModelProber } from './synchronizer-prober.js'
|
|
33
33
|
|
|
34
|
-
export function createModelSynchronizer(ctx, { getConfig = () => ({}), saveConfig = async () => {}, fetchImpl = fetch, adapterImplementations = {} } = {}) {
|
|
34
|
+
export function createModelSynchronizer(ctx, { getConfig = () => ({}), saveConfig = async () => {}, fetchImpl = fetch, adapterImplementations = {}, probeModel } = {}) {
|
|
35
35
|
const resolveCredential = createCredentialResolver(ctx)
|
|
36
36
|
const registry = createProviderAdapterRegistry({ resolveCredential, fetchImpl, adapters: adapterImplementations })
|
|
37
37
|
let running = null
|
|
@@ -467,6 +467,35 @@ export function createModelSynchronizer(ctx, { getConfig = () => ({}), saveConfi
|
|
|
467
467
|
const row = providers().find((item) => item.provider === p)
|
|
468
468
|
return row?.availableModels || row?.models || []
|
|
469
469
|
},
|
|
470
|
+
probeModel: probeModel ?? (async ({ provider, model, signal }) => {
|
|
471
|
+
const llm = llmOf(ctx)
|
|
472
|
+
if (typeof llm?.stream === "function") {
|
|
473
|
+
const stream = await llm.stream({
|
|
474
|
+
provider,
|
|
475
|
+
model,
|
|
476
|
+
messages: [{ role: "user", content: [{ type: "text", text: "ping" }] }],
|
|
477
|
+
}, { signal })
|
|
478
|
+
if (stream && typeof stream[Symbol.asyncIterator] === "function") {
|
|
479
|
+
for await (const _chunk of stream) {
|
|
480
|
+
break
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
return { ok: true }
|
|
484
|
+
}
|
|
485
|
+
const profile = profileOf(settingsOf(ctx), provider)
|
|
486
|
+
const adapter = selectAdapter(profile)
|
|
487
|
+
if (typeof adapter?.probeModel === "function") {
|
|
488
|
+
return adapter.probeModel(profile, model, { signal, fetchImpl })
|
|
489
|
+
}
|
|
490
|
+
const h = await health({ provider, model, signal })
|
|
491
|
+
const entry = h?.results?.find((r) => r.provider === provider)
|
|
492
|
+
if (entry?.status !== "ok") {
|
|
493
|
+
const err = new Error(entry?.message || "provider health failed: " + entry?.status)
|
|
494
|
+
if (entry?.status === "circuit-open") err.status = 503
|
|
495
|
+
throw err
|
|
496
|
+
}
|
|
497
|
+
return { ok: true }
|
|
498
|
+
}),
|
|
470
499
|
})
|
|
471
500
|
|
|
472
501
|
const rollbackHistory = (opts) => performHistoryRollback({ ctx, config: getConfig() ?? {}, providers: providers(), ...opts })
|
|
@@ -571,14 +600,18 @@ export function createModelSynchronizer(ctx, { getConfig = () => ({}), saveConfi
|
|
|
571
600
|
notifications: notificationLedger,
|
|
572
601
|
updateNotification: updateNotificationState,
|
|
573
602
|
rollbackHistory,
|
|
574
|
-
testWebhook: async ({ url, secret }) => {
|
|
603
|
+
testWebhook: async ({ url, secret, secretRef }) => {
|
|
575
604
|
let resolvedSecret = secret || ''
|
|
576
|
-
|
|
605
|
+
const targetRef = (secretRef || (typeof secret === 'string' && /^[A-Za-z_][A-Za-z0-9_]*$/.test(secret) ? secret : '')).trim()
|
|
606
|
+
if (targetRef) {
|
|
577
607
|
try {
|
|
578
|
-
const res = await (ctx?.credentials?.resolve ? ctx.credentials.resolve(
|
|
579
|
-
if (res) resolvedSecret = typeof res === 'string' ? res : res?.value ||
|
|
608
|
+
const res = await (ctx?.credentials?.resolve ? ctx.credentials.resolve(targetRef) : ctx?.get?.('credentials')?.resolve?.(targetRef))
|
|
609
|
+
if (res) resolvedSecret = typeof res === 'string' ? res : res?.value || ''
|
|
580
610
|
} catch (err) {
|
|
581
|
-
ctx?.logger?.
|
|
611
|
+
ctx?.logger?.warn?.('[dsh-model-sync] secret ref resolution failed in testWebhook:', err)
|
|
612
|
+
}
|
|
613
|
+
if (!resolvedSecret && secretRef) {
|
|
614
|
+
return { ok: false, error: `Failed to resolve webhook secret credential reference '${secretRef}'` }
|
|
582
615
|
}
|
|
583
616
|
}
|
|
584
617
|
return sendWebhookNotification(url, resolvedSecret, { event: 'model-sync.test', timestamp: Date.now(), test: true })
|
package/lib/updater.js
CHANGED
|
@@ -20,19 +20,33 @@ function header(request, name) {
|
|
|
20
20
|
return Array.isArray(value) ? value[0] : value
|
|
21
21
|
}
|
|
22
22
|
|
|
23
|
+
function isLoopbackIPv4(address) {
|
|
24
|
+
if (typeof address !== 'string' || !address.startsWith('127.')) return false
|
|
25
|
+
const parts = address.split('.')
|
|
26
|
+
if (parts.length !== 4) return false
|
|
27
|
+
return parts.every((part) => /^\d{1,3}$/.test(part) && Number(part) >= 0 && Number(part) <= 255)
|
|
28
|
+
}
|
|
29
|
+
|
|
23
30
|
export function isLoopback(value) {
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
address
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
)
|
|
31
|
+
if (typeof value !== 'string') return false
|
|
32
|
+
const address = value.toLowerCase().replace(/^\[|\]$/g, '')
|
|
33
|
+
if (address === 'localhost' || address === 'localhost.' || address === '::1') return true
|
|
34
|
+
if (address.startsWith('::ffff:')) {
|
|
35
|
+
return isLoopbackIPv4(address.slice('::ffff:'.length))
|
|
36
|
+
}
|
|
37
|
+
return isLoopbackIPv4(address)
|
|
32
38
|
}
|
|
33
39
|
|
|
34
|
-
export function isTrustedUpdateRequest(request) {
|
|
40
|
+
export function isTrustedUpdateRequest(request, targetCtx = null) {
|
|
35
41
|
if (header(request, UPDATE_HEADER) !== '1') return false
|
|
42
|
+
|
|
43
|
+
// 1. Core connection rejection check (#237)
|
|
44
|
+
const conn = targetCtx?.connection || targetCtx?.get?.('connection') || request?.connection
|
|
45
|
+
if (conn && typeof conn.requestRejection === 'function') {
|
|
46
|
+
const rejection = conn.requestRejection(request)
|
|
47
|
+
if (rejection !== undefined) return false
|
|
48
|
+
}
|
|
49
|
+
|
|
36
50
|
const remote = request.socket?.remoteAddress
|
|
37
51
|
if (!isLoopback(remote)) return false
|
|
38
52
|
const site = header(request, 'sec-fetch-site')
|
|
@@ -186,7 +200,12 @@ export async function latestVersion(packageName, registry = 'https://registry.np
|
|
|
186
200
|
const response = await fetch(`${registry.replace(/\/$/, '')}/${encodeURIComponent(packageName)}/latest`, {
|
|
187
201
|
signal: typeof AbortSignal !== 'undefined' && AbortSignal.timeout ? AbortSignal.timeout(8000) : undefined,
|
|
188
202
|
})
|
|
189
|
-
if (!response.ok)
|
|
203
|
+
if (!response.ok) {
|
|
204
|
+
if (typeof response.body?.cancel === "function") {
|
|
205
|
+
try { await response.body.cancel() } catch (_err) { /* best-effort cancel */ }
|
|
206
|
+
}
|
|
207
|
+
return undefined
|
|
208
|
+
}
|
|
190
209
|
const value = await response.json()
|
|
191
210
|
if (typeof value?.version !== 'string' || value.version === '') return undefined
|
|
192
211
|
latestCache = { packageName, registry, version: value.version, expiresAt: Date.now() + VERSION_CACHE_MS }
|
|
@@ -295,6 +314,21 @@ export function registerPluginUpdater(ctx, options = {}) {
|
|
|
295
314
|
|
|
296
315
|
const handler = async (request, response) => {
|
|
297
316
|
try {
|
|
317
|
+
// 1. Core connection rejection check on all updater routes (#237)
|
|
318
|
+
const conn = ctx?.connection || ctx?.get?.('connection') || request?.connection
|
|
319
|
+
if (conn && typeof conn.requestRejection === 'function') {
|
|
320
|
+
const rejection = conn.requestRejection(request)
|
|
321
|
+
if (rejection !== undefined) {
|
|
322
|
+
const status = typeof rejection === 'number' ? rejection : 401
|
|
323
|
+
response.writeHead(status, {
|
|
324
|
+
'content-type': 'application/json; charset=utf-8',
|
|
325
|
+
'cache-control': 'no-store',
|
|
326
|
+
})
|
|
327
|
+
response.end(JSON.stringify({ error: 'unauthorized', code: status }))
|
|
328
|
+
return
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
298
332
|
const target = runtime()
|
|
299
333
|
if (request.method === 'GET' || request.method === 'HEAD') {
|
|
300
334
|
const isFresh = Boolean(
|
|
@@ -317,7 +351,7 @@ export function registerPluginUpdater(ctx, options = {}) {
|
|
|
317
351
|
response.end()
|
|
318
352
|
return
|
|
319
353
|
}
|
|
320
|
-
if (!isTrustedUpdateRequest(request)) {
|
|
354
|
+
if (!isTrustedUpdateRequest(request, ctx)) {
|
|
321
355
|
json(response, 403, { error: 'Rejected non-local or cross-origin update request.' })
|
|
322
356
|
return
|
|
323
357
|
}
|