@goodandready/dsh-model-sync 0.2.7 → 0.2.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -0
- package/lib/adapter-registry.js +147 -27
- package/lib/client.js +87 -14
- package/lib/credentials.js +86 -5
- package/lib/generic-adapter.js +37 -5
- package/lib/history.js +138 -0
- package/lib/http.js +121 -4
- package/lib/index.js +62 -1
- package/lib/lifecycle.js +130 -0
- package/lib/models.js +203 -11
- package/lib/policy.js +71 -0
- package/lib/reconcile.js +8 -2
- package/lib/reliability.js +112 -0
- package/lib/reporting.js +155 -0
- package/lib/scheduler.js +127 -21
- package/lib/synchronizer.js +427 -36
- package/package.json +1 -1
package/lib/models.js
CHANGED
|
@@ -1,16 +1,170 @@
|
|
|
1
1
|
const FIELD_ALIASES = Object.freeze({
|
|
2
|
-
contextWindow: [
|
|
3
|
-
|
|
2
|
+
contextWindow: [
|
|
3
|
+
'contextWindow', 'context_window', 'contextLength', 'context_length',
|
|
4
|
+
'maxContextTokens', 'max_context_tokens', 'maxContextLength', 'max_context_length',
|
|
5
|
+
'inputTokenLimit', 'input_token_limit',
|
|
6
|
+
],
|
|
7
|
+
maxTokens: [
|
|
8
|
+
'maxTokens', 'max_tokens', 'max_output_tokens', 'maxOutputTokens',
|
|
9
|
+
'outputTokenLimit', 'output_token_limit',
|
|
10
|
+
],
|
|
4
11
|
})
|
|
5
12
|
|
|
6
|
-
|
|
13
|
+
const CAPABILITY_ALIASES = Object.freeze({
|
|
14
|
+
vision: ['vision', 'supportsVision', 'supports_vision', 'imageInput', 'image_input'],
|
|
15
|
+
tools: ['tools', 'toolUse', 'tool_use', 'functionCalling', 'function_calling', 'supportsTools', 'supports_tools'],
|
|
16
|
+
reasoning: ['reasoning', 'supportsReasoning', 'supports_reasoning', 'thinking', 'extendedThinking', 'extended_thinking'],
|
|
17
|
+
embeddings: ['embeddings', 'embedding', 'supportsEmbeddings', 'supports_embeddings'],
|
|
18
|
+
})
|
|
19
|
+
|
|
20
|
+
const CAPABILITY_TOKENS = Object.freeze({
|
|
21
|
+
vision: new Set(['vision', 'image', 'imageinput', 'multimodal']),
|
|
22
|
+
tools: new Set(['tools', 'tooluse', 'toolchoice', 'functioncalling', 'functioncall']),
|
|
23
|
+
reasoning: new Set(['reasoning', 'reasoningeffort', 'thinking', 'extendedthinking']),
|
|
24
|
+
embeddings: new Set(['embedding', 'embeddings']),
|
|
25
|
+
})
|
|
26
|
+
|
|
27
|
+
const ID_ALIASES = Object.freeze(['aliases', 'alias', 'previousIds', 'previous_ids', 'previousId', 'previous_id'])
|
|
28
|
+
|
|
29
|
+
const PRICING_ALIASES = Object.freeze({
|
|
30
|
+
inputPerToken: ['inputPerToken', 'input_per_token', 'inputPricePerToken', 'input_price_per_token'],
|
|
31
|
+
outputPerToken: ['outputPerToken', 'output_per_token', 'outputPricePerToken', 'output_price_per_token'],
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
function numericValue(value, { allowZero = false } = {}) {
|
|
35
|
+
if (typeof value !== 'number' && typeof value !== 'string') return undefined
|
|
36
|
+
if (typeof value === 'string' && value.trim() === '') return undefined
|
|
37
|
+
const number = Number(value)
|
|
38
|
+
if (!Number.isFinite(number) || (allowZero ? number < 0 : number <= 0)) return undefined
|
|
39
|
+
return number
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function firstNumber(row, fields, options) {
|
|
7
43
|
for (const field of fields) {
|
|
8
|
-
const value =
|
|
9
|
-
if (
|
|
44
|
+
const value = numericValue(row?.[field], options)
|
|
45
|
+
if (value !== undefined) return value
|
|
10
46
|
}
|
|
11
47
|
return undefined
|
|
12
48
|
}
|
|
13
49
|
|
|
50
|
+
function booleanValue(value) {
|
|
51
|
+
if (value === true || value === false) return value
|
|
52
|
+
if (typeof value !== 'string') return undefined
|
|
53
|
+
if (value.trim().toLowerCase() === 'true') return true
|
|
54
|
+
if (value.trim().toLowerCase() === 'false') return false
|
|
55
|
+
return undefined
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
function capabilityToken(value) {
|
|
59
|
+
if (typeof value !== 'string') return undefined
|
|
60
|
+
const token = value.trim().toLowerCase().replaceAll(/[_-]/g, '')
|
|
61
|
+
return Object.entries(CAPABILITY_TOKENS).find(([, tokens]) => tokens.has(token))?.[0]
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function normalizeCapabilities(row) {
|
|
65
|
+
const result = {}
|
|
66
|
+
const assign = (key, value) => {
|
|
67
|
+
const normalized = booleanValue(value)
|
|
68
|
+
if (normalized !== undefined) result[key] = normalized
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
for (const [key, fields] of Object.entries(CAPABILITY_ALIASES)) {
|
|
72
|
+
for (const field of fields) {
|
|
73
|
+
if (row?.[field] !== undefined) assign(key, row[field])
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const sources = [row?.capabilities, row?.supportedCapabilities, row?.supported_capabilities]
|
|
78
|
+
for (const source of sources) {
|
|
79
|
+
if (Array.isArray(source)) {
|
|
80
|
+
for (const token of source) {
|
|
81
|
+
const key = capabilityToken(token)
|
|
82
|
+
if (key) result[key] = true
|
|
83
|
+
}
|
|
84
|
+
} else if (source && typeof source === 'object') {
|
|
85
|
+
for (const [key, fields] of Object.entries(CAPABILITY_ALIASES)) {
|
|
86
|
+
for (const field of fields) {
|
|
87
|
+
if (source[field] !== undefined) assign(key, source[field])
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
for (const [token, value] of Object.entries(source)) {
|
|
91
|
+
const key = capabilityToken(token)
|
|
92
|
+
if (key) assign(key, value)
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
const modalitySources = [row?.modalities, row?.inputModalities, row?.input_modalities]
|
|
98
|
+
for (const source of modalitySources) {
|
|
99
|
+
if (!Array.isArray(source)) continue
|
|
100
|
+
for (const token of source) {
|
|
101
|
+
const key = capabilityToken(token)
|
|
102
|
+
if (key) result[key] = true
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const parameterSources = [row?.supportedParameters, row?.supported_parameters]
|
|
107
|
+
for (const source of parameterSources) {
|
|
108
|
+
if (!Array.isArray(source)) continue
|
|
109
|
+
for (const token of source) {
|
|
110
|
+
const key = capabilityToken(token)
|
|
111
|
+
if (key) result[key] = true
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
return Object.keys(result).length > 0 ? result : undefined
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function normalizeTags(row) {
|
|
118
|
+
const source = row?.tags ?? row?.labels
|
|
119
|
+
if (!Array.isArray(source)) return undefined
|
|
120
|
+
const tags = [...new Set(source
|
|
121
|
+
.filter((tag) => typeof tag === 'string' && tag.trim())
|
|
122
|
+
.map((tag) => tag.trim())
|
|
123
|
+
.slice(0, 50))]
|
|
124
|
+
return tags.length > 0 ? tags : undefined
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function normalizeAliases(row, id) {
|
|
128
|
+
const aliases = []
|
|
129
|
+
for (const field of ID_ALIASES) {
|
|
130
|
+
const source = row?.[field]
|
|
131
|
+
const values = Array.isArray(source) ? source : source === undefined || source === null ? [] : [source]
|
|
132
|
+
for (const value of values) {
|
|
133
|
+
if (typeof value !== 'string') continue
|
|
134
|
+
const alias = value.trim()
|
|
135
|
+
if (alias && alias !== id && !aliases.includes(alias)) aliases.push(alias)
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
return aliases.slice(0, 20)
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function normalizeLifecycle(row) {
|
|
142
|
+
const source = row?.lifecycle && typeof row.lifecycle === 'object' ? row.lifecycle : {}
|
|
143
|
+
const deprecated = booleanValue(source.deprecated ?? row?.deprecated ?? row?.isDeprecated)
|
|
144
|
+
const date = source.deprecationDate ?? source.deprecation_date ?? source.expirationDate ?? source.expiration_date ?? source.expiresAt ?? source.expires_at ?? row?.deprecationDate ?? row?.deprecation_date ?? row?.expirationDate ?? row?.expiration_date ?? row?.expiresAt ?? row?.expires_at
|
|
145
|
+
const deprecationDate = typeof date === 'string' && date.trim() ? date.trim() : undefined
|
|
146
|
+
if (deprecated === undefined && deprecationDate === undefined) return undefined
|
|
147
|
+
return {
|
|
148
|
+
...(deprecated === undefined ? {} : { deprecated }),
|
|
149
|
+
...(deprecationDate === undefined ? {} : { deprecationDate }),
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function normalizePricing(row) {
|
|
154
|
+
const source = row?.pricing ?? row?.price ?? row?.cost
|
|
155
|
+
if (!source || typeof source !== 'object') return undefined
|
|
156
|
+
const inputPerToken = firstNumber(source, PRICING_ALIASES.inputPerToken, { allowZero: true })
|
|
157
|
+
const outputPerToken = firstNumber(source, PRICING_ALIASES.outputPerToken, { allowZero: true })
|
|
158
|
+
if (inputPerToken === undefined && outputPerToken === undefined) return undefined
|
|
159
|
+
const pricing = {
|
|
160
|
+
...(inputPerToken === undefined ? {} : { inputPerToken }),
|
|
161
|
+
...(outputPerToken === undefined ? {} : { outputPerToken }),
|
|
162
|
+
}
|
|
163
|
+
if (typeof source.currency === 'string' && source.currency.trim()) pricing.currency = source.currency.trim()
|
|
164
|
+
if (typeof source.unit === 'string' && source.unit.trim()) pricing.unit = source.unit.trim()
|
|
165
|
+
return pricing
|
|
166
|
+
}
|
|
167
|
+
|
|
14
168
|
export function normalizeModel(provider, row) {
|
|
15
169
|
if (typeof row === 'string') {
|
|
16
170
|
row = { id: row }
|
|
@@ -21,6 +175,11 @@ export function normalizeModel(provider, row) {
|
|
|
21
175
|
const name = String(row.name ?? row.displayName ?? row.display_name ?? id).trim() || id
|
|
22
176
|
const contextWindow = firstNumber(row, FIELD_ALIASES.contextWindow)
|
|
23
177
|
const maxTokens = firstNumber(row, FIELD_ALIASES.maxTokens)
|
|
178
|
+
const capabilities = normalizeCapabilities(row)
|
|
179
|
+
const pricing = normalizePricing(row)
|
|
180
|
+
const tags = normalizeTags(row)
|
|
181
|
+
const aliases = normalizeAliases(row, id)
|
|
182
|
+
const lifecycle = normalizeLifecycle(row)
|
|
24
183
|
return {
|
|
25
184
|
provider,
|
|
26
185
|
id,
|
|
@@ -28,6 +187,11 @@ export function normalizeModel(provider, row) {
|
|
|
28
187
|
...(contextWindow === undefined ? {} : { contextWindow }),
|
|
29
188
|
...(maxTokens === undefined ? {} : { maxTokens }),
|
|
30
189
|
...(row.description ? { description: String(row.description) } : {}),
|
|
190
|
+
...(capabilities === undefined ? {} : { capabilities }),
|
|
191
|
+
...(pricing === undefined ? {} : { pricing }),
|
|
192
|
+
...(tags === undefined ? {} : { tags }),
|
|
193
|
+
...(aliases.length === 0 ? {} : { aliases }),
|
|
194
|
+
...(lifecycle === undefined ? {} : { lifecycle }),
|
|
31
195
|
}
|
|
32
196
|
}
|
|
33
197
|
|
|
@@ -44,19 +208,47 @@ export function normalizeModels(provider, rows) {
|
|
|
44
208
|
}
|
|
45
209
|
|
|
46
210
|
export function diffModels(previous, next) {
|
|
47
|
-
const
|
|
48
|
-
const
|
|
211
|
+
const sortModels = (rows) => normalizeModels('', rows).sort((left, right) => left.id.localeCompare(right.id))
|
|
212
|
+
const beforeRows = sortModels(previous)
|
|
213
|
+
const afterRows = sortModels(next)
|
|
214
|
+
const before = new Map(beforeRows.map((row) => [row.id, row]))
|
|
215
|
+
const after = new Map(afterRows.map((row) => [row.id, row]))
|
|
49
216
|
const added = []
|
|
50
217
|
const removed = []
|
|
218
|
+
const renamed = []
|
|
51
219
|
const changed = []
|
|
220
|
+
const renamedBefore = new Set()
|
|
52
221
|
for (const [id, model] of after) {
|
|
53
|
-
if (!before.has(id))
|
|
54
|
-
|
|
222
|
+
if (!before.has(id)) continue
|
|
223
|
+
if (JSON.stringify(before.get(id)) !== JSON.stringify(model)) {
|
|
55
224
|
changed.push({ before: before.get(id), after: model })
|
|
56
225
|
}
|
|
57
226
|
}
|
|
227
|
+
for (const [id, model] of after) {
|
|
228
|
+
if (before.has(id)) continue
|
|
229
|
+
const match = beforeRows.find((candidate) => {
|
|
230
|
+
if (renamedBefore.has(candidate.id)) return false
|
|
231
|
+
return model.aliases?.includes(candidate.id) || candidate.aliases?.includes(model.id)
|
|
232
|
+
})
|
|
233
|
+
if (match) {
|
|
234
|
+
renamed.push({ before: match, after: model })
|
|
235
|
+
renamedBefore.add(match.id)
|
|
236
|
+
} else {
|
|
237
|
+
added.push(model)
|
|
238
|
+
}
|
|
239
|
+
}
|
|
58
240
|
for (const [id, model] of before) {
|
|
59
|
-
if (!after.has(id)) removed.push(model)
|
|
241
|
+
if (!after.has(id) && !renamedBefore.has(id)) removed.push(model)
|
|
242
|
+
}
|
|
243
|
+
added.sort((left, right) => left.id.localeCompare(right.id))
|
|
244
|
+
removed.sort((left, right) => left.id.localeCompare(right.id))
|
|
245
|
+
renamed.sort((left, right) => left.before.id.localeCompare(right.before.id))
|
|
246
|
+
changed.sort((left, right) => left.before.id.localeCompare(right.before.id))
|
|
247
|
+
return {
|
|
248
|
+
added,
|
|
249
|
+
removed,
|
|
250
|
+
renamed,
|
|
251
|
+
changed,
|
|
252
|
+
hasChanges: added.length > 0 || removed.length > 0 || renamed.length > 0 || changed.length > 0,
|
|
60
253
|
}
|
|
61
|
-
return { added, removed, changed, hasChanges: added.length > 0 || removed.length > 0 || changed.length > 0 }
|
|
62
254
|
}
|
package/lib/policy.js
ADDED
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
const CAPABILITY_KEYS = new Set(['vision', 'tools', 'reasoning', 'embeddings'])
|
|
2
|
+
const MAX_PATTERNS = 100
|
|
3
|
+
const MAX_PATTERN_LENGTH = 256
|
|
4
|
+
|
|
5
|
+
function list(value) {
|
|
6
|
+
if (!Array.isArray(value)) return []
|
|
7
|
+
return [...new Set(value.filter((item) => typeof item === 'string' && item.trim()).map((item) => item.trim()).slice(0, MAX_PATTERNS))]
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
function capabilities(value) {
|
|
11
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) return {}
|
|
12
|
+
return Object.fromEntries(Object.entries(value)
|
|
13
|
+
.filter(([key, val]) => CAPABILITY_KEYS.has(key) && typeof val === 'boolean')
|
|
14
|
+
.slice(0, CAPABILITY_KEYS.size))
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function normalizePolicy(policy) {
|
|
18
|
+
const value = policy && typeof policy === 'object' && !Array.isArray(policy) ? policy : {}
|
|
19
|
+
return {
|
|
20
|
+
include: list(value.include),
|
|
21
|
+
exclude: list(value.exclude),
|
|
22
|
+
requireCapabilities: capabilities(value.requireCapabilities),
|
|
23
|
+
denyCapabilities: capabilities(value.denyCapabilities),
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function compilePatterns(patterns, field) {
|
|
28
|
+
return patterns.map((pattern) => {
|
|
29
|
+
if (pattern.length > MAX_PATTERN_LENGTH) throw Object.assign(new Error(`${field} pattern is too long`), { code: 'INVALID_POLICY' })
|
|
30
|
+
try {
|
|
31
|
+
return new RegExp(pattern, 'i')
|
|
32
|
+
} catch {
|
|
33
|
+
throw Object.assign(new Error(`invalid ${field} pattern: ${pattern}`), { code: 'INVALID_POLICY' })
|
|
34
|
+
}
|
|
35
|
+
})
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function validatePolicy(policy) {
|
|
39
|
+
const normalized = normalizePolicy(policy)
|
|
40
|
+
compilePatterns(normalized.include, 'include')
|
|
41
|
+
compilePatterns(normalized.exclude, 'exclude')
|
|
42
|
+
return normalized
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export function hasPolicy(policy) {
|
|
46
|
+
const normalized = normalizePolicy(policy)
|
|
47
|
+
return normalized.include.length > 0
|
|
48
|
+
|| normalized.exclude.length > 0
|
|
49
|
+
|| Object.keys(normalized.requireCapabilities).length > 0
|
|
50
|
+
|| Object.keys(normalized.denyCapabilities).length > 0
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function filterModels(models, policy) {
|
|
54
|
+
const normalized = validatePolicy(policy)
|
|
55
|
+
const include = compilePatterns(normalized.include, 'include')
|
|
56
|
+
const exclude = compilePatterns(normalized.exclude, 'exclude')
|
|
57
|
+
return (models ?? []).filter((model) => {
|
|
58
|
+
const haystack = [model?.id, model?.name, ...(Array.isArray(model?.tags) ? model.tags : [])]
|
|
59
|
+
.filter((value) => typeof value === 'string').join('\n')
|
|
60
|
+
if (include.length > 0 && !include.some((pattern) => pattern.test(haystack))) return false
|
|
61
|
+
if (exclude.some((pattern) => pattern.test(haystack))) return false
|
|
62
|
+
const modelCapabilities = model?.capabilities ?? {}
|
|
63
|
+
for (const [key, expected] of Object.entries(normalized.requireCapabilities)) {
|
|
64
|
+
if (modelCapabilities[key] !== expected) return false
|
|
65
|
+
}
|
|
66
|
+
for (const [key, denied] of Object.entries(normalized.denyCapabilities)) {
|
|
67
|
+
if (modelCapabilities[key] === denied) return false
|
|
68
|
+
}
|
|
69
|
+
return true
|
|
70
|
+
})
|
|
71
|
+
}
|
package/lib/reconcile.js
CHANGED
|
@@ -6,18 +6,24 @@ function sameModel(left, right) {
|
|
|
6
6
|
|
|
7
7
|
export function reconcileModels(provider, current, discovered, {
|
|
8
8
|
removeMissing = false,
|
|
9
|
+
removeIds = [],
|
|
9
10
|
} = {}) {
|
|
10
11
|
const before = normalizeModels(provider, current)
|
|
11
12
|
const advertised = normalizeModels(provider, discovered)
|
|
12
13
|
const advertisedById = new Map(advertised.map((row) => [row.id, row]))
|
|
14
|
+
const explicitRemoveIds = new Set(removeIds)
|
|
15
|
+
const discoveryDiff = diffModels(before, advertised)
|
|
16
|
+
const renamedBefore = new Set(discoveryDiff.renamed.map((row) => row.before.id))
|
|
13
17
|
const next = []
|
|
14
18
|
const stale = []
|
|
15
19
|
for (const row of before) {
|
|
16
20
|
const fresh = advertisedById.get(row.id)
|
|
17
21
|
if (fresh) next.push(fresh)
|
|
18
|
-
else {
|
|
22
|
+
else if (renamedBefore.has(row.id)) {
|
|
23
|
+
// An explicit provider alias is safe evidence that the old id was renamed.
|
|
24
|
+
} else {
|
|
19
25
|
stale.push(row)
|
|
20
|
-
if (!removeMissing) next.push(row)
|
|
26
|
+
if (!removeMissing && !explicitRemoveIds.has(row.id)) next.push(row)
|
|
21
27
|
}
|
|
22
28
|
}
|
|
23
29
|
const known = new Set(before.map((row) => row.id))
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
const RETRYABLE_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504, 522, 524])
|
|
2
|
+
const RETRYABLE_CODES = new Set(['ECONNRESET', 'ECONNREFUSED', 'ETIMEDOUT', 'EAI_AGAIN', 'UND_ERR_CONNECT_TIMEOUT'])
|
|
3
|
+
|
|
4
|
+
export function statusOfError(error) {
|
|
5
|
+
const direct = Number(error?.status ?? error?.statusCode)
|
|
6
|
+
if (Number.isInteger(direct) && direct > 0) return direct
|
|
7
|
+
const match = String(error?.message ?? '').match(/HTTP\s+(\d{3})/i)
|
|
8
|
+
return match ? Number(match[1]) : undefined
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function isRetryableError(error) {
|
|
12
|
+
if (!error) return false
|
|
13
|
+
if (error.name === 'AbortError' || error.name === 'TimeoutError') return true
|
|
14
|
+
if (RETRYABLE_CODES.has(error.code)) return true
|
|
15
|
+
const status = statusOfError(error)
|
|
16
|
+
return status !== undefined && RETRYABLE_STATUSES.has(status)
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function catalogRequestError(status) {
|
|
20
|
+
const error = new Error(`model catalog request failed with HTTP ${status}`)
|
|
21
|
+
error.status = Number(status)
|
|
22
|
+
return error
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function abortError() {
|
|
26
|
+
const error = new Error('operation aborted')
|
|
27
|
+
error.name = 'AbortError'
|
|
28
|
+
return error
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function delay(ms, signal) {
|
|
32
|
+
if (!Number.isFinite(ms) || ms <= 0) return Promise.resolve()
|
|
33
|
+
return new Promise((resolve, reject) => {
|
|
34
|
+
let timer
|
|
35
|
+
const onAbort = () => {
|
|
36
|
+
clearTimeout(timer)
|
|
37
|
+
signal?.removeEventListener('abort', onAbort)
|
|
38
|
+
reject(signal.reason ?? abortError())
|
|
39
|
+
}
|
|
40
|
+
timer = setTimeout(() => {
|
|
41
|
+
signal?.removeEventListener('abort', onAbort)
|
|
42
|
+
resolve()
|
|
43
|
+
}, ms)
|
|
44
|
+
if (signal) {
|
|
45
|
+
if (signal.aborted) onAbort()
|
|
46
|
+
else signal.addEventListener('abort', onAbort, { once: true })
|
|
47
|
+
}
|
|
48
|
+
})
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function createTimeoutSignal(parent, timeoutMs) {
|
|
52
|
+
const controller = new AbortController()
|
|
53
|
+
let timer = null
|
|
54
|
+
const onParentAbort = () => controller.abort(parent.reason ?? abortError())
|
|
55
|
+
if (parent) {
|
|
56
|
+
if (parent.aborted) onParentAbort()
|
|
57
|
+
else parent.addEventListener('abort', onParentAbort, { once: true })
|
|
58
|
+
}
|
|
59
|
+
const timeout = Number(timeoutMs)
|
|
60
|
+
if (Number.isFinite(timeout) && timeout > 0) {
|
|
61
|
+
timer = setTimeout(() => {
|
|
62
|
+
const error = new Error(`request timed out after ${timeout}ms`)
|
|
63
|
+
error.name = 'TimeoutError'
|
|
64
|
+
controller.abort(error)
|
|
65
|
+
}, timeout)
|
|
66
|
+
}
|
|
67
|
+
return {
|
|
68
|
+
signal: controller.signal,
|
|
69
|
+
cleanup() {
|
|
70
|
+
if (timer !== null) clearTimeout(timer)
|
|
71
|
+
if (parent) parent.removeEventListener('abort', onParentAbort)
|
|
72
|
+
},
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export async function retryWithBackoff(operation, {
|
|
77
|
+
attempts = 3,
|
|
78
|
+
baseDelayMs = 250,
|
|
79
|
+
maxDelayMs = 4000,
|
|
80
|
+
signal,
|
|
81
|
+
onRetry,
|
|
82
|
+
} = {}) {
|
|
83
|
+
const total = Math.max(1, Math.floor(Number(attempts) || 1))
|
|
84
|
+
let lastError
|
|
85
|
+
for (let attempt = 1; attempt <= total; attempt += 1) {
|
|
86
|
+
if (signal?.aborted) throw signal.reason ?? abortError()
|
|
87
|
+
try {
|
|
88
|
+
return await operation({ attempt })
|
|
89
|
+
} catch (error) {
|
|
90
|
+
lastError = error
|
|
91
|
+
if (attempt >= total || !isRetryableError(error)) throw error
|
|
92
|
+
const delayMs = Math.min(Math.max(0, Number(maxDelayMs) || 0), Math.max(0, Number(baseDelayMs) || 0) * (2 ** (attempt - 1)))
|
|
93
|
+
onRetry?.({ attempt, nextAttempt: attempt + 1, delayMs, error })
|
|
94
|
+
await delay(delayMs, signal)
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
throw lastError
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export async function mapWithConcurrency(items, worker, concurrency = 4) {
|
|
101
|
+
const values = new Array(items.length)
|
|
102
|
+
const limit = Math.max(1, Math.min(items.length || 1, Math.floor(Number(concurrency) || 1)))
|
|
103
|
+
let cursor = 0
|
|
104
|
+
await Promise.all(Array.from({ length: limit }, async () => {
|
|
105
|
+
while (true) {
|
|
106
|
+
const index = cursor++
|
|
107
|
+
if (index >= items.length) return
|
|
108
|
+
values[index] = await worker(items[index], index)
|
|
109
|
+
}
|
|
110
|
+
}))
|
|
111
|
+
return values
|
|
112
|
+
}
|
package/lib/reporting.js
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
|
|
3
|
+
export const DEFAULT_NOTIFICATION_LIMIT = 50
|
|
4
|
+
export const MAX_NOTIFICATION_LIMIT = 200
|
|
5
|
+
|
|
6
|
+
function boundedInteger(value, fallback, minimum, maximum) {
|
|
7
|
+
const number = Number(value)
|
|
8
|
+
if (!Number.isFinite(number)) return fallback
|
|
9
|
+
return Math.min(maximum, Math.max(minimum, Math.floor(number)))
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export function notificationLimit(value) {
|
|
13
|
+
return boundedInteger(value, DEFAULT_NOTIFICATION_LIMIT, 1, MAX_NOTIFICATION_LIMIT)
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function safeMessage(value) {
|
|
17
|
+
return String(value ?? '')
|
|
18
|
+
.replace(/(api[_-]?key|token|secret|authorization|password)\s*[:=]\s*[^\s,;]+/gi, '$1=[redacted]')
|
|
19
|
+
.replace(/([?&](?:key|token|api[_-]?key|access[_-]?token)=)[^&#\s]+/gi, '$1[redacted]')
|
|
20
|
+
.slice(0, 240)
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function stable(value) {
|
|
24
|
+
if (Array.isArray(value)) return value.map(stable)
|
|
25
|
+
if (!value || typeof value !== 'object') return value
|
|
26
|
+
return Object.fromEntries(Object.keys(value).sort().map((key) => [key, stable(value[key])]))
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function statusClass(status) {
|
|
30
|
+
if (status === 'ok') return 'success'
|
|
31
|
+
if (status === 'error' || status === 'circuit-open') return 'failure'
|
|
32
|
+
return 'skipped'
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function countsFor(rows) {
|
|
36
|
+
return rows.reduce((counts, row) => {
|
|
37
|
+
const diff = row?.diff ?? {}
|
|
38
|
+
counts.added += Array.isArray(diff.added) ? diff.added.length : 0
|
|
39
|
+
counts.removed += Array.isArray(diff.removed) ? diff.removed.length : 0
|
|
40
|
+
counts.renamed += Array.isArray(diff.renamed) ? diff.renamed.length : 0
|
|
41
|
+
counts.changed += Array.isArray(diff.changed) ? diff.changed.length : 0
|
|
42
|
+
counts.stale += Array.isArray(row?.stale) ? row.stale.length : 0
|
|
43
|
+
return counts
|
|
44
|
+
}, { added: 0, removed: 0, renamed: 0, changed: 0, stale: 0 })
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function createSyncReport(result, { source = 'manual' } = {}) {
|
|
48
|
+
const rows = Array.isArray(result?.results) ? result.results : []
|
|
49
|
+
const statuses = rows.map((row) => statusClass(row?.status))
|
|
50
|
+
const succeeded = statuses.filter((status) => status === 'success').length
|
|
51
|
+
const failed = statuses.filter((status) => status === 'failure').length
|
|
52
|
+
const skipped = statuses.filter((status) => status === 'skipped').length
|
|
53
|
+
const outcome = failed > 0 && succeeded > 0
|
|
54
|
+
? 'partial'
|
|
55
|
+
: failed > 0 ? 'failure'
|
|
56
|
+
: succeeded > 0 ? 'success'
|
|
57
|
+
: 'empty'
|
|
58
|
+
const severity = outcome === 'failure' ? 'error' : outcome === 'partial' ? 'warning' : outcome === 'success' ? 'success' : 'info'
|
|
59
|
+
const providers = rows.map((row) => ({
|
|
60
|
+
provider: String(row?.provider ?? ''),
|
|
61
|
+
status: String(row?.status ?? 'unknown'),
|
|
62
|
+
class: statusClass(row?.status),
|
|
63
|
+
changed: Boolean(row?.changed),
|
|
64
|
+
...(Number.isFinite(row?.retries) ? { retries: row.retries } : {}),
|
|
65
|
+
...(row?.message ? { message: safeMessage(row.message) } : {}),
|
|
66
|
+
counts: {
|
|
67
|
+
added: Array.isArray(row?.diff?.added) ? row.diff.added.length : 0,
|
|
68
|
+
removed: Array.isArray(row?.diff?.removed) ? row.diff.removed.length : 0,
|
|
69
|
+
renamed: Array.isArray(row?.diff?.renamed) ? row.diff.renamed.length : 0,
|
|
70
|
+
changed: Array.isArray(row?.diff?.changed) ? row.diff.changed.length : 0,
|
|
71
|
+
stale: Array.isArray(row?.stale) ? row.stale.length : 0,
|
|
72
|
+
},
|
|
73
|
+
})).sort((left, right) => left.provider.localeCompare(right.provider))
|
|
74
|
+
const report = {
|
|
75
|
+
version: 1,
|
|
76
|
+
source: source === 'schedule' ? 'schedule' : 'manual',
|
|
77
|
+
startedAt: Number.isFinite(result?.startedAt) ? result.startedAt : Date.now(),
|
|
78
|
+
finishedAt: Number.isFinite(result?.finishedAt) ? result.finishedAt : Date.now(),
|
|
79
|
+
dryRun: result?.dryRun !== false,
|
|
80
|
+
outcome,
|
|
81
|
+
severity,
|
|
82
|
+
counts: {
|
|
83
|
+
providers: rows.length,
|
|
84
|
+
succeeded,
|
|
85
|
+
failed,
|
|
86
|
+
skipped,
|
|
87
|
+
changed: providers.filter((row) => row.changed).length,
|
|
88
|
+
models: countsFor(rows),
|
|
89
|
+
},
|
|
90
|
+
providers,
|
|
91
|
+
}
|
|
92
|
+
const fingerprintPayload = { outcome, severity, dryRun: report.dryRun, counts: report.counts, providers }
|
|
93
|
+
report.fingerprint = createHash('sha256').update(JSON.stringify(stable(fingerprintPayload))).digest('hex').slice(0, 24)
|
|
94
|
+
return report
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function notificationForReport(report, { now = Date.now() } = {}) {
|
|
98
|
+
if (!report || (report.outcome !== 'failure' && report.outcome !== 'partial')) return null
|
|
99
|
+
const failed = report.providers.filter((row) => row.class === 'failure')
|
|
100
|
+
const title = report.outcome === 'partial' ? 'Model sync partially completed' : 'Model sync failed'
|
|
101
|
+
const message = report.outcome === 'partial'
|
|
102
|
+
? report.counts.succeeded + ' providers succeeded; ' + report.counts.failed + ' failed'
|
|
103
|
+
: report.counts.failed + ' providers failed'
|
|
104
|
+
return {
|
|
105
|
+
id: 'sync-notice-' + report.fingerprint,
|
|
106
|
+
fingerprint: report.fingerprint,
|
|
107
|
+
createdAt: now,
|
|
108
|
+
updatedAt: now,
|
|
109
|
+
occurrences: 1,
|
|
110
|
+
readAt: null,
|
|
111
|
+
acknowledgedAt: null,
|
|
112
|
+
severity: report.severity,
|
|
113
|
+
title,
|
|
114
|
+
message,
|
|
115
|
+
providers: failed.map((row) => ({ provider: row.provider, status: row.status, ...(row.message ? { message: row.message } : {}) })),
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
export function appendNotification(existing, report, { limit = DEFAULT_NOTIFICATION_LIMIT, now = Date.now() } = {}) {
|
|
120
|
+
const notification = notificationForReport(report, { now })
|
|
121
|
+
const rows = Array.isArray(existing) ? existing.filter((row) => row && typeof row === 'object') : []
|
|
122
|
+
if (!notification) return rows.slice(-notificationLimit(limit))
|
|
123
|
+
const index = rows.findIndex((row) => row.fingerprint === notification.fingerprint)
|
|
124
|
+
if (index >= 0) {
|
|
125
|
+
const current = structuredClone(rows[index])
|
|
126
|
+
rows.splice(index, 1)
|
|
127
|
+
rows.push({
|
|
128
|
+
...current,
|
|
129
|
+
updatedAt: now,
|
|
130
|
+
occurrences: boundedInteger(current.occurrences, 1, 1, 1000000) + 1,
|
|
131
|
+
})
|
|
132
|
+
} else {
|
|
133
|
+
rows.push(notification)
|
|
134
|
+
}
|
|
135
|
+
return rows.slice(-notificationLimit(limit))
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export function updateNotification(existing, id, field, { now = Date.now() } = {}) {
|
|
139
|
+
if (field !== 'read' && field !== 'acknowledged') throw Object.assign(new Error('notification field is invalid'), { code: 'INVALID_NOTIFICATION' })
|
|
140
|
+
if (typeof id !== 'string' || !id || id.length > 128) throw Object.assign(new Error('notification id is invalid'), { code: 'INVALID_NOTIFICATION' })
|
|
141
|
+
const rows = Array.isArray(existing) ? existing.map((row) => structuredClone(row)) : []
|
|
142
|
+
const row = rows.find((item) => item?.id === id)
|
|
143
|
+
if (!row) throw Object.assign(new Error('notification not found'), { code: 'NOTIFICATION_NOT_FOUND' })
|
|
144
|
+
row[field === 'read' ? 'readAt' : 'acknowledgedAt'] = now
|
|
145
|
+
return rows
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export function listNotifications(existing, { includeAcknowledged = true } = {}) {
|
|
149
|
+
const rows = Array.isArray(existing) ? existing : []
|
|
150
|
+
return rows
|
|
151
|
+
.filter((row) => includeAcknowledged || !row?.acknowledgedAt)
|
|
152
|
+
.slice()
|
|
153
|
+
.sort((left, right) => Number(right.updatedAt ?? right.createdAt ?? 0) - Number(left.updatedAt ?? left.createdAt ?? 0))
|
|
154
|
+
.map((row) => structuredClone(row))
|
|
155
|
+
}
|