model-eol 0.2.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +117 -22
- package/action.yml +124 -9
- package/bot/bot.mjs +1371 -0
- package/bot/lib/common.mjs +122 -0
- package/bot/lib/config.mjs +1 -0
- package/bot/lib/eval.mjs +103 -0
- package/bot/lib/feeds.mjs +48 -0
- package/bot/lib/git.mjs +131 -0
- package/bot/lib/github.mjs +187 -0
- package/bot.yml.example +145 -0
- package/check.mjs +151 -17
- package/feeds/amazon.json +1 -1
- package/feeds/anthropic.json +1 -1
- package/feeds/google.json +88 -5
- package/feeds/openai.json +11 -26
- package/lib/config.mjs +258 -0
- package/lib/feeds.mjs +38 -4
- package/lib/glob.mjs +35 -0
- package/lib/plan.mjs +27 -8
- package/lib/reports.mjs +24 -7
- package/lib/scanner.mjs +247 -70
- package/package.json +8 -4
- package/schema/model-eol.bot-config.schema.json +73 -8
- package/schema/model-eol.inventory.schema.json +21 -0
- package/schema/model-eol.plan.schema.json +22 -0
package/lib/config.mjs
ADDED
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
import fs from 'node:fs'
|
|
2
|
+
|
|
3
|
+
import { matchesAnyGlob, normalizeRepoPath } from './glob.mjs'
|
|
4
|
+
import { MAX_REPORT_BYTES, MAX_TIMEOUT_MS, MIN_TIMEOUT_MS } from './validate-feed.mjs'
|
|
5
|
+
|
|
6
|
+
const frozenArray = () => Object.freeze([])
|
|
7
|
+
|
|
8
|
+
export const DEFAULT_CONFIG = Object.freeze({
|
|
9
|
+
days: 90,
|
|
10
|
+
scope: 'direct',
|
|
11
|
+
via: null,
|
|
12
|
+
feeds: Object.freeze({ allow_vendored_fallback: false }),
|
|
13
|
+
ignore: Object.freeze({ models: frozenArray(), paths: frozenArray() }),
|
|
14
|
+
overrides: frozenArray(),
|
|
15
|
+
routes: frozenArray(),
|
|
16
|
+
issues: Object.freeze({ enabled: true }),
|
|
17
|
+
eval: Object.freeze({ command: null, timeout_ms: 600000, max_report_bytes: 65536, pass_env: frozenArray() }),
|
|
18
|
+
})
|
|
19
|
+
|
|
20
|
+
export const CLI_DEFAULT_CONFIG = Object.freeze({
|
|
21
|
+
...DEFAULT_CONFIG,
|
|
22
|
+
scope: 'all',
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
const copyDefaults = defaults => ({
|
|
26
|
+
days: defaults.days,
|
|
27
|
+
scope: defaults.scope,
|
|
28
|
+
via: defaults.via,
|
|
29
|
+
feeds: { allow_vendored_fallback: defaults.feeds.allow_vendored_fallback },
|
|
30
|
+
ignore: {
|
|
31
|
+
models: [...defaults.ignore.models],
|
|
32
|
+
paths: [...defaults.ignore.paths],
|
|
33
|
+
},
|
|
34
|
+
overrides: (defaults.overrides ?? []).map(override => ({
|
|
35
|
+
...override,
|
|
36
|
+
paths: [...override.paths],
|
|
37
|
+
...(override.ignore ? { ignore: {
|
|
38
|
+
models: [...(override.ignore.models ?? [])],
|
|
39
|
+
paths: [...(override.ignore.paths ?? [])],
|
|
40
|
+
} } : {}),
|
|
41
|
+
})),
|
|
42
|
+
routes: (defaults.routes ?? []).map(route => ({ ...route, paths: [...route.paths] })),
|
|
43
|
+
issues: { enabled: defaults.issues.enabled },
|
|
44
|
+
eval: {
|
|
45
|
+
command: defaults.eval.command,
|
|
46
|
+
timeout_ms: defaults.eval.timeout_ms,
|
|
47
|
+
max_report_bytes: defaults.eval.max_report_bytes,
|
|
48
|
+
pass_env: [...defaults.eval.pass_env],
|
|
49
|
+
},
|
|
50
|
+
})
|
|
51
|
+
|
|
52
|
+
const assert = (condition, message) => {
|
|
53
|
+
if (!condition) throw new Error(`invalid .model-eol.json: ${message}`)
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const assertKnownKeys = (value, label, allowed) => {
|
|
57
|
+
for (const key of Object.keys(value)) {
|
|
58
|
+
const fullKey = label ? `${label}.${key}` : key
|
|
59
|
+
assert(allowed.has(key), `unknown key "${fullKey}"`)
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const stringArray = (value, label) => {
|
|
64
|
+
assert(Array.isArray(value), `${label} must be an array`)
|
|
65
|
+
assert(value.every(item => typeof item === 'string' && item.length > 0), `${label} must contain non-empty strings`)
|
|
66
|
+
return value.slice()
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const nonEmptyString = (value, label) => {
|
|
70
|
+
assert(typeof value === 'string' && value.length > 0, `${label} must be a non-empty string`)
|
|
71
|
+
return value
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
const normalizeIgnore = (raw, label) => {
|
|
75
|
+
assert(raw && typeof raw === 'object' && !Array.isArray(raw), `${label} must be an object`)
|
|
76
|
+
assertKnownKeys(raw, label, new Set(['models', 'paths']))
|
|
77
|
+
return {
|
|
78
|
+
models: raw.models === undefined ? [] : stringArray(raw.models, `${label}.models`),
|
|
79
|
+
paths: raw.paths === undefined ? [] : stringArray(raw.paths, `${label}.paths`),
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const normalizeOverride = (raw, index) => {
|
|
84
|
+
const label = `overrides[${index}]`
|
|
85
|
+
assert(raw && typeof raw === 'object' && !Array.isArray(raw), `${label} must be an object`)
|
|
86
|
+
assertKnownKeys(raw, label, new Set(['paths', 'days', 'scope', 'via', 'ignore']))
|
|
87
|
+
const override = { paths: stringArray(raw.paths, `${label}.paths`) }
|
|
88
|
+
assert(override.paths.length > 0, `${label}.paths must not be empty`)
|
|
89
|
+
if (raw.days !== undefined) {
|
|
90
|
+
assert(Number.isInteger(raw.days) && raw.days >= 0, `${label}.days must be a non-negative integer`)
|
|
91
|
+
override.days = raw.days
|
|
92
|
+
}
|
|
93
|
+
if (raw.scope !== undefined) {
|
|
94
|
+
assert(raw.scope === 'all' || raw.scope === 'direct', `${label}.scope must be all or direct`)
|
|
95
|
+
override.scope = raw.scope
|
|
96
|
+
}
|
|
97
|
+
if (raw.via !== undefined) {
|
|
98
|
+
assert(raw.via === null || (typeof raw.via === 'string' && raw.via.length > 0), `${label}.via must be null or a non-empty string`)
|
|
99
|
+
override.via = raw.via
|
|
100
|
+
}
|
|
101
|
+
if (raw.ignore !== undefined) override.ignore = normalizeIgnore(raw.ignore, `${label}.ignore`)
|
|
102
|
+
return override
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const normalizeRoute = (raw, index) => {
|
|
106
|
+
const label = `routes[${index}]`
|
|
107
|
+
assert(raw && typeof raw === 'object' && !Array.isArray(raw), `${label} must be an object`)
|
|
108
|
+
assertKnownKeys(raw, label, new Set(['paths', 'via', 'match', 'model']))
|
|
109
|
+
const route = {
|
|
110
|
+
paths: stringArray(raw.paths, `${label}.paths`),
|
|
111
|
+
via: nonEmptyString(raw.via, `${label}.via`),
|
|
112
|
+
}
|
|
113
|
+
assert(route.paths.length > 0, `${label}.paths must not be empty`)
|
|
114
|
+
assert((raw.match === undefined) === (raw.model === undefined), `${label}.match and ${label}.model must be provided together`)
|
|
115
|
+
if (raw.match !== undefined) {
|
|
116
|
+
route.match = nonEmptyString(raw.match, `${label}.match`)
|
|
117
|
+
route.model = nonEmptyString(raw.model, `${label}.model`)
|
|
118
|
+
}
|
|
119
|
+
return route
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const CREDENTIAL_ENV_PATTERN = /^(?:GITHUB_|GH_|ACTIONS_|SSH_|AWS_SECRET)|(?:TOKEN|SECRET|PASSWORD|CREDENTIAL)/i
|
|
123
|
+
|
|
124
|
+
export const normalizeConfig = (raw, { defaults = DEFAULT_CONFIG } = {}) => {
|
|
125
|
+
assert(raw && typeof raw === 'object' && !Array.isArray(raw), 'root must be an object')
|
|
126
|
+
assertKnownKeys(raw, '', new Set(['days', 'scope', 'via', 'feeds', 'ignore', 'overrides', 'routes', 'issues', 'eval']))
|
|
127
|
+
const config = copyDefaults(defaults)
|
|
128
|
+
|
|
129
|
+
if (raw.days !== undefined) {
|
|
130
|
+
assert(Number.isInteger(raw.days) && raw.days >= 0, 'days must be a non-negative integer')
|
|
131
|
+
config.days = raw.days
|
|
132
|
+
}
|
|
133
|
+
if (raw.scope !== undefined) {
|
|
134
|
+
assert(raw.scope === 'all' || raw.scope === 'direct', 'scope must be all or direct')
|
|
135
|
+
config.scope = raw.scope
|
|
136
|
+
}
|
|
137
|
+
if (raw.via !== undefined) {
|
|
138
|
+
assert(raw.via === null || (typeof raw.via === 'string' && raw.via.length > 0), 'via must be null or a non-empty string')
|
|
139
|
+
config.via = raw.via
|
|
140
|
+
}
|
|
141
|
+
if (raw.feeds !== undefined) {
|
|
142
|
+
assert(raw.feeds && typeof raw.feeds === 'object' && !Array.isArray(raw.feeds), 'feeds must be an object')
|
|
143
|
+
assertKnownKeys(raw.feeds, 'feeds', new Set(['allow_vendored_fallback']))
|
|
144
|
+
if (raw.feeds.allow_vendored_fallback !== undefined) {
|
|
145
|
+
assert(typeof raw.feeds.allow_vendored_fallback === 'boolean', 'feeds.allow_vendored_fallback must be boolean')
|
|
146
|
+
config.feeds.allow_vendored_fallback = raw.feeds.allow_vendored_fallback
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
if (raw.ignore !== undefined) {
|
|
150
|
+
const ignore = normalizeIgnore(raw.ignore, 'ignore')
|
|
151
|
+
if (raw.ignore.models !== undefined) config.ignore.models = ignore.models
|
|
152
|
+
if (raw.ignore.paths !== undefined) config.ignore.paths = ignore.paths
|
|
153
|
+
}
|
|
154
|
+
if (raw.overrides !== undefined) {
|
|
155
|
+
assert(Array.isArray(raw.overrides), 'overrides must be an array')
|
|
156
|
+
config.overrides = raw.overrides.map(normalizeOverride)
|
|
157
|
+
}
|
|
158
|
+
if (raw.routes !== undefined) {
|
|
159
|
+
assert(Array.isArray(raw.routes), 'routes must be an array')
|
|
160
|
+
config.routes = raw.routes.map(normalizeRoute)
|
|
161
|
+
}
|
|
162
|
+
if (raw.issues !== undefined) {
|
|
163
|
+
assert(raw.issues && typeof raw.issues === 'object' && !Array.isArray(raw.issues), 'issues must be an object')
|
|
164
|
+
assertKnownKeys(raw.issues, 'issues', new Set(['enabled']))
|
|
165
|
+
if (raw.issues.enabled !== undefined) assert(typeof raw.issues.enabled === 'boolean', 'issues.enabled must be boolean')
|
|
166
|
+
if (raw.issues.enabled !== undefined) config.issues.enabled = raw.issues.enabled
|
|
167
|
+
}
|
|
168
|
+
if (raw.eval !== undefined) {
|
|
169
|
+
assert(raw.eval && typeof raw.eval === 'object' && !Array.isArray(raw.eval), 'eval must be an object')
|
|
170
|
+
assertKnownKeys(raw.eval, 'eval', new Set(['command', 'timeout_ms', 'max_report_bytes', 'pass_env']))
|
|
171
|
+
if (raw.eval.command !== undefined) {
|
|
172
|
+
assert(raw.eval.command === null || (typeof raw.eval.command === 'string' && raw.eval.command.length > 0), 'eval.command must be null or a non-empty string')
|
|
173
|
+
config.eval.command = raw.eval.command
|
|
174
|
+
}
|
|
175
|
+
if (raw.eval.timeout_ms !== undefined) {
|
|
176
|
+
assert(Number.isSafeInteger(raw.eval.timeout_ms) && raw.eval.timeout_ms >= MIN_TIMEOUT_MS && raw.eval.timeout_ms <= MAX_TIMEOUT_MS, `eval.timeout_ms must be a safe integer between ${MIN_TIMEOUT_MS} and ${MAX_TIMEOUT_MS}`)
|
|
177
|
+
config.eval.timeout_ms = raw.eval.timeout_ms
|
|
178
|
+
}
|
|
179
|
+
if (raw.eval.max_report_bytes !== undefined) {
|
|
180
|
+
assert(Number.isSafeInteger(raw.eval.max_report_bytes) && raw.eval.max_report_bytes > 0 && raw.eval.max_report_bytes <= MAX_REPORT_BYTES, `eval.max_report_bytes must be a safe positive integer <= ${MAX_REPORT_BYTES}`)
|
|
181
|
+
config.eval.max_report_bytes = raw.eval.max_report_bytes
|
|
182
|
+
}
|
|
183
|
+
if (raw.eval.pass_env !== undefined) {
|
|
184
|
+
const passEnv = stringArray(raw.eval.pass_env, 'eval.pass_env')
|
|
185
|
+
for (const name of passEnv) assert(!CREDENTIAL_ENV_PATTERN.test(name), `eval.pass_env variable "${name}" is not allowed`)
|
|
186
|
+
config.eval.pass_env = passEnv
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
return config
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
export const configForRepoPath = (config, repoPath) => {
|
|
194
|
+
const normalizedPath = normalizeRepoPath(repoPath)
|
|
195
|
+
const policy = {
|
|
196
|
+
days: config.days,
|
|
197
|
+
scope: config.scope,
|
|
198
|
+
via: config.via,
|
|
199
|
+
ignore: {
|
|
200
|
+
models: [...config.ignore.models],
|
|
201
|
+
paths: [...config.ignore.paths],
|
|
202
|
+
},
|
|
203
|
+
override_indexes: [],
|
|
204
|
+
}
|
|
205
|
+
for (const [index, override] of (config.overrides ?? []).entries()) {
|
|
206
|
+
if (!matchesAnyGlob(normalizedPath, override.paths)) continue
|
|
207
|
+
policy.override_indexes.push(index)
|
|
208
|
+
if (override.days !== undefined) policy.days = override.days
|
|
209
|
+
if (override.scope !== undefined) policy.scope = override.scope
|
|
210
|
+
if (override.via !== undefined) policy.via = override.via
|
|
211
|
+
if (override.ignore) {
|
|
212
|
+
policy.ignore.models.push(...override.ignore.models)
|
|
213
|
+
policy.ignore.paths.push(...override.ignore.paths)
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
policy.ignore.models = [...new Set(policy.ignore.models)]
|
|
217
|
+
policy.ignore.paths = [...new Set(policy.ignore.paths)]
|
|
218
|
+
return policy
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
export const routeForRepoReference = (config, repoPath, { matched = null, id = null } = {}) => {
|
|
222
|
+
const normalizedPath = normalizeRepoPath(repoPath)
|
|
223
|
+
let selected = null
|
|
224
|
+
for (const [index, route] of (config.routes ?? []).entries()) {
|
|
225
|
+
if (!matchesAnyGlob(normalizedPath, route.paths)) continue
|
|
226
|
+
if (route.match !== undefined && route.match !== matched && route.match !== id) continue
|
|
227
|
+
selected = { ...route, index }
|
|
228
|
+
}
|
|
229
|
+
return selected
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
export const mappedRoutesForRepoPath = (config, repoPath) => {
|
|
233
|
+
const normalizedPath = normalizeRepoPath(repoPath)
|
|
234
|
+
return (config.routes ?? []).flatMap((route, index) =>
|
|
235
|
+
route.match !== undefined && route.model !== undefined && matchesAnyGlob(normalizedPath, route.paths)
|
|
236
|
+
? [{ ...route, index }]
|
|
237
|
+
: [])
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
export const configuredVias = config => [
|
|
241
|
+
config.via,
|
|
242
|
+
...(config.overrides ?? []).map(override => override.via),
|
|
243
|
+
...(config.routes ?? []).map(route => route.via),
|
|
244
|
+
].filter(via => typeof via === 'string' && via.length > 0)
|
|
245
|
+
|
|
246
|
+
export const loadConfig = (file, { defaults = DEFAULT_CONFIG, allowMissing = true } = {}) => {
|
|
247
|
+
if (!fs.existsSync(file)) {
|
|
248
|
+
if (allowMissing) return normalizeConfig({}, { defaults })
|
|
249
|
+
throw new Error(`could not read config ${file}: file not found`)
|
|
250
|
+
}
|
|
251
|
+
let value
|
|
252
|
+
try {
|
|
253
|
+
value = JSON.parse(fs.readFileSync(file, 'utf8'))
|
|
254
|
+
} catch (error) {
|
|
255
|
+
throw new Error(`could not read config ${file}: ${error.message}`)
|
|
256
|
+
}
|
|
257
|
+
return normalizeConfig(value, { defaults })
|
|
258
|
+
}
|
package/lib/feeds.mjs
CHANGED
|
@@ -5,12 +5,23 @@ import { assertIsoDate, assertValidFeed } from './validate-feed.mjs'
|
|
|
5
5
|
|
|
6
6
|
export { assertIsoDate }
|
|
7
7
|
|
|
8
|
+
export const BUILTIN_CHANNELS = Object.freeze([
|
|
9
|
+
'azure-ai-foundry',
|
|
10
|
+
'aws-bedrock',
|
|
11
|
+
'vertex-ai',
|
|
12
|
+
'openrouter',
|
|
13
|
+
'litellm',
|
|
14
|
+
'portkey',
|
|
15
|
+
'ai-gateway',
|
|
16
|
+
])
|
|
17
|
+
|
|
8
18
|
export const esc = s => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
|
9
19
|
|
|
10
20
|
export const loadFeeds = feedsDir => {
|
|
11
21
|
const feeds = []
|
|
12
22
|
const entries = new Map()
|
|
13
23
|
const keyFiles = new Map()
|
|
24
|
+
const vias = new Set()
|
|
14
25
|
|
|
15
26
|
for (const f of fs.readdirSync(feedsDir).filter(f => f.endsWith('.json'))) {
|
|
16
27
|
const file = path.join(feedsDir, f)
|
|
@@ -22,6 +33,7 @@ export const loadFeeds = feedsDir => {
|
|
|
22
33
|
assertValidFeed(feed, file)
|
|
23
34
|
feeds.push({ ...feed, file })
|
|
24
35
|
for (const m of feed.models ?? []) {
|
|
36
|
+
for (const distribution of m.distributions ?? []) vias.add(distribution.via)
|
|
25
37
|
const record = {
|
|
26
38
|
entry: m,
|
|
27
39
|
publisher: feed.publisher,
|
|
@@ -45,6 +57,7 @@ export const loadFeeds = feedsDir => {
|
|
|
45
57
|
feeds,
|
|
46
58
|
entries,
|
|
47
59
|
keys: [...entries.keys()].sort((a, b) => b.length - a.length),
|
|
60
|
+
vias: [...vias].sort(),
|
|
48
61
|
}
|
|
49
62
|
}
|
|
50
63
|
|
|
@@ -71,13 +84,18 @@ const policySafeUntil = (policy, generated) => {
|
|
|
71
84
|
export const effectiveShutdown = (entry, via) => {
|
|
72
85
|
if (via) {
|
|
73
86
|
const d = (entry.distributions ?? []).find(d => d.via === via)
|
|
74
|
-
if (d) return {
|
|
87
|
+
if (d) return {
|
|
88
|
+
date: d.shutdown ?? null,
|
|
89
|
+
via,
|
|
90
|
+
date_precision: d.date_precision ?? null,
|
|
91
|
+
distribution_status: d.status ?? null,
|
|
92
|
+
}
|
|
75
93
|
if (entry.shutdown || entry.announced) {
|
|
76
|
-
return { date: entry.shutdown ?? null, via: 'publisher-fallback', date_precision: entry.date_precision ?? null }
|
|
94
|
+
return { date: entry.shutdown ?? null, via: 'publisher-fallback', date_precision: entry.date_precision ?? null, distribution_status: null }
|
|
77
95
|
}
|
|
78
96
|
return null
|
|
79
97
|
}
|
|
80
|
-
return entry.shutdown ? { date: entry.shutdown, via: 'publisher', date_precision: entry.date_precision ?? null } : null
|
|
98
|
+
return entry.shutdown ? { date: entry.shutdown, via: 'publisher', date_precision: entry.date_precision ?? null, distribution_status: null } : null
|
|
81
99
|
}
|
|
82
100
|
|
|
83
101
|
export const lifecycleFor = (entry, options) => {
|
|
@@ -90,12 +108,24 @@ export const lifecycleFor = (entry, options) => {
|
|
|
90
108
|
} = options
|
|
91
109
|
const sd = effectiveShutdown(entry, via)
|
|
92
110
|
if (sd) {
|
|
111
|
+
if (sd.distribution_status === 'retired') {
|
|
112
|
+
return {
|
|
113
|
+
status: 'retired',
|
|
114
|
+
shutdown: sd.date,
|
|
115
|
+
date_precision: sd.date_precision,
|
|
116
|
+
distribution_status: sd.distribution_status,
|
|
117
|
+
via: sd.via,
|
|
118
|
+
days: sd.date ? daysUntil(sd.date, today) : null,
|
|
119
|
+
safe_until: sd.date,
|
|
120
|
+
}
|
|
121
|
+
}
|
|
93
122
|
if (sd.date) {
|
|
94
123
|
const remaining = daysUntil(sd.date, today)
|
|
95
124
|
return {
|
|
96
125
|
status: remaining < 0 ? 'retired' : remaining <= days ? 'retiring' : 'scheduled',
|
|
97
126
|
shutdown: sd.date,
|
|
98
127
|
date_precision: sd.date_precision,
|
|
128
|
+
distribution_status: sd.distribution_status,
|
|
99
129
|
via: sd.via,
|
|
100
130
|
days: remaining,
|
|
101
131
|
safe_until: sd.date,
|
|
@@ -110,6 +140,7 @@ export const lifecycleFor = (entry, options) => {
|
|
|
110
140
|
status,
|
|
111
141
|
shutdown: null,
|
|
112
142
|
date_precision: sd.date_precision,
|
|
143
|
+
distribution_status: sd.distribution_status,
|
|
113
144
|
via: sd.via,
|
|
114
145
|
days: null,
|
|
115
146
|
safe_until: status === 'ok' && !entry.announced && !entry.shutdown
|
|
@@ -118,12 +149,13 @@ export const lifecycleFor = (entry, options) => {
|
|
|
118
149
|
}
|
|
119
150
|
}
|
|
120
151
|
if (entry.announced) {
|
|
121
|
-
return { status: 'watch', shutdown: null, date_precision: null, via: null, days: null, safe_until: null }
|
|
152
|
+
return { status: 'watch', shutdown: null, date_precision: null, distribution_status: null, via: null, days: null, safe_until: null }
|
|
122
153
|
}
|
|
123
154
|
return {
|
|
124
155
|
status: 'ok',
|
|
125
156
|
shutdown: null,
|
|
126
157
|
date_precision: null,
|
|
158
|
+
distribution_status: null,
|
|
127
159
|
via: null,
|
|
128
160
|
days: null,
|
|
129
161
|
safe_until: policySafeUntil(policy, generated),
|
|
@@ -157,12 +189,14 @@ export const findingFromRef = (ref, options) => {
|
|
|
157
189
|
status: lifecycle.status,
|
|
158
190
|
shutdown: lifecycle.shutdown,
|
|
159
191
|
date_precision: lifecycle.date_precision ?? null,
|
|
192
|
+
distribution_status: lifecycle.distribution_status ?? null,
|
|
160
193
|
via: lifecycle.via,
|
|
161
194
|
days: lifecycle.days,
|
|
162
195
|
safe_until: lifecycle.safe_until,
|
|
163
196
|
replacement: ref.entry.replacement ?? null,
|
|
164
197
|
replacement_options: ref.entry.replacement_options ?? null,
|
|
165
198
|
replacement_note: ref.entry.replacement_note ?? null,
|
|
199
|
+
threshold_days: days,
|
|
166
200
|
}
|
|
167
201
|
}
|
|
168
202
|
|
package/lib/glob.mjs
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
const escapeRegExp = value => value.replace(/[.+^${}()|[\]\\]/g, '\\$&')
|
|
2
|
+
|
|
3
|
+
export const normalizeRepoPath = value => String(value ?? '')
|
|
4
|
+
.replaceAll('\\', '/')
|
|
5
|
+
.replace(/^\.\//, '')
|
|
6
|
+
.replace(/^\/+/, '')
|
|
7
|
+
|
|
8
|
+
export const globRegex = pattern => {
|
|
9
|
+
let source = ''
|
|
10
|
+
const value = normalizeRepoPath(pattern)
|
|
11
|
+
for (let i = 0; i < value.length; i++) {
|
|
12
|
+
const char = value[i]
|
|
13
|
+
if (char === '*' && value[i + 1] === '*') {
|
|
14
|
+
if (value[i + 2] === '/') {
|
|
15
|
+
source += '(?:.*/)?'
|
|
16
|
+
i += 2
|
|
17
|
+
} else {
|
|
18
|
+
source += '.*'
|
|
19
|
+
i++
|
|
20
|
+
}
|
|
21
|
+
} else if (char === '*') {
|
|
22
|
+
source += '[^/]*'
|
|
23
|
+
} else if (char === '?') {
|
|
24
|
+
source += '[^/]'
|
|
25
|
+
} else {
|
|
26
|
+
source += escapeRegExp(char)
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
return new RegExp(`^${source}$`)
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export const matchesGlob = (file, pattern) => globRegex(pattern).test(normalizeRepoPath(file))
|
|
33
|
+
|
|
34
|
+
export const matchesAnyGlob = (file, patterns = []) =>
|
|
35
|
+
patterns.some(pattern => matchesGlob(file, pattern))
|
package/lib/plan.mjs
CHANGED
|
@@ -22,6 +22,10 @@ const readLines = (file, cache) => {
|
|
|
22
22
|
|
|
23
23
|
const notesFor = ref => [ref.feedNote, ref.entry.notes].filter(Boolean).join(' - ') || null
|
|
24
24
|
|
|
25
|
+
const requestedVia = (finding, fallback) => Object.hasOwn(finding, 'requested_via')
|
|
26
|
+
? finding.requested_via
|
|
27
|
+
: fallback
|
|
28
|
+
|
|
25
29
|
const issueFor = (ref, finding, reason, via) => ({
|
|
26
30
|
file: ref.file,
|
|
27
31
|
line: ref.line,
|
|
@@ -33,6 +37,11 @@ const issueFor = (ref, finding, reason, via) => ({
|
|
|
33
37
|
status: finding.status,
|
|
34
38
|
shutdown: finding.shutdown,
|
|
35
39
|
via: finding.via,
|
|
40
|
+
requested_via: requestedVia(finding, via),
|
|
41
|
+
threshold_days: finding.threshold_days,
|
|
42
|
+
effective_scope: finding.effective_scope,
|
|
43
|
+
...(finding.policy_provenance ? { policy_provenance: finding.policy_provenance } : {}),
|
|
44
|
+
...(ref.mapped_from ? { mapped_from: ref.mapped_from } : {}),
|
|
36
45
|
days: finding.days,
|
|
37
46
|
replacement: finding.replacement,
|
|
38
47
|
...(finding.replacement_options?.length ? { replacement_options: finding.replacement_options } : {}),
|
|
@@ -42,8 +51,8 @@ const issueFor = (ref, finding, reason, via) => ({
|
|
|
42
51
|
notes: notesFor(ref),
|
|
43
52
|
})
|
|
44
53
|
|
|
45
|
-
const scopeIncludes = (ref, scope) =>
|
|
46
|
-
scope !== 'direct' || ref.usage === 'direct-api' || ref.usage === 'model-reference'
|
|
54
|
+
const scopeIncludes = (ref, scope, finding = null) =>
|
|
55
|
+
(finding?.effective_scope ?? scope) !== 'direct' || ref.usage === 'direct-api' || ref.usage === 'model-reference'
|
|
47
56
|
|
|
48
57
|
const occurrenceInLine = (line, matched, index) => {
|
|
49
58
|
let occurrence = 0
|
|
@@ -62,13 +71,16 @@ export const buildPlan = ({ scan, findings, entries, days, via, scope }) => {
|
|
|
62
71
|
const lineCache = new Map()
|
|
63
72
|
|
|
64
73
|
scan.modelRefs.forEach((ref, index) => {
|
|
65
|
-
if (!scopeIncludes(ref, scope)) return
|
|
66
74
|
const finding = findings[index]
|
|
75
|
+
if (!scopeIncludes(ref, scope, finding)) return
|
|
67
76
|
if (!finding || finding.status === 'ok') return
|
|
77
|
+
const effectiveVia = requestedVia(finding, via)
|
|
78
|
+
const effectiveDays = finding.threshold_days ?? days
|
|
68
79
|
|
|
69
80
|
let reason
|
|
70
81
|
if (finding.status === 'watch') reason = 'watch'
|
|
71
82
|
else if (finding.status !== 'retired' && finding.status !== 'retiring') reason = 'outside-threshold'
|
|
83
|
+
else if (!finding.shutdown) reason = 'shutdown-date-unavailable'
|
|
72
84
|
else if (finding.via === 'publisher-fallback') reason = 'publisher-fallback'
|
|
73
85
|
else if (ref.usage !== 'direct-api' || ref.confidence !== 'high') reason = 'not-direct-api'
|
|
74
86
|
else if (!finding.replacement && finding.replacement_options?.length) reason = 'replacement-choice'
|
|
@@ -77,7 +89,7 @@ export const buildPlan = ({ scan, findings, entries, days, via, scope }) => {
|
|
|
77
89
|
const replacementRecord = entries.get(finding.replacement)
|
|
78
90
|
if (!replacementRecord || replacementRecord.publisher !== finding.publisher) reason = 'replacement-unresolved'
|
|
79
91
|
else {
|
|
80
|
-
const replacementLifecycle = lifecycleFor(replacementRecord.entry, { days, via })
|
|
92
|
+
const replacementLifecycle = lifecycleFor(replacementRecord.entry, { days: effectiveDays, via: effectiveVia })
|
|
81
93
|
if (isBad(replacementLifecycle)) reason = 'replacement-retiring'
|
|
82
94
|
}
|
|
83
95
|
}
|
|
@@ -92,7 +104,7 @@ export const buildPlan = ({ scan, findings, entries, days, via, scope }) => {
|
|
|
92
104
|
if (line === undefined) reason = reason ?? 'line-unavailable'
|
|
93
105
|
|
|
94
106
|
if (reason) {
|
|
95
|
-
issues.push(issueFor(ref, finding, reason,
|
|
107
|
+
issues.push(issueFor(ref, finding, reason, effectiveVia))
|
|
96
108
|
return
|
|
97
109
|
}
|
|
98
110
|
|
|
@@ -100,7 +112,7 @@ export const buildPlan = ({ scan, findings, entries, days, via, scope }) => {
|
|
|
100
112
|
? ref.occurrence
|
|
101
113
|
: occurrenceInLine(line, ref.matched, line.indexOf(ref.matched))
|
|
102
114
|
if (line.indexOf(ref.matched) === -1) {
|
|
103
|
-
issues.push(issueFor(ref, finding, 'match-unavailable',
|
|
115
|
+
issues.push(issueFor(ref, finding, 'match-unavailable', effectiveVia))
|
|
104
116
|
return
|
|
105
117
|
}
|
|
106
118
|
|
|
@@ -116,19 +128,26 @@ export const buildPlan = ({ scan, findings, entries, days, via, scope }) => {
|
|
|
116
128
|
shutdown: finding.shutdown,
|
|
117
129
|
days: finding.days,
|
|
118
130
|
status: finding.status,
|
|
119
|
-
|
|
131
|
+
requested_via: effectiveVia,
|
|
132
|
+
threshold_days: effectiveDays,
|
|
133
|
+
effective_scope: finding.effective_scope ?? scope,
|
|
134
|
+
...(finding.policy_provenance ? { policy_provenance: finding.policy_provenance } : {}),
|
|
135
|
+
...(ref.mapped_from ? { mapped_from: ref.mapped_from } : {}),
|
|
136
|
+
sources: sourceList(ref, finding, effectiveVia),
|
|
120
137
|
notes: notesFor(ref),
|
|
121
138
|
})
|
|
122
139
|
})
|
|
123
140
|
|
|
124
141
|
for (const candidate of scan.candidateRefs) {
|
|
125
|
-
if (scope === 'direct' && candidate.usage !== 'direct-api' && candidate.usage !== 'model-reference') continue
|
|
142
|
+
if ((candidate.effective_scope ?? scope) === 'direct' && candidate.usage !== 'direct-api' && candidate.usage !== 'model-reference') continue
|
|
126
143
|
issues.push({
|
|
127
144
|
file: candidate.file,
|
|
128
145
|
line: candidate.line,
|
|
129
146
|
matched: candidate.matched,
|
|
130
147
|
usage: candidate.usage,
|
|
131
148
|
confidence: candidate.confidence,
|
|
149
|
+
effective_scope: candidate.effective_scope ?? scope,
|
|
150
|
+
...(candidate.policy_provenance ? { policy_provenance: candidate.policy_provenance } : {}),
|
|
132
151
|
reason: 'candidate',
|
|
133
152
|
notes: candidate.reason,
|
|
134
153
|
})
|
package/lib/reports.mjs
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { isBad } from './feeds.mjs'
|
|
2
2
|
import { color } from './color.mjs'
|
|
3
|
+
import { incompleteScanNotes } from './scanner.mjs'
|
|
3
4
|
|
|
4
5
|
export const marker = status =>
|
|
5
6
|
status === 'retired' ? '✗' :
|
|
@@ -18,9 +19,10 @@ const replacementTail = finding => finding.replacement
|
|
|
18
19
|
: ''
|
|
19
20
|
|
|
20
21
|
export const findingTail = (f, days) =>
|
|
22
|
+
f.status === 'retired' && !f.shutdown ? `RETIRED (shutdown date unavailable)${replacementTail(f)}` :
|
|
21
23
|
f.status === 'retired' ? `RETIRED ${shutdownText(f)} (${-f.days} days ago)${replacementTail(f)}` :
|
|
22
24
|
f.status === 'retiring' ? `RETIRES ${shutdownText(f)} (${f.days} days)${replacementTail(f)}` :
|
|
23
|
-
f.status === 'scheduled' ? `scheduled ${shutdownText(f)} (${f.days} days, outside --days ${days})` :
|
|
25
|
+
f.status === 'scheduled' ? `scheduled ${shutdownText(f)} (${f.days} days, outside --days ${f.threshold_days ?? days})` :
|
|
24
26
|
f.status === 'watch' ? 'deprecation announced, no shutdown date yet' :
|
|
25
27
|
'no retirement scheduled'
|
|
26
28
|
|
|
@@ -53,8 +55,14 @@ export const buildInventory = ({ scan, findings, days, via, scope, targets }) =>
|
|
|
53
55
|
status: lifecycle.status,
|
|
54
56
|
shutdown: lifecycle.shutdown,
|
|
55
57
|
date_precision: lifecycle.date_precision,
|
|
58
|
+
distribution_status: lifecycle.distribution_status,
|
|
56
59
|
via: lifecycle.via,
|
|
60
|
+
requested_via: lifecycle.requested_via,
|
|
57
61
|
days: lifecycle.days,
|
|
62
|
+
threshold_days: lifecycle.threshold_days,
|
|
63
|
+
effective_scope: lifecycle.effective_scope,
|
|
64
|
+
...(lifecycle.policy_provenance ? { policy_provenance: lifecycle.policy_provenance } : {}),
|
|
65
|
+
...(ref.mapped_from ? { mapped_from: ref.mapped_from } : {}),
|
|
58
66
|
safe_until: lifecycle.safe_until,
|
|
59
67
|
replacement: lifecycle.replacement,
|
|
60
68
|
source: ref.source,
|
|
@@ -87,7 +95,7 @@ export const buildInventory = ({ scan, findings, days, via, scope, targets }) =>
|
|
|
87
95
|
export const buildSchedule = inventory => {
|
|
88
96
|
const severity = { retired: 0, retiring: 1, scheduled: 2, watch: 3, ok: 4 }
|
|
89
97
|
const inScope = item =>
|
|
90
|
-
inventory.scope !== 'direct' || item.usage === 'direct-api' || item.usage === 'model-reference'
|
|
98
|
+
(item.effective_scope ?? inventory.scope) !== 'direct' || item.usage === 'direct-api' || item.usage === 'model-reference'
|
|
91
99
|
const items = inventory.model_references
|
|
92
100
|
.filter(item => item.status !== 'ok')
|
|
93
101
|
.filter(inScope)
|
|
@@ -172,7 +180,8 @@ export const formatCheck = ({ findings, bad, scannedFiles, days, scope }) => {
|
|
|
172
180
|
|
|
173
181
|
export const formatInventory = (inventory, days) => {
|
|
174
182
|
const out = [color('bold', `model-eol inventory: ${inventory.summary.model_references} tracked model reference(s), ${inventory.summary.candidate_model_references} candidate(s), ${inventory.summary.integration_hints} integration hint(s) across ${inventory.scanned_files} files`)]
|
|
175
|
-
|
|
183
|
+
const incompleteNotes = incompleteScanNotes(inventory.scan_notes ?? [])
|
|
184
|
+
if (incompleteNotes.length) out.push(`WARNING: scan incomplete; ${incompleteNotes.length} coverage-loss note(s) recorded`)
|
|
176
185
|
if (inventory.model_references.length) {
|
|
177
186
|
out.push('\nModel references')
|
|
178
187
|
for (const item of inventory.model_references) {
|
|
@@ -197,7 +206,8 @@ export const formatInventory = (inventory, days) => {
|
|
|
197
206
|
|
|
198
207
|
export const formatSchedule = (schedule, days) => {
|
|
199
208
|
const out = [color('bold', `model-eol schedule: ${schedule.items.length} scheduled/watch finding(s), ${schedule.candidate_model_references.length} candidate(s), ${schedule.unresolved_integrations.length} unresolved cloud/gateway hint(s)`)]
|
|
200
|
-
|
|
209
|
+
const incompleteNotes = incompleteScanNotes(schedule.scan_notes ?? [])
|
|
210
|
+
if (incompleteNotes.length) out.push(`WARNING: scan incomplete; ${incompleteNotes.length} coverage-loss note(s) recorded`)
|
|
201
211
|
for (const item of schedule.items) {
|
|
202
212
|
const safeUntil = item.safe_until ? ` [safe until ${item.safe_until}]` : ''
|
|
203
213
|
out.push(humanFinding(item.status, `${marker(item.status)} ${item.shutdown ? shutdownText(item) : 'TBD'} ${item.file}:${item.line} ${item.matched} ${findingTail(item, days)} [${item.usage}]${safeUntil}`))
|
|
@@ -247,7 +257,12 @@ export const formatInventoryCycloneDX = inventory => {
|
|
|
247
257
|
bomFormat: 'CycloneDX',
|
|
248
258
|
specVersion: '1.6',
|
|
249
259
|
version: 1,
|
|
250
|
-
metadata: {
|
|
260
|
+
metadata: {
|
|
261
|
+
timestamp: inventory.generated,
|
|
262
|
+
properties: [
|
|
263
|
+
{ name: 'model-eol:generator', value: 'model-eol/inventory-cyclonedx@0.1' },
|
|
264
|
+
],
|
|
265
|
+
},
|
|
251
266
|
components: [...components.values()],
|
|
252
267
|
}
|
|
253
268
|
}
|
|
@@ -261,7 +276,8 @@ const annotationEscape = value => String(value)
|
|
|
261
276
|
|
|
262
277
|
export const formatAlertGithub = (alert, days) => {
|
|
263
278
|
const out = []
|
|
264
|
-
|
|
279
|
+
const incompleteNotes = incompleteScanNotes(alert.scan_notes ?? [])
|
|
280
|
+
if (incompleteNotes.length) out.push(`::warning title=${annotationEscape('model-eol scan')}::${annotationEscape(`Scan incomplete; ${incompleteNotes.length} coverage-loss note(s) recorded`)}`)
|
|
265
281
|
for (const item of alert.errors) {
|
|
266
282
|
const title = `model-eol ${item.status}: ${item.matched}`
|
|
267
283
|
out.push(`::error file=${annotationEscape(item.file)},line=${item.line},title=${annotationEscape(title)}::${annotationEscape(findingTail(item, days))}`)
|
|
@@ -283,7 +299,8 @@ export const formatAlertGithub = (alert, days) => {
|
|
|
283
299
|
|
|
284
300
|
export const formatAlertMarkdown = (alert, days) => {
|
|
285
301
|
const out = ['# model-eol alert', '']
|
|
286
|
-
|
|
302
|
+
const incompleteNotes = incompleteScanNotes(alert.scan_notes ?? [])
|
|
303
|
+
if (incompleteNotes.length) out.push(`> Warning: scan incomplete; ${incompleteNotes.length} coverage-loss note(s) recorded.`, '')
|
|
287
304
|
out.push(`- Errors: ${alert.errors.length}`)
|
|
288
305
|
out.push(`- Warnings: ${alert.warnings.length}`)
|
|
289
306
|
if (alert.errors.length) {
|