model-eol 0.9.1 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -4
- package/check.mjs +153 -62
- package/examples/workflows/model-eol.yml +3 -2
- package/lib/changed.mjs +627 -0
- package/lib/config.mjs +12 -7
- package/lib/glob.mjs +11 -3
- package/lib/reports.mjs +24 -1
- package/lib/scanner.mjs +98 -20
- package/package.json +2 -2
- package/schema/model-eol.check.schema.json +53 -1
package/lib/changed.mjs
ADDED
|
@@ -0,0 +1,627 @@
|
|
|
1
|
+
import fs from 'node:fs'
|
|
2
|
+
import os from 'node:os'
|
|
3
|
+
import path from 'node:path'
|
|
4
|
+
import { spawnSync } from 'node:child_process'
|
|
5
|
+
|
|
6
|
+
import { configForRepoPath, mappedRoutesForRepoPath, repoPathMatches, routesForRepoPath } from './config.mjs'
|
|
7
|
+
import { isBad } from './feeds.mjs'
|
|
8
|
+
import { MAX_FILE_BYTES, SKIP_DIRS, baseLineFor, changedLinesForTargets, displayPath, incompleteScanNotes, isAddedLine } from './scanner.mjs'
|
|
9
|
+
|
|
10
|
+
export class BaseUnavailableError extends Error {
|
|
11
|
+
constructor(message) {
|
|
12
|
+
super(message)
|
|
13
|
+
this.name = 'BaseUnavailableError'
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export const inCheckScope = finding =>
|
|
18
|
+
finding.effective_scope !== 'direct' || finding.usage === 'direct-api' || finding.usage === 'model-reference'
|
|
19
|
+
|
|
20
|
+
const MAX_SYMLINK_DEPTH = 40
|
|
21
|
+
const SUBMODULE_GIT_CONFIG = ['-c', 'protocol.allow=never', '-c', 'core.fsmonitor=false']
|
|
22
|
+
|
|
23
|
+
const runGit = (gitRoot, label, args, { input, hardened = false } = {}) => {
|
|
24
|
+
const result = spawnSync('git', hardened ? [...SUBMODULE_GIT_CONFIG, ...args] : args, {
|
|
25
|
+
cwd: gitRoot,
|
|
26
|
+
input,
|
|
27
|
+
env: hardened ? { ...process.env, GIT_NO_LAZY_FETCH: '1' } : process.env,
|
|
28
|
+
stdio: [input === undefined ? 'ignore' : 'pipe', 'pipe', 'pipe'],
|
|
29
|
+
maxBuffer: 1024 * 1024 * 1024,
|
|
30
|
+
})
|
|
31
|
+
if (result.error || result.status !== 0) {
|
|
32
|
+
const detail = String(result.stderr ?? '').trim() || result.error?.message || `exit ${result.status}`
|
|
33
|
+
throw new Error(`git ${label} failed: ${detail}`)
|
|
34
|
+
}
|
|
35
|
+
return result.stdout
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const nulRecords = buffer => buffer.toString('utf8').split('\0').filter(Boolean)
|
|
39
|
+
|
|
40
|
+
const repoRelative = (gitRoot, absolute) => {
|
|
41
|
+
const relative = path.relative(gitRoot, absolute)
|
|
42
|
+
if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) return null
|
|
43
|
+
return relative.split(path.sep).join('/')
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const onDiskName = (directory, name) => {
|
|
47
|
+
try {
|
|
48
|
+
const names = fs.readdirSync(directory)
|
|
49
|
+
if (names.includes(name)) return name
|
|
50
|
+
const wanted = fs.lstatSync(path.join(directory, name))
|
|
51
|
+
return names.find(other => {
|
|
52
|
+
const stat = fs.lstatSync(path.join(directory, other))
|
|
53
|
+
return stat.dev === wanted.dev && stat.ino === wanted.ino
|
|
54
|
+
}) ?? name
|
|
55
|
+
} catch {
|
|
56
|
+
return name
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
const lexicalRepoPath = (gitRoot, absolute) => {
|
|
61
|
+
const parts = path.resolve(absolute).split(path.sep)
|
|
62
|
+
for (let index = 1; index <= parts.length; index++) {
|
|
63
|
+
let real
|
|
64
|
+
try {
|
|
65
|
+
real = fs.realpathSync(parts.slice(0, index).join(path.sep) || path.sep)
|
|
66
|
+
} catch {
|
|
67
|
+
return null
|
|
68
|
+
}
|
|
69
|
+
if (real !== gitRoot) continue
|
|
70
|
+
const names = []
|
|
71
|
+
for (const part of parts.slice(index)) names.push(onDiskName(path.join(gitRoot, ...names), part))
|
|
72
|
+
return names.join('/')
|
|
73
|
+
}
|
|
74
|
+
return null
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const nativeRealpath = target => {
|
|
78
|
+
try {
|
|
79
|
+
return fs.realpathSync.native(target)
|
|
80
|
+
} catch {
|
|
81
|
+
return null
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
const repoLocation = (gitRoot, target) => {
|
|
86
|
+
const physical = nativeRealpath(target)
|
|
87
|
+
const lexical = lexicalRepoPath(gitRoot, target)
|
|
88
|
+
// path.resolve folds "link/.." before the OS follows link, so trust the lexical path only when both agree.
|
|
89
|
+
if (lexical !== null && (physical === null || nativeRealpath(path.resolve(target)) === physical)) return lexical
|
|
90
|
+
return physical === null ? null : repoRelative(gitRoot, physical)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
const isIgnored = (gitRoot, relative) => {
|
|
94
|
+
if (relative === '') return false
|
|
95
|
+
const result = spawnSync('git', ['check-ignore', '-q', '--', `./${relative}`], {
|
|
96
|
+
cwd: gitRoot,
|
|
97
|
+
encoding: 'utf8',
|
|
98
|
+
stdio: ['ignore', 'ignore', 'pipe'],
|
|
99
|
+
})
|
|
100
|
+
if (!result.error && (result.status === 0 || result.status === 1)) return result.status === 0
|
|
101
|
+
const detail = String(result.stderr ?? '').trim() || result.error?.message || `exit ${result.status}`
|
|
102
|
+
if (result.status === 128 && /in submodule|beyond a symbolic link/i.test(detail)) return false
|
|
103
|
+
throw new Error(`git check-ignore failed: ${detail}`)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const readRegular = absolute => {
|
|
107
|
+
let descriptor
|
|
108
|
+
try {
|
|
109
|
+
descriptor = fs.openSync(absolute, fs.constants.O_RDONLY | (fs.constants.O_NONBLOCK ?? 0))
|
|
110
|
+
} catch (error) {
|
|
111
|
+
return error.code === 'ENOENT' ? { bytes: null } : { message: error.message }
|
|
112
|
+
}
|
|
113
|
+
try {
|
|
114
|
+
return fs.fstatSync(descriptor).isFile() ? { bytes: fs.readFileSync(descriptor) } : { message: 'not a regular file' }
|
|
115
|
+
} catch (error) {
|
|
116
|
+
return { message: error.message }
|
|
117
|
+
} finally {
|
|
118
|
+
fs.closeSync(descriptor)
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
const readHead = absolute => {
|
|
123
|
+
const result = readRegular(absolute)
|
|
124
|
+
if (result.message) throw new Error(`could not read ${absolute}: ${result.message}`)
|
|
125
|
+
return result.bytes
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const scannableTreePath = relative => relative.split('/').every(part =>
|
|
129
|
+
part && part !== '.' && part !== '..' && part.toLowerCase() !== '.git' && !SKIP_DIRS.has(part))
|
|
130
|
+
|
|
131
|
+
const regularBlob = entry => entry.type === 'blob' && (entry.mode === '100644' || entry.mode === '100755')
|
|
132
|
+
|
|
133
|
+
const untrackedFiles = (gitRoot, relativeTargets) => new Set(nulRecords(runGit(gitRoot, 'ls-files', [
|
|
134
|
+
'-c', 'core.fsmonitor=false', '--literal-pathspecs', 'ls-files', '-z', '--others', '--exclude-standard', '--', ...relativeTargets,
|
|
135
|
+
])).filter(relative => !relative.endsWith('/')).map(relative => path.resolve(gitRoot, relative)))
|
|
136
|
+
|
|
137
|
+
const HASH_CHUNK_BYTES = 64 * 1024
|
|
138
|
+
|
|
139
|
+
// Hashes raw bytes with paths as literal arguments: --stdin-paths unquotes names, and filters would run PR-routed drivers.
|
|
140
|
+
const rawOids = (gitRoot, relatives) => {
|
|
141
|
+
const oids = []
|
|
142
|
+
for (let start = 0; start < relatives.length;) {
|
|
143
|
+
let end = start
|
|
144
|
+
let bytes = 0
|
|
145
|
+
while (end < relatives.length && (end === start || bytes + relatives[end].length < HASH_CHUNK_BYTES)) bytes += relatives[end++].length + 1
|
|
146
|
+
const chunk = relatives.slice(start, end)
|
|
147
|
+
let output = []
|
|
148
|
+
try {
|
|
149
|
+
output = runGit(gitRoot, 'hash-object', ['hash-object', '--no-filters', '--', ...chunk]).toString('utf8').split('\n')
|
|
150
|
+
} catch {
|
|
151
|
+
}
|
|
152
|
+
for (let index = 0; index < chunk.length; index++) oids.push(output[index] ?? null)
|
|
153
|
+
start = end
|
|
154
|
+
}
|
|
155
|
+
return oids
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// git diff trusts index stat data and skips assume-unchanged and skip-worktree entries, so compare the bytes the scanner reads.
|
|
159
|
+
const editedWithoutHunks = (gitRoot, checks, baseListing) => {
|
|
160
|
+
if (!checks.length) return new Set()
|
|
161
|
+
const listing = baseListing()
|
|
162
|
+
const hashable = checks.filter(check => listing.get(check.baseRelative)?.type === 'blob')
|
|
163
|
+
const oids = rawOids(gitRoot, hashable.map(check => check.headRelative))
|
|
164
|
+
const same = new Set(hashable.filter((check, index) => oids[index] === listing.get(check.baseRelative).oid).map(check => check.absolute))
|
|
165
|
+
return new Set(checks.filter(check => !same.has(check.absolute)).map(check => check.absolute))
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const parseTree = output => {
|
|
169
|
+
const entries = new Map()
|
|
170
|
+
for (const record of nulRecords(output)) {
|
|
171
|
+
const separator = record.indexOf('\t')
|
|
172
|
+
const [mode, type, oid, size] = record.slice(0, separator).split(/ +/)
|
|
173
|
+
if (separator === -1 || !/^[0-7]{6}$/.test(mode) || !/^[0-9a-f]{40,64}$/.test(oid ?? '')) {
|
|
174
|
+
throw new Error('git ls-tree returned a malformed record')
|
|
175
|
+
}
|
|
176
|
+
entries.set(record.slice(separator + 1), { mode, type, oid, size: size === '-' ? null : Number(size) })
|
|
177
|
+
}
|
|
178
|
+
return entries
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const readBlobs = (gitRoot, entries, { hardened = false } = {}) => {
|
|
182
|
+
const output = runGit(gitRoot, 'cat-file', ['cat-file', '--batch'], { input: entries.map(entry => `${entry.oid}\n`).join(''), hardened })
|
|
183
|
+
const blobs = []
|
|
184
|
+
let offset = 0
|
|
185
|
+
for (const entry of entries) {
|
|
186
|
+
const newline = output.indexOf(0x0a, offset)
|
|
187
|
+
if (newline === -1) throw new Error('git cat-file returned a truncated record')
|
|
188
|
+
const [oid, type, size] = output.toString('utf8', offset, newline).split(' ')
|
|
189
|
+
if (oid !== entry.oid || type !== 'blob' || !/^\d+$/.test(size ?? '')) {
|
|
190
|
+
throw new Error(`git cat-file could not read blob ${entry.oid}`)
|
|
191
|
+
}
|
|
192
|
+
const start = newline + 1
|
|
193
|
+
const end = start + Number(size)
|
|
194
|
+
if (end + 1 > output.length) throw new Error('git cat-file returned a truncated record')
|
|
195
|
+
blobs.push(output.subarray(start, end))
|
|
196
|
+
offset = end + 1
|
|
197
|
+
}
|
|
198
|
+
return blobs
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
const baseTreeReader = (gitRoot, baseCommit, { hardened = false } = {}) => {
|
|
202
|
+
const lsTree = (pathspecs, { recursive = false } = {}) => parseTree(runGit(gitRoot, 'ls-tree', [
|
|
203
|
+
'--literal-pathspecs', 'ls-tree', ...(recursive ? ['-r'] : []), '-z', '-l', '--full-tree', baseCommit,
|
|
204
|
+
...(pathspecs.length ? ['--', ...pathspecs] : []),
|
|
205
|
+
], { hardened }))
|
|
206
|
+
const entries = new Map()
|
|
207
|
+
const lookup = relative => {
|
|
208
|
+
if (!entries.has(relative)) entries.set(relative, lsTree([relative]).get(relative) ?? null)
|
|
209
|
+
return entries.get(relative)
|
|
210
|
+
}
|
|
211
|
+
const blobs = new Map()
|
|
212
|
+
const read = entry => {
|
|
213
|
+
if (!blobs.has(entry.oid)) blobs.set(entry.oid, readBlobs(gitRoot, [entry], { hardened })[0])
|
|
214
|
+
return blobs.get(entry.oid)
|
|
215
|
+
}
|
|
216
|
+
const children = relative => [...lsTree(relative === '' ? [] : [`${relative}/`])].map(([child, entry]) => ({
|
|
217
|
+
name: relative === '' ? child : child.slice(relative.length + 1),
|
|
218
|
+
entry,
|
|
219
|
+
}))
|
|
220
|
+
const followTarget = (parent, target, depth) => {
|
|
221
|
+
const parts = target.split('/')
|
|
222
|
+
let current = parent
|
|
223
|
+
let unresolved = false
|
|
224
|
+
for (const [index, part] of parts.entries()) {
|
|
225
|
+
if (part === '' || part === '.') continue
|
|
226
|
+
if (part !== '..') {
|
|
227
|
+
current = current ? `${current}/${part}` : part
|
|
228
|
+
unresolved = true
|
|
229
|
+
continue
|
|
230
|
+
}
|
|
231
|
+
if (unresolved) {
|
|
232
|
+
const directory = resolve(current, depth + 1)
|
|
233
|
+
if (directory.state !== 'tree') {
|
|
234
|
+
return { result: directory.state === 'absent' ? { state: 'absent' } : { state: 'unavailable', message: `symbolic link target ${target} does not resolve to a directory` } }
|
|
235
|
+
}
|
|
236
|
+
current = directory.relative
|
|
237
|
+
unresolved = false
|
|
238
|
+
}
|
|
239
|
+
if (current === '') return { escaped: path.join(path.dirname(gitRoot), ...parts.slice(index + 1)) }
|
|
240
|
+
current = path.posix.dirname(current) === '.' ? '' : path.posix.dirname(current)
|
|
241
|
+
}
|
|
242
|
+
return { relative: current }
|
|
243
|
+
}
|
|
244
|
+
const resolve = (relative, depth = 0) => {
|
|
245
|
+
if (depth > MAX_SYMLINK_DEPTH) return { state: 'unavailable', message: 'too many levels of symbolic links' }
|
|
246
|
+
const parts = relative === '' ? [] : relative.split('/')
|
|
247
|
+
for (let index = 1; index <= parts.length; index++) {
|
|
248
|
+
const prefix = parts.slice(0, index).join('/')
|
|
249
|
+
const entry = lookup(prefix)
|
|
250
|
+
if (!entry) return { state: 'absent' }
|
|
251
|
+
const rest = parts.slice(index)
|
|
252
|
+
if (entry.mode === '160000') return { state: 'submodule', prefix, commit: entry.oid, rest: rest.join('/') }
|
|
253
|
+
if (entry.mode === '120000') {
|
|
254
|
+
const target = read(entry).toString('utf8')
|
|
255
|
+
const parent = path.posix.dirname(prefix) === '.' ? '' : path.posix.dirname(prefix)
|
|
256
|
+
if (path.posix.isAbsolute(target) && target.split('/').includes('..')) {
|
|
257
|
+
return { state: 'unavailable', message: `absolute symbolic link target ${target} contains ..` }
|
|
258
|
+
}
|
|
259
|
+
const followed = path.posix.isAbsolute(target) ? { escaped: path.resolve(target) } : followTarget(parent, target, depth)
|
|
260
|
+
if (followed.result) return followed.result
|
|
261
|
+
if (followed.relative !== undefined) return resolve([followed.relative, ...rest].filter(Boolean).join('/'), depth + 1)
|
|
262
|
+
const absolute = followed.escaped
|
|
263
|
+
const inside = repoRelative(gitRoot, absolute) ?? lexicalRepoPath(gitRoot, absolute)
|
|
264
|
+
if (inside === null) return { state: 'outside', absolute: path.join(absolute, ...rest) }
|
|
265
|
+
return resolve([inside, ...rest].filter(Boolean).join('/'), depth + 1)
|
|
266
|
+
}
|
|
267
|
+
if (rest.length && entry.type !== 'tree') return { state: 'absent' }
|
|
268
|
+
if (!rest.length && entry.type !== 'tree') {
|
|
269
|
+
return regularBlob(entry) ? { state: 'blob', entry } : { state: 'unavailable', message: 'not a regular file' }
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
return { state: 'tree', relative }
|
|
273
|
+
}
|
|
274
|
+
return { root: gitRoot, hardened, lsTree, read, children, resolve }
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
const submoduleReader = (reader, prefix, commit) => {
|
|
278
|
+
const unavailable = { state: 'unavailable', message: `in submodule ${prefix}, which is not checked out with base commit ${commit}` }
|
|
279
|
+
const candidate = path.join(reader.root, ...prefix.split('/'))
|
|
280
|
+
let root
|
|
281
|
+
try {
|
|
282
|
+
root = fs.realpathSync(candidate)
|
|
283
|
+
// A PR can commit a directory laid out as a git dir, but never a .git entry or a gitlink's checkout.
|
|
284
|
+
if (root !== candidate || !fs.lstatSync(root).isDirectory() || !fs.existsSync(path.join(root, '.git'))) return unavailable
|
|
285
|
+
const index = runGit(reader.root, 'ls-files', ['--literal-pathspecs', 'ls-files', '-s', '-z', '--', prefix], { hardened: reader.hardened })
|
|
286
|
+
if (!nulRecords(index).some(record => record.startsWith('160000 ') && record.endsWith(`\t${prefix}`))) return unavailable
|
|
287
|
+
const toplevel = runGit(root, 'rev-parse', ['rev-parse', '--show-toplevel'], { hardened: true }).toString('utf8').trim()
|
|
288
|
+
if (fs.realpathSync(toplevel) !== root) return unavailable
|
|
289
|
+
} catch {
|
|
290
|
+
return unavailable
|
|
291
|
+
}
|
|
292
|
+
const present = spawnSync('git', [...SUBMODULE_GIT_CONFIG, 'cat-file', '-e', `${commit}^{commit}`], {
|
|
293
|
+
cwd: root,
|
|
294
|
+
env: { ...process.env, GIT_NO_LAZY_FETCH: '1' },
|
|
295
|
+
stdio: 'ignore',
|
|
296
|
+
})
|
|
297
|
+
return present.status === 0 ? baseTreeReader(root, commit, { hardened: true }) : unavailable
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
const resolveBase = (reader, relative, depth = 0) => {
|
|
301
|
+
const resolved = reader.resolve(relative)
|
|
302
|
+
if (resolved.state !== 'submodule') return { ...resolved, reader }
|
|
303
|
+
if (depth > MAX_SYMLINK_DEPTH) return { state: 'unavailable', message: 'too many nested submodules' }
|
|
304
|
+
const submodule = submoduleReader(reader, resolved.prefix, resolved.commit)
|
|
305
|
+
return submodule.state === 'unavailable' ? submodule : resolveBase(submodule, resolved.rest, depth + 1)
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
const sameBytes = (left, right) => left === null || right === null ? left === right : left.equals(right)
|
|
309
|
+
|
|
310
|
+
const configInput = ({ gitRoot, tree, configFile }) => {
|
|
311
|
+
const location = repoLocation(gitRoot, configFile ?? path.join(gitRoot, '.model-eol.json'))
|
|
312
|
+
if (location === null || location === '') return { changed: false }
|
|
313
|
+
const resolved = resolveBase(tree, location)
|
|
314
|
+
if (resolved.state === 'absent' && isIgnored(gitRoot, location)) return { changed: false }
|
|
315
|
+
const head = configFile ? readHead(configFile) : null
|
|
316
|
+
const base = resolved.state === 'blob' ? { bytes: resolved.reader.read(resolved.entry) }
|
|
317
|
+
: resolved.state === 'absent' ? { bytes: null }
|
|
318
|
+
: resolved.state === 'outside' ? readRegular(resolved.absolute)
|
|
319
|
+
: { message: resolved.message ?? 'a directory' }
|
|
320
|
+
if (base.message) return { changed: true, location, unavailable: `base config ${location} is ${base.message}` }
|
|
321
|
+
return { changed: !sameBytes(head, base.bytes), location, bytes: base.bytes }
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
const readFeedDirectory = directory => {
|
|
325
|
+
const files = new Map()
|
|
326
|
+
for (const name of fs.readdirSync(directory).filter(name => name.endsWith('.json'))) {
|
|
327
|
+
const result = readRegular(path.join(directory, name))
|
|
328
|
+
if (result.message || result.bytes === null) return { message: `entry ${name} is ${result.message ?? 'missing'}` }
|
|
329
|
+
files.set(name, result.bytes)
|
|
330
|
+
}
|
|
331
|
+
return { files }
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
const feedsInput = ({ gitRoot, tree, feedsDir }) => {
|
|
335
|
+
const location = repoLocation(gitRoot, feedsDir)
|
|
336
|
+
if (location === null) return { changed: false }
|
|
337
|
+
const directory = resolveBase(tree, location)
|
|
338
|
+
if (directory.state === 'absent' && isIgnored(gitRoot, location)) return { changed: false }
|
|
339
|
+
const label = location || '.'
|
|
340
|
+
const headFiles = readFeedDirectory(feedsDir)
|
|
341
|
+
if (headFiles.message) throw new Error(`could not read feeds ${feedsDir}: ${headFiles.message}`)
|
|
342
|
+
const head = headFiles.files
|
|
343
|
+
const unavailable = message => ({ changed: true, location, unavailable: `base feeds ${message}` })
|
|
344
|
+
let base
|
|
345
|
+
if (directory.state === 'tree') {
|
|
346
|
+
base = new Map()
|
|
347
|
+
const { reader } = directory
|
|
348
|
+
for (const { name, entry } of reader.children(directory.relative)) {
|
|
349
|
+
if (!name.endsWith('.json')) continue
|
|
350
|
+
const resolved = regularBlob(entry) ? { state: 'blob', entry, reader }
|
|
351
|
+
: entry.mode === '120000' ? resolveBase(reader, directory.relative ? `${directory.relative}/${name}` : name)
|
|
352
|
+
: null
|
|
353
|
+
const outside = resolved?.state === 'outside' ? readRegular(resolved.absolute) : null
|
|
354
|
+
if (resolved?.state === 'blob') base.set(name, resolved.reader.read(resolved.entry))
|
|
355
|
+
else if (outside?.bytes) base.set(name, outside.bytes)
|
|
356
|
+
else return unavailable(`entry ${label}/${name} is not a readable regular file at base`)
|
|
357
|
+
}
|
|
358
|
+
} else if (directory.state === 'outside') {
|
|
359
|
+
let listed
|
|
360
|
+
try {
|
|
361
|
+
listed = readFeedDirectory(directory.absolute)
|
|
362
|
+
} catch (error) {
|
|
363
|
+
return unavailable(`directory ${label} could not be read at base: ${error.message}`)
|
|
364
|
+
}
|
|
365
|
+
if (listed.message) return unavailable(`directory ${label} at base: ${listed.message}`)
|
|
366
|
+
base = listed.files
|
|
367
|
+
} else if (directory.state === 'unavailable') {
|
|
368
|
+
return unavailable(`directory ${label} is ${directory.message}`)
|
|
369
|
+
} else {
|
|
370
|
+
return unavailable(`directory ${label} is ${directory.state === 'absent' ? 'missing' : 'not a directory'} at base`)
|
|
371
|
+
}
|
|
372
|
+
const changed = head.size !== base.size || [...head].some(([name, bytes]) => !base.has(name) || !bytes.equals(base.get(name)))
|
|
373
|
+
return { changed, location, files: base }
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
const temporaryRoot = gitRoot => {
|
|
377
|
+
const tmp = fs.realpathSync(os.tmpdir())
|
|
378
|
+
if (repoRelative(gitRoot, tmp) !== null) {
|
|
379
|
+
throw new Error(`the temporary directory ${tmp} is inside the git repository; set TMPDIR to a directory outside it`)
|
|
380
|
+
}
|
|
381
|
+
return fs.realpathSync(fs.mkdtempSync(path.join(tmp, 'model-eol-base-')))
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
const findingKey = (relative, line, id, matched) => JSON.stringify([relative, line, id, matched])
|
|
385
|
+
|
|
386
|
+
const baseSummary = finding => ({
|
|
387
|
+
status: finding.status,
|
|
388
|
+
via: finding.via,
|
|
389
|
+
usage: finding.usage,
|
|
390
|
+
threshold_days: finding.threshold_days,
|
|
391
|
+
effective_scope: finding.effective_scope,
|
|
392
|
+
})
|
|
393
|
+
|
|
394
|
+
// Waiver order, reason, and owner never change whether a finding is bad; route order does (the last match wins).
|
|
395
|
+
const policyKey = (config, repoPath, overrides) => {
|
|
396
|
+
const policy = configForRepoPath(config, repoPath)
|
|
397
|
+
return JSON.stringify({
|
|
398
|
+
days: overrides.days === undefined ? policy.days : null,
|
|
399
|
+
scope: overrides.scope === undefined ? policy.scope : null,
|
|
400
|
+
via: overrides.via === undefined ? policy.via : null,
|
|
401
|
+
ignore_models: [...policy.ignore.models].sort(),
|
|
402
|
+
excluded: repoPathMatches(repoPath, policy.ignore.paths),
|
|
403
|
+
waivers: policy.waivers.map(({ model, via = null, expires }) => JSON.stringify([model, via, expires])).sort(),
|
|
404
|
+
routes: routesForRepoPath(config, repoPath).map(({ via, match = null, model = null }) => [via, match, model]),
|
|
405
|
+
})
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
const feedRecords = feeds =>
|
|
409
|
+
new Map([...feeds.entries.values()].map(record => [record.entry.id, JSON.stringify([record.publisher, record.entry])]))
|
|
410
|
+
|
|
411
|
+
const changedFeedIds = (head, base) => {
|
|
412
|
+
const headRecords = feedRecords(head)
|
|
413
|
+
const baseRecords = feedRecords(base)
|
|
414
|
+
return new Set([...headRecords.keys(), ...baseRecords.keys()].filter(id => headRecords.get(id) !== baseRecords.get(id)))
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
const changedFeedKeys = (head, base) =>
|
|
418
|
+
new Set([...head.keys, ...base.keys].filter(key => head.entries.get(key)?.entry.id !== base.entries.get(key)?.entry.id))
|
|
419
|
+
|
|
420
|
+
const inputNarrowing = ({ head, base, findings }) => {
|
|
421
|
+
const ids = changedFeedIds(head.feeds, base.feeds)
|
|
422
|
+
const keys = changedFeedKeys(head.feeds, base.feeds)
|
|
423
|
+
const changedTarget = key => keys.has(key) || ids.has(head.feeds.entries.get(key)?.entry.id)
|
|
424
|
+
const byFile = new Map()
|
|
425
|
+
for (const finding of findings) {
|
|
426
|
+
const absolute = path.resolve(finding.file)
|
|
427
|
+
if (!byFile.has(absolute)) byFile.set(absolute, [])
|
|
428
|
+
byFile.get(absolute).push(finding)
|
|
429
|
+
}
|
|
430
|
+
return {
|
|
431
|
+
config: ({ absolute, baseRepoPath }) => baseRepoPath === null ||
|
|
432
|
+
policyKey(head.config, head.repoPathFor(absolute), head.overrides) !== policyKey(base.config, baseRepoPath, head.overrides),
|
|
433
|
+
feeds: ({ absolute }) => {
|
|
434
|
+
const fileFindings = byFile.get(absolute) ?? []
|
|
435
|
+
return fileFindings.some(finding => ids.has(finding.id) || keys.has(finding.matched)) ||
|
|
436
|
+
(fileFindings.length > 0 && mappedRoutesForRepoPath(head.config, head.repoPathFor(absolute)).some(route => changedTarget(route.model)))
|
|
437
|
+
},
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
const materializeInputs = ({ tempRoot, baseRef, config, feeds }) => {
|
|
442
|
+
const treeRoot = path.join(tempRoot, 'tree')
|
|
443
|
+
fs.mkdirSync(treeRoot)
|
|
444
|
+
const replacements = [[`${treeRoot}${path.sep}`, `${baseRef}:`], [treeRoot, `${baseRef}:`]]
|
|
445
|
+
let configFile
|
|
446
|
+
if (config.changed) {
|
|
447
|
+
configFile = null
|
|
448
|
+
if (config.bytes !== null) {
|
|
449
|
+
configFile = path.join(tempRoot, 'config', path.posix.basename(config.location))
|
|
450
|
+
fs.mkdirSync(path.dirname(configFile))
|
|
451
|
+
fs.writeFileSync(configFile, config.bytes, { flag: 'wx' })
|
|
452
|
+
replacements.unshift([configFile, `${baseRef}:${config.location}`])
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
let feedsDir
|
|
456
|
+
if (feeds.changed) {
|
|
457
|
+
feedsDir = path.join(tempRoot, 'feeds')
|
|
458
|
+
fs.mkdirSync(feedsDir)
|
|
459
|
+
for (const [name, bytes] of feeds.files) fs.writeFileSync(path.join(feedsDir, name), bytes, { flag: 'wx' })
|
|
460
|
+
replacements.unshift([`${feedsDir}${path.sep}`, `${baseRef}:${feeds.location ? `${feeds.location}/` : ''}`], [feedsDir, `${baseRef}:${feeds.location || '.'}`])
|
|
461
|
+
}
|
|
462
|
+
return { treeRoot, configFile, feedsDir, replacements }
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
const AFFECTED_INPUTS = ['config', 'feeds', 'rename', 'file', 'untracked']
|
|
466
|
+
|
|
467
|
+
export const selectChangedFindings = ({ targets, baseRef, findings, checkFindings, scannedFiles, configFile, feedsDir, head, loadBase, evaluateBase }) => {
|
|
468
|
+
const diff = changedLinesForTargets(targets, baseRef)
|
|
469
|
+
const { gitRoot, baseCommit } = diff
|
|
470
|
+
const tree = baseTreeReader(gitRoot, baseCommit)
|
|
471
|
+
const untracked = untrackedFiles(gitRoot, diff.relativeTargets)
|
|
472
|
+
const config = configInput({ gitRoot, tree, configFile })
|
|
473
|
+
const feeds = feedsInput({ gitRoot, tree, feedsDir })
|
|
474
|
+
let unavailable = (config.changed && config.unavailable) || (feeds.changed && feeds.unavailable) || null
|
|
475
|
+
const stats = {
|
|
476
|
+
scanned_files: scannedFiles.length,
|
|
477
|
+
affected_files: 0,
|
|
478
|
+
affected_by: Object.fromEntries(AFFECTED_INPUTS.map(input => [input, 0])),
|
|
479
|
+
candidate_findings: 0,
|
|
480
|
+
base_materialized_files: 0,
|
|
481
|
+
base_scan_ms: 0,
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
let tempRoot = null
|
|
485
|
+
let materialized = null
|
|
486
|
+
try {
|
|
487
|
+
let baseInputs
|
|
488
|
+
let narrowing = null
|
|
489
|
+
if ((config.changed || feeds.changed) && unavailable === null) {
|
|
490
|
+
try {
|
|
491
|
+
tempRoot = temporaryRoot(gitRoot)
|
|
492
|
+
materialized = materializeInputs({ tempRoot, baseRef, config, feeds })
|
|
493
|
+
baseInputs = loadBase({ configFile: materialized.configFile, feedsDir: materialized.feedsDir })
|
|
494
|
+
narrowing = inputNarrowing({ head, base: baseInputs, findings })
|
|
495
|
+
} catch {
|
|
496
|
+
// No narrowing: every file stays affected and the base step below succeeds or fails as it did before.
|
|
497
|
+
if (tempRoot !== null) fs.rmSync(tempRoot, { recursive: true, force: true })
|
|
498
|
+
tempRoot = null
|
|
499
|
+
materialized = null
|
|
500
|
+
baseInputs = undefined
|
|
501
|
+
narrowing = null
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
let listing = null
|
|
506
|
+
const baseListing = () => (listing ??= tree.lsTree(diff.relativeTargets, { recursive: true }))
|
|
507
|
+
const unaddedBad = checkFindings.filter(finding => !isAddedLine(diff.added, finding.file, finding.line) && isBad(finding))
|
|
508
|
+
const contentChecks = [...new Set(unaddedBad.map(finding => path.resolve(finding.file)))].flatMap(absolute => {
|
|
509
|
+
const record = diff.files.get(absolute)
|
|
510
|
+
const headRelative = repoRelative(gitRoot, absolute)
|
|
511
|
+
const baseRelative = record ? record.basePath : headRelative
|
|
512
|
+
return record?.hunks.length || untracked.has(absolute) || headRelative === null || baseRelative === null ? [] : [{ absolute, headRelative, baseRelative }]
|
|
513
|
+
})
|
|
514
|
+
const edited = editedWithoutHunks(gitRoot, contentChecks, baseListing)
|
|
515
|
+
|
|
516
|
+
const affected = new Map()
|
|
517
|
+
for (const file of scannedFiles) {
|
|
518
|
+
const absolute = path.resolve(file)
|
|
519
|
+
const record = diff.files.get(absolute)
|
|
520
|
+
const isUntracked = untracked.has(absolute)
|
|
521
|
+
const baseRelative = isUntracked ? null : record ? record.basePath : repoRelative(gitRoot, absolute)
|
|
522
|
+
const location = { absolute, baseRepoPath: baseRelative ?? repoRelative(gitRoot, absolute) }
|
|
523
|
+
const inputs = [
|
|
524
|
+
...(config.changed && (narrowing === null || narrowing.config(location)) ? ['config'] : []),
|
|
525
|
+
...(feeds.changed && (narrowing === null || narrowing.feeds(location)) ? ['feeds'] : []),
|
|
526
|
+
...(record?.renamed ? ['rename'] : []),
|
|
527
|
+
...(record?.hunks.length || edited.has(absolute) ? ['file'] : []),
|
|
528
|
+
...(isUntracked ? ['untracked'] : []),
|
|
529
|
+
]
|
|
530
|
+
if (!inputs.length) continue
|
|
531
|
+
for (const input of inputs) stats.affected_by[input]++
|
|
532
|
+
affected.set(absolute, { inputs, baseRelative, hunks: record?.hunks ?? [] })
|
|
533
|
+
}
|
|
534
|
+
stats.affected_files = affected.size
|
|
535
|
+
|
|
536
|
+
const occurrences = new Map()
|
|
537
|
+
const seen = new Map()
|
|
538
|
+
for (const finding of findings) {
|
|
539
|
+
const key = findingKey(path.resolve(finding.file), finding.line, finding.id, finding.matched)
|
|
540
|
+
const index = seen.get(key) ?? 0
|
|
541
|
+
seen.set(key, index + 1)
|
|
542
|
+
occurrences.set(finding, index)
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
const candidates = checkFindings.filter(finding =>
|
|
546
|
+
!isAddedLine(diff.added, finding.file, finding.line) && isBad(finding) && affected.has(path.resolve(finding.file)))
|
|
547
|
+
stats.candidate_findings = candidates.length
|
|
548
|
+
const baseRelatives = [...new Set(candidates
|
|
549
|
+
.map(finding => affected.get(path.resolve(finding.file)).baseRelative)
|
|
550
|
+
.filter(relative => relative !== null))]
|
|
551
|
+
|
|
552
|
+
const baseFindings = new Map()
|
|
553
|
+
const incomplete = new Set()
|
|
554
|
+
if (baseRelatives.length && unavailable === null) {
|
|
555
|
+
if (tempRoot === null) {
|
|
556
|
+
tempRoot = temporaryRoot(gitRoot)
|
|
557
|
+
materialized = materializeInputs({ tempRoot, baseRef, config, feeds })
|
|
558
|
+
}
|
|
559
|
+
const { treeRoot, replacements } = materialized
|
|
560
|
+
const baseEntries = baseListing()
|
|
561
|
+
const writes = []
|
|
562
|
+
for (const relative of baseRelatives) {
|
|
563
|
+
const entry = baseEntries.get(relative)
|
|
564
|
+
if (!entry || !scannableTreePath(relative)) continue
|
|
565
|
+
if (!regularBlob(entry) || entry.size > MAX_FILE_BYTES) {
|
|
566
|
+
incomplete.add(relative)
|
|
567
|
+
continue
|
|
568
|
+
}
|
|
569
|
+
writes.push({ relative, entry, target: path.join(treeRoot, ...relative.split('/')) })
|
|
570
|
+
}
|
|
571
|
+
const blobs = writes.length ? readBlobs(gitRoot, writes.map(write => write.entry)) : []
|
|
572
|
+
const files = []
|
|
573
|
+
for (const [index, write] of writes.entries()) {
|
|
574
|
+
try {
|
|
575
|
+
fs.mkdirSync(path.dirname(write.target), { recursive: true })
|
|
576
|
+
fs.writeFileSync(write.target, blobs[index], { flag: 'wx' })
|
|
577
|
+
files.push(write.target)
|
|
578
|
+
} catch {
|
|
579
|
+
incomplete.add(write.relative)
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
stats.base_materialized_files = files.length
|
|
583
|
+
if (files.length) {
|
|
584
|
+
const relativeToTree = file => path.relative(treeRoot, path.resolve(file)).split(path.sep).join('/')
|
|
585
|
+
const started = performance.now()
|
|
586
|
+
try {
|
|
587
|
+
const base = evaluateBase({
|
|
588
|
+
treeRoot,
|
|
589
|
+
files,
|
|
590
|
+
configFile: materialized.configFile,
|
|
591
|
+
feedsDir: materialized.feedsDir,
|
|
592
|
+
inputs: baseInputs,
|
|
593
|
+
classificationPathFor: file => displayPath(path.join(gitRoot, ...relativeToTree(file).split('/'))),
|
|
594
|
+
})
|
|
595
|
+
for (const note of incompleteScanNotes(base.notes)) if (note.file) incomplete.add(relativeToTree(note.file))
|
|
596
|
+
for (const finding of base.findings) {
|
|
597
|
+
const key = findingKey(relativeToTree(finding.file), finding.line, finding.id, finding.matched)
|
|
598
|
+
if (!baseFindings.has(key)) baseFindings.set(key, [])
|
|
599
|
+
baseFindings.get(key).push(finding)
|
|
600
|
+
}
|
|
601
|
+
} catch (error) {
|
|
602
|
+
if (!(error instanceof BaseUnavailableError)) throw error
|
|
603
|
+
unavailable = replacements.reduce((message, [from, to]) => message.split(from).join(to), error.message)
|
|
604
|
+
}
|
|
605
|
+
stats.base_scan_ms = Math.round(performance.now() - started)
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
const selected = checkFindings.flatMap(finding => {
|
|
610
|
+
if (isAddedLine(diff.added, finding.file, finding.line)) return [{ ...finding, changed: { reason: 'added-line' } }]
|
|
611
|
+
const file = isBad(finding) ? affected.get(path.resolve(finding.file)) : null
|
|
612
|
+
if (!file) return []
|
|
613
|
+
const newlyActionable = base => [{ ...finding, changed: { reason: 'newly-actionable', inputs: [...file.inputs], base } }]
|
|
614
|
+
if (file.baseRelative === null) return newlyActionable(null)
|
|
615
|
+
if (unavailable !== null) return [{ ...finding, changed: { reason: 'base-unavailable', message: unavailable } }]
|
|
616
|
+
const baseLine = baseLineFor(file.hunks, finding.line)
|
|
617
|
+
if (baseLine === null || incomplete.has(file.baseRelative)) return newlyActionable(null)
|
|
618
|
+
const match = baseFindings.get(findingKey(file.baseRelative, baseLine, finding.id, finding.matched))?.[occurrences.get(finding)]
|
|
619
|
+
if (!match) return newlyActionable(null)
|
|
620
|
+
if (isBad(match) && inCheckScope(match)) return []
|
|
621
|
+
return newlyActionable(baseSummary(match))
|
|
622
|
+
})
|
|
623
|
+
return { findings: selected, stats }
|
|
624
|
+
} finally {
|
|
625
|
+
if (tempRoot !== null) fs.rmSync(tempRoot, { recursive: true, force: true })
|
|
626
|
+
}
|
|
627
|
+
}
|
package/lib/config.mjs
CHANGED
|
@@ -242,12 +242,17 @@ export const normalizeConfig = (raw, { defaults = DEFAULT_CONFIG } = {}) => {
|
|
|
242
242
|
return config
|
|
243
243
|
}
|
|
244
244
|
|
|
245
|
-
export const
|
|
245
|
+
export const repoPathMatches = (repoPath, patterns) => {
|
|
246
246
|
const normalizedPath = normalizeRepoPath(repoPath)
|
|
247
|
-
|
|
247
|
+
return matchesAnyGlob(normalizedPath, patterns) || patterns.some(pattern => {
|
|
248
248
|
const normalizedPattern = normalizeRepoPath(pattern).replace(/\/+$/, '')
|
|
249
249
|
return normalizedPattern && !/[*?]/.test(normalizedPattern) && normalizedPath.startsWith(`${normalizedPattern}/`)
|
|
250
250
|
})
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
export const configForRepoPath = (config, repoPath) => {
|
|
254
|
+
const normalizedPath = normalizeRepoPath(repoPath)
|
|
255
|
+
const matchesPath = patterns => repoPathMatches(repoPath, patterns)
|
|
251
256
|
const applicableWaivers = waivers => (waivers ?? []).filter(waiver => waiver.paths === undefined || matchesPath(waiver.paths))
|
|
252
257
|
const policy = {
|
|
253
258
|
days: config.days,
|
|
@@ -288,14 +293,14 @@ export const routeForRepoReference = (config, repoPath, { matched = null, id = n
|
|
|
288
293
|
return selected
|
|
289
294
|
}
|
|
290
295
|
|
|
291
|
-
export const
|
|
296
|
+
export const routesForRepoPath = (config, repoPath) => {
|
|
292
297
|
const normalizedPath = normalizeRepoPath(repoPath)
|
|
293
|
-
return (config.routes ?? []).flatMap((route, index) =>
|
|
294
|
-
route.match !== undefined && route.model !== undefined && matchesAnyGlob(normalizedPath, route.paths)
|
|
295
|
-
? [{ ...route, index }]
|
|
296
|
-
: [])
|
|
298
|
+
return (config.routes ?? []).flatMap((route, index) => matchesAnyGlob(normalizedPath, route.paths) ? [{ ...route, index }] : [])
|
|
297
299
|
}
|
|
298
300
|
|
|
301
|
+
export const mappedRoutesForRepoPath = (config, repoPath) =>
|
|
302
|
+
routesForRepoPath(config, repoPath).filter(route => route.match !== undefined && route.model !== undefined)
|
|
303
|
+
|
|
299
304
|
export const configuredVias = config => [
|
|
300
305
|
config.via,
|
|
301
306
|
...(config.overrides ?? []).map(override => override.via),
|