model-eol 0.9.1 → 0.10.1

package/lib/glob.mjs

@@ -29,7 +29,15 @@ export const globRegex = pattern => {
   return new RegExp(`^${source}$`)
 }
 
-export const matchesGlob = (file, pattern) => globRegex(pattern).test(normalizeRepoPath(file))
+const compiledGlobs = new Map()
+const compiledGlob = pattern => {
+  if (!compiledGlobs.has(pattern)) compiledGlobs.set(pattern, globRegex(pattern))
+  return compiledGlobs.get(pattern)
+}
+
+export const matchesGlob = (file, pattern) => compiledGlob(pattern).test(normalizeRepoPath(file))
 
-export const matchesAnyGlob = (file, patterns = []) =>
-  patterns.some(pattern => matchesGlob(file, pattern))
+export const matchesAnyGlob = (file, patterns = []) => {
+  const normalized = normalizeRepoPath(file)
+  return patterns.some(pattern => compiledGlob(pattern).test(normalized))
+}

package/lib/reports.mjs

@@ -1,6 +1,7 @@
 import { isBad } from './feeds.mjs'
 import { color } from './color.mjs'
 import { incompleteScanNotes } from './scanner.mjs'
+import { printable } from './verify.mjs'
 
 export const marker = status =>
   status === 'retired' ? '✗' :
@@ -196,6 +197,28 @@ export const buildAlert = schedule => {
   }
 }
 
+const CHANGED_INPUT_LABELS = new Map([
+  ['config', 'config changed'],
+  ['feeds', 'feeds changed'],
+  ['rename', 'file renamed'],
+  ['file', 'nearby lines changed'],
+  ['untracked', 'untracked file'],
+])
+
+const changedTail = finding => {
+  const changed = finding.changed
+  if (changed?.reason === 'base-unavailable') return `  [base unavailable: ${printable(changed.message)}]`
+  if (changed?.reason !== 'newly-actionable') return ''
+  const inputs = changed.inputs.map(input => CHANGED_INPUT_LABELS.get(input) ?? printable(input)).join(', ')
+  const base = changed.base
+  const outOfScope = base && (base.status === 'retired' || base.status === 'retiring') &&
+    base.effective_scope === 'direct' && base.usage !== 'direct-api' && base.usage !== 'model-reference'
+  const was = !base
+    ? 'no finding at base'
+    : `was ${base.status}${base.via && base.via !== 'publisher' ? ` via ${printable(base.via)}` : ''}${outOfScope ? ` but out of direct scope (${base.usage})` : ''} at base`
+  return `  [newly actionable: ${inputs}; ${was}]`
+}
+
 export const formatCheck = ({ findings, bad, scannedFiles, days, scope }) => {
   const out = []
   if (findings.length === 0) {
@@ -203,7 +226,7 @@ export const formatCheck = ({ findings, bad, scannedFiles, days, scope }) => {
   }
   for (const f of findings) {
     const where = `${f.file}:${f.line}`
-    out.push(humanFinding(f, `${marker(f.status)} ${where}  ${f.matched}  ${findingTail(f, days)}${f.via && f.via !== 'publisher' ? `  [via ${f.via}]` : ''}`))
+    out.push(humanFinding(f, `${marker(f.status)} ${where}  ${f.matched}  ${findingTail(f, days)}${f.via && f.via !== 'publisher' ? `  [via ${f.via}]` : ''}${changedTail(f)}`))
   }
   if (bad.length) out.push(`\n${color('bold', `model-eol: ${bad.length} finding(s) at or past the ${days}-day threshold`)}`)
   return out.join('\n')

package/lib/scanner.mjs

@@ -20,6 +20,7 @@ export const SKIP_DIRS = new Set([
   '.tox', '.terraform', 'target', 'vendor', 'coverage', 'logs', 'tmp',
 ])
 export const MAX_FILE_BYTES = 2 * 1024 * 1024
+export const GIT_MAX_BUFFER = 1024 * 1024 * 1024
 export const MAX_FILES = 100_000
 export const INCOMPLETE_SCAN_REASONS = new Set([
   'file-too-large',
@@ -153,7 +154,7 @@ const isScannable = (file, includeDocs) => {
 
 const escapeRegExp = value => value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
 
-const displayPath = absolute => {
+export const displayPath = absolute => {
   const relative = path.relative(process.cwd(), absolute)
   return relative && !relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)
     ? relative
@@ -227,14 +228,19 @@ const decodeGitQuotedPath = raw => {
   return decoded
 }
 
-export const parseDiffPath = value => {
-  const raw = value.endsWith('\r') ? value.slice(0, -1) : value
+const decodeDiffPath = value => {
+  const line = value.endsWith('\r') ? value.slice(0, -1) : value
+  // Git appends a tab to ---/+++ names that contain a space; a literal tab in a name is always quoted.
+  const raw = line.endsWith('\t') ? line.slice(0, -1) : line
   const decoded = raw.startsWith('"') ? decodeGitQuotedPath(raw) : raw
-  if (decoded === '/dev/null') return null
-  return decoded.startsWith('b/') ? decoded.slice(2) : decoded
+  return decoded === '/dev/null' ? null : decoded
 }
 
-export const addedLinesForTargets = (targets, baseRef) => {
+const stripDiffPrefix = (decoded, prefix) => decoded !== null && decoded.startsWith(prefix) ? decoded.slice(prefix.length) : decoded
+
+export const parseDiffPath = value => stripDiffPrefix(decodeDiffPath(value), 'b/')
+
+export const resolveChangedRepository = (targets, baseRef) => {
   if (!baseRef) throw new Error('--changed requires a base ref')
   const roots = new Set()
   const resolvedTargets = []
@@ -279,42 +285,93 @@ export const addedLinesForTargets = (targets, baseRef) => {
     const detail = (resolvedBase.stderr ?? '').trim()
     throw new Error(`--changed could not resolve base ref "${baseRef}" to a commit${detail ? `: ${detail}` : ''}`)
   }
+  return { gitRoot, baseCommit, relativeTargets }
+}
+
+export const changedLinesForTargets = (targets, baseRef) => {
+  const { gitRoot, baseCommit, relativeTargets } = resolveChangedRepository(targets, baseRef)
+  const { GIT_DIFF_OPTS, ...env } = process.env
   const result = spawnSync('git', [
-    '-c', 'core.quotePath=true',
-    'diff', '--no-color', '--src-prefix=a/', '--dst-prefix=b/',
+    '-c', 'core.quotePath=true', '-c', 'diff.relative=false', '-c', 'core.fsmonitor=false', '--literal-pathspecs',
+    'diff', '--no-color', '--src-prefix=a/', '--dst-prefix=b/', '--find-renames',
+    '--diff-algorithm=myers', '--indent-heuristic', '--inter-hunk-context=0',
     '--text', '--no-ext-diff', '--no-textconv', '--unified=0', baseCommit, '--', ...relativeTargets,
   ], {
     cwd: gitRoot,
     encoding: 'utf8',
+    env,
     stdio: ['ignore', 'pipe', 'pipe'],
+    maxBuffer: GIT_MAX_BUFFER,
   })
   if (result.error || result.status !== 0) {
-    const detail = (result.stderr ?? '').trim()
+    const detail = (result.stderr ?? '').trim() || result.error?.message || ''
     throw new Error(`--changed git diff failed for base ref "${baseRef}"${detail ? `: ${detail}` : ''}`)
   }
 
   const added = new Set()
+  const files = new Map()
+  let header = null
   let file = null
+  let record = null
   let inHunk = false
   let newLine = 0
   let newLinesRemaining = 0
+  const recordFor = headPath => {
+    const absolute = path.resolve(gitRoot, headPath)
+    if (!files.has(absolute)) {
+      files.set(absolute, {
+        headPath,
+        basePath: header.renameFrom ?? header.minusPath,
+        renamed: header.renameFrom !== null && header.renameFrom !== headPath,
+        hunks: [],
+      })
+    }
+    return files.get(absolute)
+  }
+  const finishHeader = () => {
+    if (header && !record && header.renameTo !== null) recordFor(header.renameTo)
+  }
   for (const line of result.stdout.split('\n')) {
     if (line.startsWith('diff --git ')) {
+      finishHeader()
+      header = { minusPath: null, renameFrom: null, renameTo: null }
       file = null
+      record = null
       inHunk = false
       newLinesRemaining = 0
       continue
     }
+    if (header && !inHunk) {
+      if (line.startsWith('rename from ')) {
+        header.renameFrom = decodeDiffPath(line.slice('rename from '.length))
+        continue
+      }
+      if (line.startsWith('rename to ')) {
+        header.renameTo = decodeDiffPath(line.slice('rename to '.length))
+        continue
+      }
+      if (line.startsWith('--- ')) {
+        header.minusPath = stripDiffPrefix(decodeDiffPath(line.slice(4)), 'a/')
+        continue
+      }
+    }
     if (!inHunk && line.startsWith('+++ ')) {
       const relative = parseDiffPath(line.slice(4))
       file = relative ? path.resolve(gitRoot, relative) : null
+      record = relative && header ? recordFor(relative) : null
       continue
     }
-    const hunk = line.match(/^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/)
+    const hunk = line.match(/^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/)
     if (hunk) {
       inHunk = true
-      newLine = Number(hunk[1])
-      newLinesRemaining = hunk[2] === undefined ? 1 : Number(hunk[2])
+      newLine = Number(hunk[3])
+      newLinesRemaining = hunk[4] === undefined ? 1 : Number(hunk[4])
+      record?.hunks.push({
+        oldStart: Number(hunk[1]),
+        oldCount: hunk[2] === undefined ? 1 : Number(hunk[2]),
+        newStart: newLine,
+        newCount: newLinesRemaining,
+      })
       continue
     }
     // Unified-diff metadata such as "\" does not
@@ -327,11 +384,26 @@ export const addedLinesForTargets = (targets, baseRef) => {
       newLinesRemaining--
     }
   }
-  return added
+  finishHeader()
+  return { gitRoot, baseCommit, relativeTargets, added, files }
 }
 
+export const addedLinesForTargets = (targets, baseRef) => changedLinesForTargets(targets, baseRef).added
+
+export const baseLineFor = (hunks, line) => {
+  let offset = 0
+  for (const hunk of hunks) {
+    const newEnd = hunk.newCount === 0 ? hunk.newStart : hunk.newStart + hunk.newCount - 1
+    if (hunk.newCount > 0 && line >= hunk.newStart && line <= newEnd) return null
+    if (newEnd < line) offset += hunk.newCount - hunk.oldCount
+  }
+  return line - offset
+}
+
+export const isAddedLine = (addedLines, file, line) => addedLines.has(changedKey(file, line))
+
 export const filterFindingsToChanged = (findings, addedLines) =>
-  findings.filter(finding => addedLines.has(changedKey(finding.file, finding.line)))
+  findings.filter(finding => isAddedLine(addedLines, finding.file, finding.line))
 
 const collectFilesDetailed = (targets, {
   includeDocs = false,
@@ -339,6 +411,7 @@ const collectFilesDetailed = (targets, {
   ignoreRoots = [],
   ignoredFiles = [],
   policyForFile = null,
+  noGit = false,
 } = {}) => {
   const files = []
   const notes = []
@@ -472,11 +545,12 @@ const collectFilesDetailed = (targets, {
   const listGitFiles = (absoluteTarget, gitRoot) => {
     const relativeTarget = path.relative(gitRoot, absoluteTarget) || '.'
     const result = spawnSync('git', [
-      'ls-files', '-z', '--cached', '--others', '--exclude-standard', '--', relativeTarget,
+      '--literal-pathspecs', 'ls-files', '-z', '--cached', '--others', '--exclude-standard', '--', relativeTarget,
     ], {
       cwd: gitRoot,
       encoding: 'utf8',
       stdio: ['ignore', 'pipe', 'pipe'],
+      maxBuffer: GIT_MAX_BUFFER,
     })
     if (result.error || result.status !== 0) {
       const detail = result.error?.message || result.stderr?.trim() || `exit ${result.status}`
@@ -488,11 +562,12 @@ const collectFilesDetailed = (targets, {
       return
     }
     const tracked = spawnSync('git', [
-      'ls-files', '-z', '--cached', '--stage', '--', relativeTarget,
+      '--literal-pathspecs', 'ls-files', '-z', '--cached', '--stage', '--', relativeTarget,
     ], {
       cwd: gitRoot,
       encoding: 'utf8',
       stdio: ['ignore', 'pipe', 'pipe'],
+      maxBuffer: GIT_MAX_BUFFER,
     })
     if (tracked.error || tracked.status !== 0) {
       const detail = tracked.error?.message || tracked.stderr?.trim() || `exit ${tracked.status}`
@@ -557,7 +632,7 @@ const collectFilesDetailed = (targets, {
     } catch {
       resolvedTarget = absoluteTarget
     }
-    const gitRoot = gitRootFor(resolvedTarget)
+    const gitRoot = noGit ? null : gitRootFor(resolvedTarget)
     if (gitRoot) listGitFiles(resolvedTarget, gitRoot)
     else walk(absoluteTarget)
   }
@@ -579,8 +654,10 @@ export const scanTargets = ({
   policyForFile = null,
   routeForReference = null,
   mappedRoutesForFile = null,
+  noGit = false,
+  classificationPathFor = null,
 }) => {
-  const collected = collectFilesDetailed(targets, { includeDocs, ignorePaths, ignoreRoots, ignoredFiles, policyForFile })
+  const collected = collectFilesDetailed(targets, { includeDocs, ignorePaths, ignoreRoots, ignoredFiles, policyForFile, noGit })
   if (collected.validTargets.length === 0) throw new Error('no valid targets remain')
   const { files } = collected
   const pattern = buildModelPattern(keys)
@@ -665,6 +742,7 @@ export const scanTargets = ({
   }
 
   for (const file of files) {
+    const classifiedFile = classificationPathFor ? classificationPathFor(file) : file
     let bytes
     try {
       bytes = fs.readFileSync(file)
@@ -732,7 +810,7 @@ export const scanTargets = ({
           const record = entries.get(m[1])
           const route = routeForReference?.({ file, matched: m[1], id: record.entry.id }) ?? null
           const classification = route ? routeClassification(route) : classifyRef({
-            file,
+            file: classifiedFile,
             line: i + 1,
             lines,
             publisher: record.publisher,
@@ -755,7 +833,7 @@ export const scanTargets = ({
         if (mappedSources.has(candidate) || isKnownModel(candidate, entries) || obviousCandidateNoise(candidate)) continue
         if (ignoresModel(file, candidate)) continue
         const classification = classifyRef({
-          file,
+          file: classifiedFile,
           line: i + 1,
           lines,
           publisher: 'unknown',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "model-eol",
-  "version": "0.9.1",
+  "version": "0.10.1",
   "description": "AI model deprecation feeds with a zero-dependency checker, inventory CLI, and migration bot.",
   "type": "module",
   "bin": {
@@ -8,7 +8,7 @@
     "model-eol-bot": "bot/bot.mjs"
   },
   "scripts": {
-    "test": "node test/run.mjs && node refresh/test/run.mjs && node bot/test/run.mjs && node scripts/test-action-contract.mjs && node scripts/test-document-validation.mjs && node scripts/test-receipts.mjs && node scripts/test-verify.mjs && node scripts/test-eval-harness.mjs && node bot/test/package.mjs && node scripts/test-feed-changelog.mjs && node scripts/test-public-site.mjs"
+    "test": "node test/run.mjs && node refresh/test/run.mjs && node bot/test/run.mjs && node scripts/test-action-contract.mjs && node scripts/test-document-validation.mjs && node scripts/test-receipts.mjs && node scripts/test-verify.mjs && node scripts/test-eval-harness.mjs && node bot/test/package.mjs && node scripts/test-feed-changelog.mjs && node scripts/test-public-site.mjs && node scripts/test-e2e-changed.mjs"
   },
   "repository": {
     "type": "git",

package/schema/model-eol.check.schema.json

@@ -49,9 +49,61 @@
         "requested_via": { "type": ["string", "null"], "minLength": 1 },
         "waiver": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/waiver" },
         "policy_provenance": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/policyProvenance" },
-        "mapped_from": { "type": "string", "minLength": 1 }
+        "mapped_from": { "type": "string", "minLength": 1 },
+        "changed": { "$ref": "#/definitions/changed" }
       },
       "additionalProperties": false
+    },
+    "changed": {
+      "description": "Present only with --changed: why the finding is reported for this change.",
+      "oneOf": [
+        {
+          "type": "object",
+          "required": ["reason"],
+          "properties": { "reason": { "const": "added-line" } },
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "required": ["reason", "inputs", "base"],
+          "properties": {
+            "reason": { "const": "newly-actionable" },
+            "inputs": {
+              "type": "array",
+              "minItems": 1,
+              "uniqueItems": true,
+              "items": { "enum": ["config", "feeds", "rename", "file", "untracked"] }
+            },
+            "base": {
+              "oneOf": [
+                { "type": "null" },
+                {
+                  "type": "object",
+                  "required": ["status", "via", "usage", "threshold_days", "effective_scope"],
+                  "properties": {
+                    "status": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/status" },
+                    "via": { "type": ["string", "null"], "minLength": 1 },
+                    "usage": { "$ref": "https://thossullivan.github.io/model-eol/schema/0.1/model-eol.inventory.schema.json#/definitions/usage" },
+                    "threshold_days": { "type": "integer", "minimum": 0 },
+                    "effective_scope": { "enum": ["all", "direct"] }
+                  },
+                  "additionalProperties": false
+                }
+              ]
+            }
+          },
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "required": ["reason", "message"],
+          "properties": {
+            "reason": { "const": "base-unavailable" },
+            "message": { "type": "string", "minLength": 1 }
+          },
+          "additionalProperties": false
+        }
+      ]
     }
   }
 }