@preventive/compare 1.2.2 → 1.2.3

package/compare.d.ts

@@ -110,7 +110,17 @@ export interface LineOptions {
 // and their directories but for one -- not a package's name under
 // node_modules -- each the other's only such partner. Either gives way only
 // to a candidate that is a rename by itself and scores at least halfway from
-// the pair's score to 100%. git has no such pass, so false is git's own -M.
+// the pair's score to 100%. Last, of the files no other pass took, it pairs
+// bundle chunks whose hash changed with their content: in one directory, of
+// one extension, names alike but for what follows the last separator (`-`,
+// `.`, `_`) they share, six or more characters of a hash's (letters, digits,
+// `-` and `_`, not lowercase letters alone), as `index-CyWAfh-p.js` and
+// `index-hRvO4yU2.js` are. Minified onto a line, such a chunk scores little
+// as git splits it, so it is scored with its spans ending after `,` `;` `=`
+// `{` `}` too, and taken at git's stricter score for a basename, halfway from
+// -M's to 100%: the best pairs first, its similarity that score, and none
+// past the rename limit, as they are scored. git has no such passes, so false
+// is git's own -M.
 //
 // `renameLimit` is git's diff.renameLimit: when more deleted files times added
 // files are left, in a group, after the renames that need no scoring than its

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@preventive/compare",
-  "version": "1.2.2",
+  "version": "1.2.3",
   "description": "Git-compatible file tree diffs and minified JavaScript comparison",
   "type": "module",
   "main": "./compare.js",
@@ -48,6 +48,7 @@
     "pretty.js",
     "src/blocks.js",
     "src/changes.js",
+    "src/chunks.js",
     "src/css.js",
     "src/erasure.js",
     "src/error.js",

package/pretty.d.ts

@@ -1,7 +1,9 @@
 // `text` formatted by oxfmt as a file with `extension`, changing what is written
 // no more than layout needs (keys keep their quotes, names their escapes, a comment
 // its place after an operator; no trailing commas or arrow parentheses are added,
-// template literals stay), the copy used only where checks of oxfmt's output, apart
+// template literals stay), but for an untyped script's (`js`, `mjs`, `cjs`, `jsx`)
+// `!0` and `!1`, as a minifier writes `true` and `false`, written so, as TypeScript
+// types them otherwise; the copy used only where checks of oxfmt's output, apart
 // from oxfmt, find it changes the layout alone:
 // - a script's (`js`, `mjs`, `cjs`, `jsx`, `ts`, `mts`, `cts`, `tsx`) syntax tree as
 //   oxc parses it, its comments and its text;

package/src/chunks.js

@@ -0,0 +1,51 @@
+// Bundle chunks a build renamed: `index-CyWAfh-p.js` for `index-hRvO4yU2.js`
+// in one directory, the hash in a name following its content. Minified code
+// is a line or a few, so a chunk is scored with spans ending after punctuation.
+
+// A content hash: six or more of base64url's characters, not all lowercase
+// letters, which a word is.
+const HASH = /^[\w-]{6,}$/u
+const RANDOM = /[\dA-Z]/u
+const SEPARATORS = '-._'
+
+function nameOf(path) {
+  const slash = path.lastIndexOf('/')
+  const name = path.slice(slash + 1)
+  const dot = name.lastIndexOf('.')
+  const extension = dot > 0 ? name.slice(dot) : ''
+  const stem = name.slice(0, name.length - extension.length)
+  return { stem, key: `${path.slice(0, slash + 1)}\0${extension}\0${stem.length}` }
+}
+
+// Whether two stems of a length are one chunk's, its hash all that differs:
+// what follows the last separator they share before they differ.
+function oneChunk(a, b) {
+  let start = 0
+  while (start < a.length && a[start] === b[start]) start++
+  while (start > 0 && !SEPARATORS.includes(a[start - 1])) start--
+  const x = a.slice(start), y = b.slice(start)
+  return start > 0 && HASH.test(x) && HASH.test(y) && (RANDOM.test(x) || RANDOM.test(y))
+}
+
+// Chunks no other pass claimed, by directory, extension and name's length,
+// each pair at least `bar`: the best first, as git takes its candidates.
+export function chunkRenames({ sources, dests, pairs, used, record, punctuatedScoreOf }, { remaining, open }, bar) {
+  const names = (files, indices) => new Map(indices.filter((i) => files[i].type === 'file').map((i) => [i, nameOf(files[i].path)]))
+  const destNames = names(dests, open), sourceNames = names(sources, remaining)
+  const byKey = Map.groupBy(sourceNames.keys(), (s) => sourceNames.get(s).key)
+  const opens = Map.groupBy(destNames.keys(), (d) => destNames.get(d).key)
+  const candidates = []
+  for (const [key, ds] of opens) {
+    const ss = byKey.get(key) ?? []
+    for (const d of ds) {
+      for (const s of ss) {
+        if (!oneChunk(sourceNames.get(s).stem, destNames.get(d).stem)) continue
+        const score = punctuatedScoreOf(s, d, bar)
+        if (score >= bar) candidates.push({ s, d, score })
+      }
+    }
+  }
+  for (const { s, d, score } of candidates.sort((x, y) => y.score - x.score)) {
+    if (!pairs.has(d) && !used.has(s)) record(d, s, score)
+  }
+}

package/src/pretty.js

@@ -61,12 +61,25 @@ function chain(node, operator) {
   }
   return operands
 }
+// `!0` and `!1`, as a minifier writes `true` and `false`: the literal each reads as, which nothing binds
+// otherwise, so that either reads alike wherever it is; in a script without types alone, as TypeScript
+// types `!0` as `boolean` and `true` as `true`, and refuses `!0 as const`.
+const typed = (extension) => /^[cm]?tsx?$/u.test(extension)
+const BOOLEANS = new Map([['0', true], ['1', false]])
+const BOOLEAN_LITERALS = new Map([...BOOLEANS].map(([raw, value]) => [raw, { type: 'Literal', value }]))
+const booleanRaw = (node) => (node?.type === 'UnaryExpression' && node.operator === '!' && node.argument?.type === 'Literal'
+  && typeof node.argument.value === 'number' && BOOLEANS.has(node.argument.raw) ? node.argument.raw : null)
+const literal = (node) => BOOLEAN_LITERALS.get(booleanRaw(node)) ?? node
 const fields = (node) => Object.keys(node).filter((key) => !POSITION.has(key) && !(node.type === 'Literal' && (key === 'raw' || key === 'regex' || (key === 'value' && node.regex))))
 
-function sameTree(a, b) {
+function sameTree(a, b, booleans) {
   const work = [[a, b]]
   while (work.length > 0) {
-    const [x, y] = work.pop()
+    let [x, y] = work.pop()
+    if (booleans) {
+      x = literal(x)
+      y = literal(y)
+    }
     if (x === y) continue
     if (typeof x !== 'object' || typeof y !== 'object' || x === null || y === null || Array.isArray(x) !== Array.isArray(y)) return false
     let pairs
@@ -109,14 +122,17 @@ function edited(text, edits) {
 }
 
 // A script as the formatter may write it, read off its syntax tree: each regular expression's flags
-// sorted (`/x/yu` as `/x/uy`), and where it has `types`, a member's separator `;` where it was `,`
-// (`{ a: string, b: number }`), and no `|` or `&` before a union's or an intersection's first type
-// (`type P =\n  | A\n  | B`, `[K in\n  | A\n  | B]`). oxc reads a member with the separator ending it
-// and a union from the mark leading it, so each is taken where it is, in whatever context.
-function respelled(text, program, types) {
+// sorted (`/x/yu` as `/x/uy`), `!0` and `!1` as `true` and `false` where it reads `booleans` so, and
+// where it has `types`, a member's separator `;` where it was `,` (`{ a: string, b: number }`), and
+// no `|` or `&` before a union's or an intersection's first type (`type P =\n  | A\n  | B`,
+// `[K in\n  | A\n  | B]`). oxc reads a member with the separator ending it and a union from the mark
+// leading it, so each is taken where it is, in whatever context.
+function respelled(text, program, types, booleans) {
   const edits = []
   visit(program, (node) => {
     if (node.regex) edits.push([node.end - node.regex.flags.length, node.end, sorted(node.regex.flags)])
+    const raw = booleans ? booleanRaw(node) : null
+    if (raw !== null) edits.push([node.start, node.end, String(BOOLEANS.get(raw))])
     if (!types) return
     const members = node.type === 'TSTypeLiteral' ? node.members : node.type === 'TSInterfaceBody' ? node.body : []
     for (const member of members) if (text[member.end - 1] === ',') edits.push([member.end - 1, member.end, ';'])
@@ -127,15 +143,20 @@ function respelled(text, program, types) {
 }
 
 // A safeguard on oxfmt's own output, not a comparison of any two texts: exported
-// for tests alone, not by the package. CSS is read by `csstree` where given.
-export function sameCode(text, formatted, extension = 'js', csstree = null) {
+// for tests alone, not by the package. CSS is read by `csstree` where given; a
+// script's text by `tree`, its syntax tree as parsed, where given.
+export function sameCode(text, formatted, extension = 'js', csstree = null, tree = parsed(text, extension)) {
   if (extension === 'css') return csstree !== null && sameCss(csstree, text, formatted)
   if (extension === 'json') return sameList(jsonForm(text), jsonForm(formatted))
   if (!SCRIPTS.has(extension)) return false
-  const [before, after] = [parsed(text, extension), parsed(formatted, extension)]
+  const [before, after] = [tree, parsed(formatted, extension)]
   if (before === null || after === null) return false
-  const form = (code, { program }) => scriptForm(respelled(code, program, /^[cm]?tsx?$/u.test(extension)))
-  return form(text, before) === form(formatted, after) && sameList(comments(before), comments(after)) && sameTree(before.program, after.program)
+  // `!0` and `!1` read as `true` and `false` only where the text writes one so: format() writes them so
+  // nowhere else, so a copy of any other has none.
+  const types = typed(extension)
+  const booleans = !types && /![01]/u.test(text)
+  const form = (code, { program }) => scriptForm(respelled(code, program, types, booleans))
+  return form(text, before) === form(formatted, after) && sameList(comments(before), comments(after)) && sameTree(before.program, after.program, booleans)
 }
 
 // A name as written from where it starts, escapes and all.
@@ -173,9 +194,9 @@ function blanksFrom(text, at) {
 //   ends its line (`a &&/* c */\nb` as `a /* c */ && b`): where a comment is may matter to what reads
 //   it, a bundler a `#__PURE__` mark say, and a reader sees it where the file has it.
 // Names and comments are paired in order, as the check finds them alike or refuses the copy.
-function asWritten(text, code, extension) {
+function asWritten(text, code, extension, tree) {
   if (!SCRIPTS.has(extension) || !/\\u|\/\*/u.test(text)) return code
-  const [before, after] = [parsed(text, extension), parsed(code, extension)]
+  const [before, after] = [tree, parsed(code, extension)]
   if (before === null || after === null) return code
   const edits = []
   const [olds, news] = [names(text, before.program), names(code, after.program)]
@@ -204,6 +225,19 @@ function asWritten(text, code, extension) {
   return edited(code, edits)
 }
 
+// A script's `!0` and `!1` written as `true` and `false`, where it writes them so, as they read, from its
+// syntax tree as parsed (`tree`): set apart from a word it follows, as a minifier writes `return!0`.
+function withBooleans(text, tree) {
+  if (tree === null || !/![01]/u.test(text)) return text
+  const edits = []
+  visit(tree.program, (node) => {
+    const raw = booleanRaw(node)
+    if (raw === null || text.slice(node.start, node.end) !== `!${raw}`) return
+    edits.push([node.start, node.end, `${NAME_PART.test(text[node.start - 1] ?? '') ? ' ' : ''}${BOOLEANS.get(raw)}`])
+  })
+  return edited(text, edits)
+}
+
 // A script's block comments over more than one line, a license's say, each end their line: what oxfmt
 // leaves after one on its last line (`*/ class A {`) goes to the next, indented as the comment's first
 // line is. A comment holding a line break ends a line to the parser already (`return /*\n*/ 1` returns),
@@ -231,14 +265,22 @@ export async function format(text, extension) {
   const csstree = extension === 'css' ? await loadCssTree() : null
   if (!(SCRIPTS.has(extension) || extension === 'json' || csstree)) return { error: 'unsupported' }
   const oxfmt = await loadFormatter()
-  const result = await oxfmt(`pretty.${extension}`, text, PRETTY_OPTIONS).catch(() => null)
+  // A script's syntax tree, read once for its copy's check: its `!0` and `!1` written as `true` and
+  // `false` before oxfmt formats it, where it has no types. A check that fails in itself, its stack say, refuses the copy as
+  // one that finds it changed would.
+  let tree
+  try {
+    tree = SCRIPTS.has(extension) ? parsed(text, extension) : null
+  } catch {
+    return { error: 'pretty-mismatch' }
+  }
+  const result = await oxfmt(`pretty.${extension}`, withBooleans(text, typed(extension) ? null : tree), PRETTY_OPTIONS).catch(() => null)
   if (result === null || result.errors.length > 0) return { error: 'unformattable' }
   // oxfmt's copy as it is, or where the check refuses it, as the file writes what oxfmt respelled: a
-  // copy the check takes respelled nothing so, as its text would differ. A check that fails in itself,
-  // its stack say, refuses the copy as one that finds it changed would.
-  const checked = (code) => (sameCode(text, code, extension, csstree) ? { code } : null)
+  // copy the check takes respelled nothing so, as its text would differ.
+  const checked = (code) => (sameCode(text, code, extension, csstree, tree) ? { code } : null)
   try {
-    return checked(breakAfterComments(result.code, extension)) ?? checked(breakAfterComments(asWritten(text, result.code, extension), extension)) ?? { error: 'pretty-mismatch' }
+    return checked(breakAfterComments(result.code, extension)) ?? checked(breakAfterComments(asWritten(text, result.code, extension, tree), extension)) ?? { error: 'pretty-mismatch' }
   } catch {
     return { error: 'pretty-mismatch' }
   }

package/src/renames.js

@@ -1,9 +1,11 @@
 // git's -M pairing (diffcore_rename_extended): identical content, then a
 // basename unique among both sides' leftovers at a stricter score, then every
-// pair left scored; between the first two, ours, renames told by path.
+// pair left scored; between the first two and after the last, ours, renames
+// told by path, and bundle chunks (chunks.js).
 
 import { crc32 } from '@exodus/bytes/crc.js'
-import { MAX_SCORE, similarity, spanCounts } from './similarity.js'
+import { chunkRenames } from './chunks.js'
+import { MAX_SCORE, PUNCTUATION, similarity, spanCounts } from './similarity.js'
 import { bytesOf, sameContent } from './tree.js'
 
 // git's limits: identical files weighed per destination, scores kept per one.
@@ -175,16 +177,21 @@ const unpaired = ({ sources, dests, pairs, used }) => ({
   open: dests.keys().filter((d) => !pairs.has(d)).toArray(),
 })
 
+// git's stricter score, halfway from -M's to 100%, for a pair that names alone
+// put forward: a basename both sides have once, or a chunk's.
+const stricter = (minimum) => minimum + Math.trunc(0.5 * (MAX_SCORE - minimum))
+
 // Each added file's rename by its index in `dests`: { source, score }. A
 // `minimum` of MAX_SCORE takes identical content alone, but for `byPath`, and
-// more than `limit` squared pairs left skips the scoring.
+// more than `limit` squared pairs left skips the scoring, chunks' too.
 export function findRenames(sources, dests, { minimum, limit, byPath }) {
   const pairs = new Map(), used = new Set()
   if (sources.length === 0 || dests.length === 0) return pairs
-  // A file's bytes, its text encoded once, and their span counts once.
-  const counted = new Map(), encoded = new Map()
+  // A file's bytes, its text encoded once, and their span counts once each way.
+  const counted = new Map(), encoded = new Map(), punctuated = new Map()
   const bytes = (file) => (file.type === 'file' ? encoded.get(file) ?? encoded.set(file, bytesOf(file.data)).get(file) : null)
   const counts = (data) => counted.get(data) ?? counted.set(data, spanCounts(data)).get(data)
+  const punctuatedCounts = (data) => punctuated.get(data) ?? punctuated.set(data, spanCounts(data, PUNCTUATION)).get(data)
   const context = {
     sources, dests, pairs, used,
     sourceNames: sources.map((file) => basename(file.path)),
@@ -195,12 +202,15 @@ export function findRenames(sources, dests, { minimum, limit, byPath }) {
     },
     bytes,
     scoreOf: (s, d, least) => similarity(bytes(sources[s]), bytes(dests[d]), least, counts),
+    punctuatedScoreOf: (s, d, least) => similarity(bytes(sources[s]), bytes(dests[d]), least, punctuatedCounts, true),
   }
   exactRenames(context)
   if (byPath) pathRenames(context, minimum, limit)
   if (minimum >= MAX_SCORE) return pairs
-  basenameRenames(context, unpaired(context), minimum + Math.trunc(0.5 * (MAX_SCORE - minimum)))
+  basenameRenames(context, unpaired(context), stricter(minimum))
   const left = unpaired(context)
-  if (left.remaining.length * left.open.length <= limit * limit) scoredRenames(context, left, minimum)
+  if (left.remaining.length * left.open.length > limit * limit) return pairs
+  scoredRenames(context, left, minimum)
+  if (byPath) chunkRenames(context, unpaired(context), stricter(minimum))
   return pairs
 }

package/src/similarity.js

@@ -12,9 +12,15 @@ const looksBinary = (data) => data.subarray(0, 8000).includes(0)
 
 const spanHash = (accum1, accum2) => ((accum1 + Math.imul(accum2, 0x61)) >>> 0) % HASHBASE
 
-// Bytes per span hash, a span ending at an LF or after 64 bytes; sorted by
-// hash, as git keeps them, so two files compare in one walk down both.
-export function spanCounts(data) {
+// The bytes a span ends after: git's LF, and for code minified onto a line or
+// a few (chunks.js), punctuation too, so that an edit shifts no span past it.
+const endingAfter = (chars) => Uint8Array.from({ length: 256 }, (_, c) => Number(c === 0x0A || chars.includes(String.fromCodePoint(c))))
+export const LINES = endingAfter('')
+export const PUNCTUATION = endingAfter(',;={}')
+
+// Bytes per span hash, a span ending after a byte `ends` has or 64 bytes;
+// sorted by hash, as git keeps them, so two files compare in one walk down both.
+export function spanCounts(data, ends = LINES) {
   const text = !looksBinary(data)
   const counts = new Map()
   const add = (hash, n) => counts.set(hash, (counts.get(hash) ?? 0) + n)
@@ -25,7 +31,7 @@ export function spanCounts(data) {
     const old1 = accum1
     accum1 = (((accum1 << 7) ^ (accum2 >>> 25)) + c) >>> 0
     accum2 = ((accum2 << 7) ^ (old1 >>> 25)) >>> 0
-    if (++n < 64 && c !== 0x0A) continue
+    if (++n < 64 && ends[c] === 0) continue
     add(spanHash(accum1, accum2), n)
     n = 0
     accum1 = 0
@@ -33,21 +39,46 @@ export function spanCounts(data) {
   }
   if (n > 0) add(spanHash(accum1, accum2), n)
   const hashes = Uint32Array.from(counts.keys()).sort()
-  return { hashes, bytes: Float64Array.from(hashes, (hash) => counts.get(hash)) }
+  return { hashes, bytes: Float64Array.from(hashes, (hash) => counts.get(hash)), total: counts.values().reduce((sum, bytes) => sum + bytes, 0) }
 }
 
-// Two files' bytes, null for a link, which scores 0, as do sizes too far apart
-// for `minimum`, unread. `counts` gives span counts, cached: one file meets many.
-export function similarity(src, dst, minimum, counts) {
-  if (src === null || dst === null || dst.length === 0) return 0
-  const delta = Math.abs(src.length - dst.length), max = Math.max(src.length, dst.length)
-  if (max * (MAX_SCORE - minimum) < delta * MAX_SCORE) return 0
-  const a = counts(src), b = counts(dst)
+// Bytes of spans both have: git's walk down both.
+function copiedBytes(a, b) {
   let copied = 0
   for (let i = 0, j = 0; i < a.hashes.length && j < b.hashes.length;) {
     if (a.hashes[i] < b.hashes[j]) i++
     else if (a.hashes[i] > b.hashes[j]) j++
     else copied += Math.min(a.bytes[i++], b.bytes[j++])
   }
+  return copied
+}
+
+// The same, or 0 once the spans left could no longer bring it to `least`: a
+// score times the larger file's size, as the bytes copied times MAX_SCORE are.
+function copiedReaching(a, b, least) {
+  let copied = 0, restA = a.total, restB = b.total
+  for (let i = 0, j = 0; i < a.hashes.length && j < b.hashes.length;) {
+    if (a.hashes[i] < b.hashes[j]) restA -= a.bytes[i++]
+    else if (a.hashes[i] > b.hashes[j]) restB -= b.bytes[j++]
+    else {
+      copied += Math.min(a.bytes[i], b.bytes[j])
+      restA -= a.bytes[i++]
+      restB -= b.bytes[j++]
+    }
+    if ((copied + Math.min(restA, restB)) * MAX_SCORE < least) return 0
+  }
+  return copied
+}
+
+// Two files' bytes, null for a link, which scores 0, as do sizes too far apart
+// for `minimum`, unread. `counts` gives span counts, cached: one file meets
+// many. `early`, spans that can no longer reach `minimum` score 0 unwalked: not
+// for git's passes, whose candidates below it break ties by their scores.
+export function similarity(src, dst, minimum, counts, early = false) {
+  if (src === null || dst === null || dst.length === 0) return 0
+  const delta = Math.abs(src.length - dst.length), max = Math.max(src.length, dst.length)
+  if (max * (MAX_SCORE - minimum) < delta * MAX_SCORE) return 0
+  const a = counts(src), b = counts(dst)
+  const copied = early ? copiedReaching(a, b, minimum * max) : copiedBytes(a, b)
   return Math.trunc(copied * MAX_SCORE / max)
 }