@preventive/compare 1.2.2 → 1.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/compare.d.ts +11 -1
- package/names.d.ts +13 -12
- package/package.json +2 -1
- package/pretty.d.ts +3 -1
- package/src/chunks.js +51 -0
- package/src/names/analyze.js +12 -1
- package/src/names/rename.js +6 -2
- package/src/pretty.js +61 -19
- package/src/renames.js +17 -7
- package/src/similarity.js +43 -12
package/compare.d.ts
CHANGED
|
@@ -110,7 +110,17 @@ export interface LineOptions {
|
|
|
110
110
|
// and their directories but for one -- not a package's name under
|
|
111
111
|
// node_modules -- each the other's only such partner. Either gives way only
|
|
112
112
|
// to a candidate that is a rename by itself and scores at least halfway from
|
|
113
|
-
// the pair's score to 100%.
|
|
113
|
+
// the pair's score to 100%. Last, of the files no other pass took, it pairs
|
|
114
|
+
// bundle chunks whose hash changed with their content: in one directory, of
|
|
115
|
+
// one extension, names alike but for what follows the last separator (`-`,
|
|
116
|
+
// `.`, `_`) they share, six or more characters of a hash's (letters, digits,
|
|
117
|
+
// `-` and `_`, not lowercase letters alone), as `index-CyWAfh-p.js` and
|
|
118
|
+
// `index-hRvO4yU2.js` are. Minified onto a line, such a chunk scores little
|
|
119
|
+
// as git splits it, so it is scored with its spans ending after `,` `;` `=`
|
|
120
|
+
// `{` `}` too, and taken at git's stricter score for a basename, halfway from
|
|
121
|
+
// -M's to 100%: the best pairs first, its similarity that score, and none
|
|
122
|
+
// past the rename limit, as they are scored. git has no such passes, so false
|
|
123
|
+
// is git's own -M.
|
|
114
124
|
//
|
|
115
125
|
// `renameLimit` is git's diff.renameLimit: when more deleted files times added
|
|
116
126
|
// files are left, in a group, after the renames that need no scoring than its
|
package/names.d.ts
CHANGED
|
@@ -21,18 +21,19 @@ export type RenameReason = 'unparsed' | 'dynamic' | 'jsx' | 'block-function' | '
|
|
|
21
21
|
// The diff of two JavaScript files minified on both sides with renamed names
|
|
22
22
|
// set aside. The old side is given the new side's names wherever a minifier
|
|
23
23
|
// renamed a variable alike throughout (a declared variable's short ASCII name,
|
|
24
|
-
// not a script's top level
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
//
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
// `
|
|
32
|
-
//
|
|
33
|
-
//
|
|
34
|
-
//
|
|
35
|
-
// `
|
|
24
|
+
// not a script's top level -- but where the file's other version is a module,
|
|
25
|
+
// as a bundle's chunk is once it exports, the same file loaded alike -- a name
|
|
26
|
+
// exported or imported as itself, or a shorthand property's), checked to bind
|
|
27
|
+
// every name as before by this package's scope analysis and again by oxc's
|
|
28
|
+
// minifier; formatted again where that changes its layout alone and still binds
|
|
29
|
+
// alike; diffed as diff -U0 has it; and the blocks checked to take `before` to
|
|
30
|
+
// the new side (thrown should they not). `reason` says why no names were given;
|
|
31
|
+
// `reads` the properties either text reads names through (`name`, `toString`,
|
|
32
|
+
// `stack`, `callee`, `caller`), sorted, as code reading its names may act
|
|
33
|
+
// otherwise renamed: none where none is renamed, and `name` only where a name
|
|
34
|
+
// renamed is a function's or class's, the one name it reads (`function f`,
|
|
35
|
+
// `f = () => {}`); `marks` the plain diff's blocks that differ in renamed names
|
|
36
|
+
// alone; and `warnings` what a check that should never fail found.
|
|
36
37
|
export function renamedDiff(before: string, after: string, options?: RenameOptions & { whitespace?: Whitespace }): Promise<{
|
|
37
38
|
before: string
|
|
38
39
|
blocks: Pick<Block, 'a0' | 'a1' | 'b0' | 'b1'>[]
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@preventive/compare",
|
|
3
|
-
"version": "1.2.
|
|
3
|
+
"version": "1.2.4",
|
|
4
4
|
"description": "Git-compatible file tree diffs and minified JavaScript comparison",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./compare.js",
|
|
@@ -48,6 +48,7 @@
|
|
|
48
48
|
"pretty.js",
|
|
49
49
|
"src/blocks.js",
|
|
50
50
|
"src/changes.js",
|
|
51
|
+
"src/chunks.js",
|
|
51
52
|
"src/css.js",
|
|
52
53
|
"src/erasure.js",
|
|
53
54
|
"src/error.js",
|
package/pretty.d.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
// `text` formatted by oxfmt as a file with `extension`, changing what is written
|
|
2
2
|
// no more than layout needs (keys keep their quotes, names their escapes, a comment
|
|
3
3
|
// its place after an operator; no trailing commas or arrow parentheses are added,
|
|
4
|
-
// template literals stay),
|
|
4
|
+
// template literals stay), but for an untyped script's (`js`, `mjs`, `cjs`, `jsx`)
|
|
5
|
+
// `!0` and `!1`, as a minifier writes `true` and `false`, written so, as TypeScript
|
|
6
|
+
// types them otherwise; the copy used only where checks of oxfmt's output, apart
|
|
5
7
|
// from oxfmt, find it changes the layout alone:
|
|
6
8
|
// - a script's (`js`, `mjs`, `cjs`, `jsx`, `ts`, `mts`, `cts`, `tsx`) syntax tree as
|
|
7
9
|
// oxc parses it, its comments and its text;
|
package/src/chunks.js
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// Bundle chunks a build renamed: `index-CyWAfh-p.js` for `index-hRvO4yU2.js`
|
|
2
|
+
// in one directory, the hash in a name following its content. Minified code
|
|
3
|
+
// is a line or a few, so a chunk is scored with spans ending after punctuation.
|
|
4
|
+
|
|
5
|
+
// A content hash: six or more of base64url's characters, not all lowercase
|
|
6
|
+
// letters, which a word is.
|
|
7
|
+
const HASH = /^[\w-]{6,}$/u
|
|
8
|
+
const RANDOM = /[\dA-Z]/u
|
|
9
|
+
const SEPARATORS = '-._'
|
|
10
|
+
|
|
11
|
+
function nameOf(path) {
|
|
12
|
+
const slash = path.lastIndexOf('/')
|
|
13
|
+
const name = path.slice(slash + 1)
|
|
14
|
+
const dot = name.lastIndexOf('.')
|
|
15
|
+
const extension = dot > 0 ? name.slice(dot) : ''
|
|
16
|
+
const stem = name.slice(0, name.length - extension.length)
|
|
17
|
+
return { stem, key: `${path.slice(0, slash + 1)}\0${extension}\0${stem.length}` }
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
// Whether two stems of a length are one chunk's, its hash all that differs:
|
|
21
|
+
// what follows the last separator they share before they differ.
|
|
22
|
+
function oneChunk(a, b) {
|
|
23
|
+
let start = 0
|
|
24
|
+
while (start < a.length && a[start] === b[start]) start++
|
|
25
|
+
while (start > 0 && !SEPARATORS.includes(a[start - 1])) start--
|
|
26
|
+
const x = a.slice(start), y = b.slice(start)
|
|
27
|
+
return start > 0 && HASH.test(x) && HASH.test(y) && (RANDOM.test(x) || RANDOM.test(y))
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// Chunks no other pass claimed, by directory, extension and name's length,
|
|
31
|
+
// each pair at least `bar`: the best first, as git takes its candidates.
|
|
32
|
+
export function chunkRenames({ sources, dests, pairs, used, record, punctuatedScoreOf }, { remaining, open }, bar) {
|
|
33
|
+
const names = (files, indices) => new Map(indices.filter((i) => files[i].type === 'file').map((i) => [i, nameOf(files[i].path)]))
|
|
34
|
+
const destNames = names(dests, open), sourceNames = names(sources, remaining)
|
|
35
|
+
const byKey = Map.groupBy(sourceNames.keys(), (s) => sourceNames.get(s).key)
|
|
36
|
+
const opens = Map.groupBy(destNames.keys(), (d) => destNames.get(d).key)
|
|
37
|
+
const candidates = []
|
|
38
|
+
for (const [key, ds] of opens) {
|
|
39
|
+
const ss = byKey.get(key) ?? []
|
|
40
|
+
for (const d of ds) {
|
|
41
|
+
for (const s of ss) {
|
|
42
|
+
if (!oneChunk(sourceNames.get(s).stem, destNames.get(d).stem)) continue
|
|
43
|
+
const score = punctuatedScoreOf(s, d, bar)
|
|
44
|
+
if (score >= bar) candidates.push({ s, d, score })
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
for (const { s, d, score } of candidates.sort((x, y) => y.score - x.score)) {
|
|
49
|
+
if (!pairs.has(d) && !used.has(s)) record(d, s, score)
|
|
50
|
+
}
|
|
51
|
+
}
|
package/src/names/analyze.js
CHANGED
|
@@ -234,7 +234,7 @@ function resolved(program, s) {
|
|
|
234
234
|
if (s.naming.has(node)) variable.naming = true
|
|
235
235
|
}
|
|
236
236
|
const own = program.sourceType === 'module' || program.sourceType === 'commonjs' || globals.some(([node]) => COMMONJS.has(node.name))
|
|
237
|
-
for (const variable of s.variables) variable.aside =
|
|
237
|
+
for (const variable of s.variables) variable.aside = renamable(variable) && (own || variable.scope !== s.root)
|
|
238
238
|
// A shorthand property's two names are one, at one place.
|
|
239
239
|
const byStart = (x, y) => x[0].start - y[0].start
|
|
240
240
|
occurrences.sort(byStart)
|
|
@@ -248,6 +248,17 @@ function resolved(program, s) {
|
|
|
248
248
|
return { scopes: s.scopes, variables: s.variables, occurrences: deduplicated, globals, names, reads: s.reads, module: program.sourceType === 'module', commonjs: program.sourceType === 'commonjs', own }
|
|
249
249
|
}
|
|
250
250
|
|
|
251
|
+
// A short minified name, declared, and kept by no export, import or shorthand.
|
|
252
|
+
const renamable = (variable) => variable.name.length <= RENAMED_MAX_LENGTH && MINIFIED.test(variable.name) && !variable.kept
|
|
253
|
+
|
|
254
|
+
// A script's analysis with its top level its own, as a module's: renamable there
|
|
255
|
+
// too. Still read as a script, `this` and all, but no global of it set aside.
|
|
256
|
+
export function ownTopLevel(analysis) {
|
|
257
|
+
if (analysis.reason || analysis.own) return analysis
|
|
258
|
+
for (const variable of analysis.variables) variable.aside = renamable(variable)
|
|
259
|
+
return { ...analysis, own: true }
|
|
260
|
+
}
|
|
261
|
+
|
|
251
262
|
export function analyze(text, extension) {
|
|
252
263
|
const program = parsed(text, extension)
|
|
253
264
|
if (!program) return { reason: 'unparsed' }
|
package/src/names/rename.js
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// to read wrongly, and the result checked again by oxc (bindsAlike).
|
|
4
4
|
|
|
5
5
|
import { getOxc } from '#oxc'
|
|
6
|
-
import { analyze } from './analyze.js'
|
|
6
|
+
import { analyze, ownTopLevel } from './analyze.js'
|
|
7
7
|
import { bindsAlike } from './binds.js'
|
|
8
8
|
import { checkAnalysis, checkGiven, checkPairing, checkRenamed } from './check.js'
|
|
9
9
|
import { misread } from './misread.js'
|
|
@@ -33,8 +33,12 @@ function givenNames(a, b, { map, weak }) {
|
|
|
33
33
|
}
|
|
34
34
|
|
|
35
35
|
function renamedNames(before, after, { extension = 'js', otherExtension = extension }) {
|
|
36
|
-
|
|
36
|
+
let a = analyze(before, extension), b = analyze(after, otherExtension)
|
|
37
37
|
if (a.reason || b.reason) return { reason: a.reason ?? b.reason }
|
|
38
|
+
// A file is a module by how it is loaded, which its versions share, more than
|
|
39
|
+
// by its syntax: a bundle's chunk that one version exports from, the other not.
|
|
40
|
+
// Where its other version is a module, a script's top level is its own too.
|
|
41
|
+
if (a.module || b.module) [a, b] = [ownTopLevel(a), ownTopLevel(b)]
|
|
38
42
|
checkAnalysis(before, a)
|
|
39
43
|
checkAnalysis(after, b)
|
|
40
44
|
const paired = pairing(before, after, a, b)
|
package/src/pretty.js
CHANGED
|
@@ -61,12 +61,25 @@ function chain(node, operator) {
|
|
|
61
61
|
}
|
|
62
62
|
return operands
|
|
63
63
|
}
|
|
64
|
+
// `!0` and `!1`, as a minifier writes `true` and `false`: the literal each reads as, which nothing binds
|
|
65
|
+
// otherwise, so that either reads alike wherever it is; in a script without types alone, as TypeScript
|
|
66
|
+
// types `!0` as `boolean` and `true` as `true`, and refuses `!0 as const`.
|
|
67
|
+
const typed = (extension) => /^[cm]?tsx?$/u.test(extension)
|
|
68
|
+
const BOOLEANS = new Map([['0', true], ['1', false]])
|
|
69
|
+
const BOOLEAN_LITERALS = new Map([...BOOLEANS].map(([raw, value]) => [raw, { type: 'Literal', value }]))
|
|
70
|
+
const booleanRaw = (node) => (node?.type === 'UnaryExpression' && node.operator === '!' && node.argument?.type === 'Literal'
|
|
71
|
+
&& typeof node.argument.value === 'number' && BOOLEANS.has(node.argument.raw) ? node.argument.raw : null)
|
|
72
|
+
const literal = (node) => BOOLEAN_LITERALS.get(booleanRaw(node)) ?? node
|
|
64
73
|
const fields = (node) => Object.keys(node).filter((key) => !POSITION.has(key) && !(node.type === 'Literal' && (key === 'raw' || key === 'regex' || (key === 'value' && node.regex))))
|
|
65
74
|
|
|
66
|
-
function sameTree(a, b) {
|
|
75
|
+
function sameTree(a, b, booleans) {
|
|
67
76
|
const work = [[a, b]]
|
|
68
77
|
while (work.length > 0) {
|
|
69
|
-
|
|
78
|
+
let [x, y] = work.pop()
|
|
79
|
+
if (booleans) {
|
|
80
|
+
x = literal(x)
|
|
81
|
+
y = literal(y)
|
|
82
|
+
}
|
|
70
83
|
if (x === y) continue
|
|
71
84
|
if (typeof x !== 'object' || typeof y !== 'object' || x === null || y === null || Array.isArray(x) !== Array.isArray(y)) return false
|
|
72
85
|
let pairs
|
|
@@ -109,14 +122,17 @@ function edited(text, edits) {
|
|
|
109
122
|
}
|
|
110
123
|
|
|
111
124
|
// A script as the formatter may write it, read off its syntax tree: each regular expression's flags
|
|
112
|
-
// sorted (`/x/yu` as `/x/uy`), and
|
|
113
|
-
// (`{ a: string, b: number }`), and
|
|
114
|
-
// (`type P =\n | A\n | B`,
|
|
115
|
-
//
|
|
116
|
-
|
|
125
|
+
// sorted (`/x/yu` as `/x/uy`), `!0` and `!1` as `true` and `false` where it reads `booleans` so, and
|
|
126
|
+
// where it has `types`, a member's separator `;` where it was `,` (`{ a: string, b: number }`), and
|
|
127
|
+
// no `|` or `&` before a union's or an intersection's first type (`type P =\n | A\n | B`,
|
|
128
|
+
// `[K in\n | A\n | B]`). oxc reads a member with the separator ending it and a union from the mark
|
|
129
|
+
// leading it, so each is taken where it is, in whatever context.
|
|
130
|
+
function respelled(text, program, types, booleans) {
|
|
117
131
|
const edits = []
|
|
118
132
|
visit(program, (node) => {
|
|
119
133
|
if (node.regex) edits.push([node.end - node.regex.flags.length, node.end, sorted(node.regex.flags)])
|
|
134
|
+
const raw = booleans ? booleanRaw(node) : null
|
|
135
|
+
if (raw !== null) edits.push([node.start, node.end, String(BOOLEANS.get(raw))])
|
|
120
136
|
if (!types) return
|
|
121
137
|
const members = node.type === 'TSTypeLiteral' ? node.members : node.type === 'TSInterfaceBody' ? node.body : []
|
|
122
138
|
for (const member of members) if (text[member.end - 1] === ',') edits.push([member.end - 1, member.end, ';'])
|
|
@@ -127,15 +143,20 @@ function respelled(text, program, types) {
|
|
|
127
143
|
}
|
|
128
144
|
|
|
129
145
|
// A safeguard on oxfmt's own output, not a comparison of any two texts: exported
|
|
130
|
-
// for tests alone, not by the package. CSS is read by `csstree` where given
|
|
131
|
-
|
|
146
|
+
// for tests alone, not by the package. CSS is read by `csstree` where given; a
|
|
147
|
+
// script's text by `tree`, its syntax tree as parsed, where given.
|
|
148
|
+
export function sameCode(text, formatted, extension = 'js', csstree = null, tree = parsed(text, extension)) {
|
|
132
149
|
if (extension === 'css') return csstree !== null && sameCss(csstree, text, formatted)
|
|
133
150
|
if (extension === 'json') return sameList(jsonForm(text), jsonForm(formatted))
|
|
134
151
|
if (!SCRIPTS.has(extension)) return false
|
|
135
|
-
const [before, after] = [
|
|
152
|
+
const [before, after] = [tree, parsed(formatted, extension)]
|
|
136
153
|
if (before === null || after === null) return false
|
|
137
|
-
|
|
138
|
-
|
|
154
|
+
// `!0` and `!1` read as `true` and `false` only where the text writes one so: format() writes them so
|
|
155
|
+
// nowhere else, so a copy of any other has none.
|
|
156
|
+
const types = typed(extension)
|
|
157
|
+
const booleans = !types && /![01]/u.test(text)
|
|
158
|
+
const form = (code, { program }) => scriptForm(respelled(code, program, types, booleans))
|
|
159
|
+
return form(text, before) === form(formatted, after) && sameList(comments(before), comments(after)) && sameTree(before.program, after.program, booleans)
|
|
139
160
|
}
|
|
140
161
|
|
|
141
162
|
// A name as written from where it starts, escapes and all.
|
|
@@ -173,9 +194,9 @@ function blanksFrom(text, at) {
|
|
|
173
194
|
// ends its line (`a &&/* c */\nb` as `a /* c */ && b`): where a comment is may matter to what reads
|
|
174
195
|
// it, a bundler a `#__PURE__` mark say, and a reader sees it where the file has it.
|
|
175
196
|
// Names and comments are paired in order, as the check finds them alike or refuses the copy.
|
|
176
|
-
function asWritten(text, code, extension) {
|
|
197
|
+
function asWritten(text, code, extension, tree) {
|
|
177
198
|
if (!SCRIPTS.has(extension) || !/\\u|\/\*/u.test(text)) return code
|
|
178
|
-
const [before, after] = [
|
|
199
|
+
const [before, after] = [tree, parsed(code, extension)]
|
|
179
200
|
if (before === null || after === null) return code
|
|
180
201
|
const edits = []
|
|
181
202
|
const [olds, news] = [names(text, before.program), names(code, after.program)]
|
|
@@ -204,6 +225,19 @@ function asWritten(text, code, extension) {
|
|
|
204
225
|
return edited(code, edits)
|
|
205
226
|
}
|
|
206
227
|
|
|
228
|
+
// A script's `!0` and `!1` written as `true` and `false`, where it writes them so, as they read, from its
|
|
229
|
+
// syntax tree as parsed (`tree`): set apart from a word it follows, as a minifier writes `return!0`.
|
|
230
|
+
function withBooleans(text, tree) {
|
|
231
|
+
if (tree === null || !/![01]/u.test(text)) return text
|
|
232
|
+
const edits = []
|
|
233
|
+
visit(tree.program, (node) => {
|
|
234
|
+
const raw = booleanRaw(node)
|
|
235
|
+
if (raw === null || text.slice(node.start, node.end) !== `!${raw}`) return
|
|
236
|
+
edits.push([node.start, node.end, `${NAME_PART.test(text[node.start - 1] ?? '') ? ' ' : ''}${BOOLEANS.get(raw)}`])
|
|
237
|
+
})
|
|
238
|
+
return edited(text, edits)
|
|
239
|
+
}
|
|
240
|
+
|
|
207
241
|
// A script's block comments over more than one line, a license's say, each end their line: what oxfmt
|
|
208
242
|
// leaves after one on its last line (`*/ class A {`) goes to the next, indented as the comment's first
|
|
209
243
|
// line is. A comment holding a line break ends a line to the parser already (`return /*\n*/ 1` returns),
|
|
@@ -231,14 +265,22 @@ export async function format(text, extension) {
|
|
|
231
265
|
const csstree = extension === 'css' ? await loadCssTree() : null
|
|
232
266
|
if (!(SCRIPTS.has(extension) || extension === 'json' || csstree)) return { error: 'unsupported' }
|
|
233
267
|
const oxfmt = await loadFormatter()
|
|
234
|
-
|
|
268
|
+
// A script's syntax tree, read once for its copy's check: its `!0` and `!1` written as `true` and
|
|
269
|
+
// `false` before oxfmt formats it, where it has no types. A check that fails in itself, its stack say, refuses the copy as
|
|
270
|
+
// one that finds it changed would.
|
|
271
|
+
let tree
|
|
272
|
+
try {
|
|
273
|
+
tree = SCRIPTS.has(extension) ? parsed(text, extension) : null
|
|
274
|
+
} catch {
|
|
275
|
+
return { error: 'pretty-mismatch' }
|
|
276
|
+
}
|
|
277
|
+
const result = await oxfmt(`pretty.${extension}`, withBooleans(text, typed(extension) ? null : tree), PRETTY_OPTIONS).catch(() => null)
|
|
235
278
|
if (result === null || result.errors.length > 0) return { error: 'unformattable' }
|
|
236
279
|
// oxfmt's copy as it is, or where the check refuses it, as the file writes what oxfmt respelled: a
|
|
237
|
-
// copy the check takes respelled nothing so, as its text would differ.
|
|
238
|
-
|
|
239
|
-
const checked = (code) => (sameCode(text, code, extension, csstree) ? { code } : null)
|
|
280
|
+
// copy the check takes respelled nothing so, as its text would differ.
|
|
281
|
+
const checked = (code) => (sameCode(text, code, extension, csstree, tree) ? { code } : null)
|
|
240
282
|
try {
|
|
241
|
-
return checked(breakAfterComments(result.code, extension)) ?? checked(breakAfterComments(asWritten(text, result.code, extension), extension)) ?? { error: 'pretty-mismatch' }
|
|
283
|
+
return checked(breakAfterComments(result.code, extension)) ?? checked(breakAfterComments(asWritten(text, result.code, extension, tree), extension)) ?? { error: 'pretty-mismatch' }
|
|
242
284
|
} catch {
|
|
243
285
|
return { error: 'pretty-mismatch' }
|
|
244
286
|
}
|
package/src/renames.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
// git's -M pairing (diffcore_rename_extended): identical content, then a
|
|
2
2
|
// basename unique among both sides' leftovers at a stricter score, then every
|
|
3
|
-
// pair left scored; between the first two, ours, renames
|
|
3
|
+
// pair left scored; between the first two and after the last, ours, renames
|
|
4
|
+
// told by path, and bundle chunks (chunks.js).
|
|
4
5
|
|
|
5
6
|
import { crc32 } from '@exodus/bytes/crc.js'
|
|
6
|
-
import {
|
|
7
|
+
import { chunkRenames } from './chunks.js'
|
|
8
|
+
import { MAX_SCORE, PUNCTUATION, similarity, spanCounts } from './similarity.js'
|
|
7
9
|
import { bytesOf, sameContent } from './tree.js'
|
|
8
10
|
|
|
9
11
|
// git's limits: identical files weighed per destination, scores kept per one.
|
|
@@ -175,16 +177,21 @@ const unpaired = ({ sources, dests, pairs, used }) => ({
|
|
|
175
177
|
open: dests.keys().filter((d) => !pairs.has(d)).toArray(),
|
|
176
178
|
})
|
|
177
179
|
|
|
180
|
+
// git's stricter score, halfway from -M's to 100%, for a pair that names alone
|
|
181
|
+
// put forward: a basename both sides have once, or a chunk's.
|
|
182
|
+
const stricter = (minimum) => minimum + Math.trunc(0.5 * (MAX_SCORE - minimum))
|
|
183
|
+
|
|
178
184
|
// Each added file's rename by its index in `dests`: { source, score }. A
|
|
179
185
|
// `minimum` of MAX_SCORE takes identical content alone, but for `byPath`, and
|
|
180
|
-
// more than `limit` squared pairs left skips the scoring.
|
|
186
|
+
// more than `limit` squared pairs left skips the scoring, chunks' too.
|
|
181
187
|
export function findRenames(sources, dests, { minimum, limit, byPath }) {
|
|
182
188
|
const pairs = new Map(), used = new Set()
|
|
183
189
|
if (sources.length === 0 || dests.length === 0) return pairs
|
|
184
|
-
// A file's bytes, its text encoded once, and their span counts once.
|
|
185
|
-
const counted = new Map(), encoded = new Map()
|
|
190
|
+
// A file's bytes, its text encoded once, and their span counts once each way.
|
|
191
|
+
const counted = new Map(), encoded = new Map(), punctuated = new Map()
|
|
186
192
|
const bytes = (file) => (file.type === 'file' ? encoded.get(file) ?? encoded.set(file, bytesOf(file.data)).get(file) : null)
|
|
187
193
|
const counts = (data) => counted.get(data) ?? counted.set(data, spanCounts(data)).get(data)
|
|
194
|
+
const punctuatedCounts = (data) => punctuated.get(data) ?? punctuated.set(data, spanCounts(data, PUNCTUATION)).get(data)
|
|
188
195
|
const context = {
|
|
189
196
|
sources, dests, pairs, used,
|
|
190
197
|
sourceNames: sources.map((file) => basename(file.path)),
|
|
@@ -195,12 +202,15 @@ export function findRenames(sources, dests, { minimum, limit, byPath }) {
|
|
|
195
202
|
},
|
|
196
203
|
bytes,
|
|
197
204
|
scoreOf: (s, d, least) => similarity(bytes(sources[s]), bytes(dests[d]), least, counts),
|
|
205
|
+
punctuatedScoreOf: (s, d, least) => similarity(bytes(sources[s]), bytes(dests[d]), least, punctuatedCounts, true),
|
|
198
206
|
}
|
|
199
207
|
exactRenames(context)
|
|
200
208
|
if (byPath) pathRenames(context, minimum, limit)
|
|
201
209
|
if (minimum >= MAX_SCORE) return pairs
|
|
202
|
-
basenameRenames(context, unpaired(context),
|
|
210
|
+
basenameRenames(context, unpaired(context), stricter(minimum))
|
|
203
211
|
const left = unpaired(context)
|
|
204
|
-
if (left.remaining.length * left.open.length
|
|
212
|
+
if (left.remaining.length * left.open.length > limit * limit) return pairs
|
|
213
|
+
scoredRenames(context, left, minimum)
|
|
214
|
+
if (byPath) chunkRenames(context, unpaired(context), stricter(minimum))
|
|
205
215
|
return pairs
|
|
206
216
|
}
|
package/src/similarity.js
CHANGED
|
@@ -12,9 +12,15 @@ const looksBinary = (data) => data.subarray(0, 8000).includes(0)
|
|
|
12
12
|
|
|
13
13
|
const spanHash = (accum1, accum2) => ((accum1 + Math.imul(accum2, 0x61)) >>> 0) % HASHBASE
|
|
14
14
|
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
|
|
15
|
+
// The bytes a span ends after: git's LF, and for code minified onto a line or
|
|
16
|
+
// a few (chunks.js), punctuation too, so that an edit shifts no span past it.
|
|
17
|
+
const endingAfter = (chars) => Uint8Array.from({ length: 256 }, (_, c) => Number(c === 0x0A || chars.includes(String.fromCodePoint(c))))
|
|
18
|
+
export const LINES = endingAfter('')
|
|
19
|
+
export const PUNCTUATION = endingAfter(',;={}')
|
|
20
|
+
|
|
21
|
+
// Bytes per span hash, a span ending after a byte `ends` has or 64 bytes;
|
|
22
|
+
// sorted by hash, as git keeps them, so two files compare in one walk down both.
|
|
23
|
+
export function spanCounts(data, ends = LINES) {
|
|
18
24
|
const text = !looksBinary(data)
|
|
19
25
|
const counts = new Map()
|
|
20
26
|
const add = (hash, n) => counts.set(hash, (counts.get(hash) ?? 0) + n)
|
|
@@ -25,7 +31,7 @@ export function spanCounts(data) {
|
|
|
25
31
|
const old1 = accum1
|
|
26
32
|
accum1 = (((accum1 << 7) ^ (accum2 >>> 25)) + c) >>> 0
|
|
27
33
|
accum2 = ((accum2 << 7) ^ (old1 >>> 25)) >>> 0
|
|
28
|
-
if (++n < 64 && c
|
|
34
|
+
if (++n < 64 && ends[c] === 0) continue
|
|
29
35
|
add(spanHash(accum1, accum2), n)
|
|
30
36
|
n = 0
|
|
31
37
|
accum1 = 0
|
|
@@ -33,21 +39,46 @@ export function spanCounts(data) {
|
|
|
33
39
|
}
|
|
34
40
|
if (n > 0) add(spanHash(accum1, accum2), n)
|
|
35
41
|
const hashes = Uint32Array.from(counts.keys()).sort()
|
|
36
|
-
return { hashes, bytes: Float64Array.from(hashes, (hash) => counts.get(hash)) }
|
|
42
|
+
return { hashes, bytes: Float64Array.from(hashes, (hash) => counts.get(hash)), total: counts.values().reduce((sum, bytes) => sum + bytes, 0) }
|
|
37
43
|
}
|
|
38
44
|
|
|
39
|
-
//
|
|
40
|
-
|
|
41
|
-
export function similarity(src, dst, minimum, counts) {
|
|
42
|
-
if (src === null || dst === null || dst.length === 0) return 0
|
|
43
|
-
const delta = Math.abs(src.length - dst.length), max = Math.max(src.length, dst.length)
|
|
44
|
-
if (max * (MAX_SCORE - minimum) < delta * MAX_SCORE) return 0
|
|
45
|
-
const a = counts(src), b = counts(dst)
|
|
45
|
+
// Bytes of spans both have: git's walk down both.
|
|
46
|
+
function copiedBytes(a, b) {
|
|
46
47
|
let copied = 0
|
|
47
48
|
for (let i = 0, j = 0; i < a.hashes.length && j < b.hashes.length;) {
|
|
48
49
|
if (a.hashes[i] < b.hashes[j]) i++
|
|
49
50
|
else if (a.hashes[i] > b.hashes[j]) j++
|
|
50
51
|
else copied += Math.min(a.bytes[i++], b.bytes[j++])
|
|
51
52
|
}
|
|
53
|
+
return copied
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// The same, or 0 once the spans left could no longer bring it to `least`: a
|
|
57
|
+
// score times the larger file's size, as the bytes copied times MAX_SCORE are.
|
|
58
|
+
function copiedReaching(a, b, least) {
|
|
59
|
+
let copied = 0, restA = a.total, restB = b.total
|
|
60
|
+
for (let i = 0, j = 0; i < a.hashes.length && j < b.hashes.length;) {
|
|
61
|
+
if (a.hashes[i] < b.hashes[j]) restA -= a.bytes[i++]
|
|
62
|
+
else if (a.hashes[i] > b.hashes[j]) restB -= b.bytes[j++]
|
|
63
|
+
else {
|
|
64
|
+
copied += Math.min(a.bytes[i], b.bytes[j])
|
|
65
|
+
restA -= a.bytes[i++]
|
|
66
|
+
restB -= b.bytes[j++]
|
|
67
|
+
}
|
|
68
|
+
if ((copied + Math.min(restA, restB)) * MAX_SCORE < least) return 0
|
|
69
|
+
}
|
|
70
|
+
return copied
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// Two files' bytes, null for a link, which scores 0, as do sizes too far apart
|
|
74
|
+
// for `minimum`, unread. `counts` gives span counts, cached: one file meets
|
|
75
|
+
// many. `early`, spans that can no longer reach `minimum` score 0 unwalked: not
|
|
76
|
+
// for git's passes, whose candidates below it break ties by their scores.
|
|
77
|
+
export function similarity(src, dst, minimum, counts, early = false) {
|
|
78
|
+
if (src === null || dst === null || dst.length === 0) return 0
|
|
79
|
+
const delta = Math.abs(src.length - dst.length), max = Math.max(src.length, dst.length)
|
|
80
|
+
if (max * (MAX_SCORE - minimum) < delta * MAX_SCORE) return 0
|
|
81
|
+
const a = counts(src), b = counts(dst)
|
|
82
|
+
const copied = early ? copiedReaching(a, b, minimum * max) : copiedBytes(a, b)
|
|
52
83
|
return Math.trunc(copied * MAX_SCORE / max)
|
|
53
84
|
}
|