@aburi/diff 0.2.0 → 0.3.0

package/src/similarity.ts

@@ -14,6 +14,15 @@
  * The result is de-duplicated so `InvoiceService.createInvoice` collapses to
  * `["invoice", "service", "create"]` — Jaccard similarity relies on set semantics, so
  * repeated tokens must not double-count.
+ *
+ * **The camel boundary is ASCII.** `isCamelBoundary` compares code points against `a`–`z`,
+ * `A`–`Z` and `0`–`9`, so a name written in a script with no ASCII case boundary and no
+ * separator comes back whole: `获取用户信息` and `ユーザー情報を取得する` are one token each,
+ * and so is `получитьПользователя` — its camel hump does not register. A separator still
+ * splits (`ユーザー.取得` gives two), and a mixed name splits on its ASCII half.
+ *
+ * That makes the token count a poor measure of how much such a name says, which matters
+ * wherever a count is read as a proxy for that — see §3.4.3's admissibility rule.
  */
 export function tokenizeName(input: string): string[] {
   const seen = new Set<string>()
@@ -70,50 +79,321 @@ function isCamelBoundary(prev: string, curr: string): boolean {
  * purpose of §3.4.1). Empty on one side only returns 0.0.
  */
 export function jaccard(a: readonly string[], b: readonly string[]): number {
-  if (a.length === 0 && b.length === 0) return 1
-  if (a.length === 0 || b.length === 0) return 0
-  const setA = new Set(a)
-  const setB = new Set(b)
+  return jaccardSets(new Set(a), new Set(b))
+}
+
+function jaccardSets(a: ReadonlySet<string>, b: ReadonlySet<string>): number {
+  if (a.size === 0 && b.size === 0) return 1
+  if (a.size === 0 || b.size === 0) return 0
+  const [small, large] = a.size <= b.size ? [a, b] : [b, a]
   let intersection = 0
-  for (const t of setA) if (setB.has(t)) intersection++
-  const union = setA.size + setB.size - intersection
-  return intersection / union
+  for (const token of small) if (large.has(token)) intersection++
+  return intersection / (a.size + b.size - intersection)
 }
 
-/** name-token Jaccard shortcut used by nameSimilarity and ownerSimilarity call sites. */
+/**
+ * Jaccard over the tokens of two strings. Part of the module's surface rather than an
+ * internal shortcut: the two formulas below go through a token table they share for one
+ * matching pass, so this has no caller inside this file.
+ */
 export function jaccardTokens(a: string, b: string): number {
   return jaccard(tokenizeName(a), tokenizeName(b))
 }
 
+/** How a formula gets the token set of a name. The only thing the memo below changes. */
+type TokenSetOf = (value: string) => ReadonlySet<string>
+
+const tokenizeEveryTime: TokenSetOf = (value) => new Set(tokenizeName(value))
+
 /**
  * §3.4.1 — Jaccard over the tokens of the full qualified name. The last segment is not
  * given extra weight; the entire path contributes uniformly because the intersection is
  * what the score should reward.
  */
 export function nameSimilarity(baseName: string, headName: string): number {
-  return jaccardTokens(baseName, headName)
+  return nameFormula(tokenizeEveryTime, baseName, headName)
+}
+
+/**
+ * §3.4.1 — Jaccard over the tokens of the **last segment**: the member name with its owner
+ * removed. What §3.4's composite reads, because §3.4.6 decides the owner separately and
+ * reading it on both axes charges for it twice.
+ */
+export function memberSimilarity(baseName: string, headName: string): number {
+  return memberFormula(tokenizeEveryTime, baseName, headName)
 }
 
 /**
- * §3.4.6 — owner similarity: same as nameSimilarity but restricted to everything **before**
- * the last segment. Compensates for the R-8 same-name-different-class collision, where
- * `UserRepo.getUser` should not be pair-swapped with `AdminRepo.getUser`.
+ * §3.4.6 (R-8) — whether two Symbols are close enough in *scope* to be the same Symbol: the
+ * same owner, or one whose owner was renamed. A gate rather than a score, because grading the
+ * owner cannot do what R-8 asks. `UserRepo.findById` and `AdminRepo.findById` agree on their
+ * member name and their signature, so a shared `Repo` token at weight 0.2 carries them to
+ * 0.8667 against a 0.85 threshold — while `UsersRepository.findById`, which *is* the rename,
+ * shares no owner token and scores 0.8000. The collision outscores the rename, and raising the
+ * weight only moves both: at 0.3 two three-token class names sharing two tokens land on
+ * exactly 0.85.
  *
  * Owner extraction:
  * - `Class::method` → `Class`
  * - `A.B.C.method` → `A.B.C`
  * - `topLevel`     → `` (empty)
  *
- * When both owners are empty (top-level functions), returns 1.0 because both live in the
- * same "no-owner" bucket. When one is empty and the other is not, returns 0.0 because the
- * two Symbols live in structurally different scopes.
+ * Two empty owners are compatible: top-level Symbols share the one outer scope. One empty and
+ * one not are never compatible — the two live at different depths.
+ *
+ * Otherwise the owners must correspond segment for segment, and within a segment every token
+ * on each side must find a distinct partner on the other under `sameWord`. Both sides must be
+ * covered, so `UserRepo` and `UserRepoV2` are two classes rather than one renamed — an added
+ * token is as much evidence of a sibling as of a rename, and R-8's business is refusing the
+ * collision.
+ */
+export function ownersAreCompatible(baseName: string, headName: string): boolean {
+  return ownerGate(tokenizeEveryTime, baseName, headName)
+}
+
+/**
+ * §3.4.6 — the tokens `a` and `b` name the same thing.
+ *
+ * Equal, or the same word inflected: `user`/`users`, `entity`/`entities`. Nothing else, and
+ * that is the whole of the rule rather than a first approximation.
+ *
+ * A bare prefix test is the obvious generalisation and it cannot be made to work. It admits
+ * `repo`/`report`, `cache`/`cached`, `con`/`controller` — two distinct classes, which is
+ * exactly the collision R-8 exists to refuse. Every measure that might separate those from a
+ * real rename fails, because the two populations interleave rather than sitting on opposite
+ * sides of anything:
+ *
+ * ```
+ *                              dice    levenshtein
+ *   accept  UserRepo/UsersRepository   0.571   7
+ *   reject  RepoManager/ReportManager  0.818   2
+ *   accept  Repo/Repository            0.500   6
+ *   reject  CacheStore/CachedStore     0.842   1
+ * ```
+ *
+ * The renames to accept score *lower* than the collisions to refuse, on both. There is no
+ * threshold, and a length-growth rule fares no better: `con`→`controller` grows by 7 and
+ * `user`→`users` by 1, so anything admitting the second admits the first.
+ *
+ * Inflection is not on that spectrum. It is a closed, mechanical relation between two spellings
+ * of one word, so it can be recognised rather than estimated — and it covers the case a plain
+ * equality test misses most often, a class pluralised in place.
+ *
+ * What this gives up is the abbreviation family: `UserRepo` → `UsersRepository` no longer
+ * clears the gate on `repo`/`repository`. That is a real rename reported as `added` +
+ * `removed`, and §3.4.6 records it as the price of refusing `repo`/`report`, which no rule
+ * over the two strings alone can tell apart. The evidence that would settle it is not in the
+ * strings: the owner is itself a Symbol, and whether *it* paired is the question being
+ * guessed at here.
+ */
+function sameWord(a: string, b: string): boolean {
+  if (a === b) return true
+  const [short, long] = a.length <= b.length ? [a, b] : [b, a]
+  return pluralises(short, long)
+}
+
+/**
+ * The longest owner segment the matching will search.
+ *
+ * `augment` is Kuhn's, so the search is cubic in the token count and recursive with it. A
+ * class name is a handful of words; anything past this is not a name the gate has an opinion
+ * about, and `buildDiff` takes IR JSON from a caller rather than only from `aburi scan`, so
+ * the bound is what turns an adversarial 1600-token identifier from a hang into an answer.
+ * Refusing is the safe direction: the pair falls to `added` + `removed`.
  */
-export function ownerSimilarity(baseName: string, headName: string): number {
+const MAX_OWNER_SEGMENT_TOKENS = 32
+
+/** English noun inflection, which is what a pluralised class name goes through. */
+function pluralises(singular: string, plural: string): boolean {
+  if (plural === `${singular}s` || plural === `${singular}es`) return true
+  return singular.endsWith("y") && plural === `${singular.slice(0, -1)}ies`
+}
+
+function nameFormula(setOf: TokenSetOf, baseName: string, headName: string): number {
+  return jaccardSets(setOf(baseName), setOf(headName))
+}
+
+function memberFormula(setOf: TokenSetOf, baseName: string, headName: string): number {
+  return jaccardSets(setOf(lastSegment(baseName)), setOf(lastSegment(headName)))
+}
+
+function ownerGate(setOf: TokenSetOf, baseName: string, headName: string): boolean {
   const baseOwner = extractOwner(baseName)
   const headOwner = extractOwner(headName)
-  if (baseOwner === "" && headOwner === "") return 1
-  if (baseOwner === "" || headOwner === "") return 0
-  return jaccardTokens(baseOwner, headOwner)
+  if (baseOwner === "" && headOwner === "") return true
+  if (baseOwner === "" || headOwner === "") return false
+  // The common case, and the one a bucket of methods on one class hits every time.
+  if (baseOwner === headOwner) return true
+  return segmentsCorrespond(baseOwner, headOwner, setOf)
+}
+
+/**
+ * §3.4.6 — the two owners name the same scope: segment for segment, token for token.
+ *
+ * Compared per segment rather than over the owner as a whole, because `tokenizeName` dedups
+ * and an owner is a *path*. `Users.UserRepo` collapses to `{users, user, repo}` while
+ * `Users.UsersRepository` collapses to `{users, repository}` — the namespace and the class
+ * share a word, so one side loses a token and the two stop being comparable before any
+ * spelling is looked at. Segment by segment they line up: `Users`/`Users`, then
+ * `UserRepo`/`UsersRepository`.
+ *
+ * A rename changes what a class is called, not how deeply it is nested, so a differing segment
+ * count is a differing scope.
+ */
+function segmentsCorrespond(baseOwner: string, headOwner: string, setOf: TokenSetOf): boolean {
+  const baseSegments = baseOwner.split(".")
+  const headSegments = headOwner.split(".")
+  if (baseSegments.length !== headSegments.length) return false
+  return baseSegments.every((segment, index) => {
+    const counterpart = headSegments[index]
+    if (counterpart === undefined) return false
+    if (segment === counterpart) return true
+    return coversBothWays(setOf(segment), setOf(counterpart))
+  })
+}
+
+/**
+ * Whether the two token sets admit a perfect matching under `sameWord`.
+ *
+ * Sizes must agree, after which an injection from one side is a bijection — so only one
+ * direction is searched. The search is augmenting-path rather than greedy, because a greedy
+ * pass can strand a token a different choice would have matched: over `{user, users}` and
+ * `{user, userx}`, taking the identical pair first leaves `users` with nothing, though
+ * `user`→`userx` with `users`→`user` covers both. One segment runs to a handful of tokens, so
+ * the exact answer costs nothing worth saving.
+ */
+function coversBothWays(a: ReadonlySet<string>, b: ReadonlySet<string>): boolean {
+  if (a.size !== b.size) return false
+  if (a.size > MAX_OWNER_SEGMENT_TOKENS) return false
+  const right = [...b]
+  const partnerOf = new Map<string, string>()
+  for (const token of a) {
+    if (!augment(token, right, partnerOf, new Set())) return false
+  }
+  return true
+}
+
+/** Kuhn's step: claim a free partner for `token`, or displace one that can move on. */
+function augment(
+  token: string,
+  right: readonly string[],
+  partnerOf: Map<string, string>,
+  visited: Set<string>,
+): boolean {
+  for (const candidate of right) {
+    if (visited.has(candidate) || !sameWord(token, candidate)) continue
+    visited.add(candidate)
+    const holder = partnerOf.get(candidate)
+    if (holder === undefined || augment(holder, right, partnerOf, visited)) {
+      partnerOf.set(candidate, token)
+      return true
+    }
+  }
+  return false
+}
+
+/**
+ * §3.4.6 — a necessary condition on two non-empty owners, cheaper than the full gate: their
+ * first segments must correspond, since segments are compared positionally.
+ *
+ * A filter, not a decision. It relaxes `coversBothWays`' perfect matching to "every token
+ * finds some partner", so it can only ever admit more than the gate does; answering `true`
+ * says the gate has to be asked. Loosening it further costs correctness nothing and speed a
+ * little — refusing something the gate would accept is the direction that would break recall.
+ */
+function firstSegmentsCouldAgree(baseOwner: string, headOwner: string, setOf: TokenSetOf): boolean {
+  const baseFirst = baseOwner.slice(0, dotOrEnd(baseOwner))
+  const headFirst = headOwner.slice(0, dotOrEnd(headOwner))
+  if (baseFirst === headFirst) return true
+  const baseTokens = setOf(baseFirst)
+  const headTokens = setOf(headFirst)
+  if (baseTokens.size !== headTokens.size) return false
+  // Still a necessary condition at the ceiling, because `coversBothWays` refuses a segment
+  // this wide outright — and refusing here keeps that an O(1) answer rather than an all-pairs
+  // scan run ahead of it, which is the adversarial input the ceiling exists for.
+  if (baseTokens.size > MAX_OWNER_SEGMENT_TOKENS) return false
+  for (const token of baseTokens) {
+    let partnered = false
+    for (const candidate of headTokens) {
+      if (sameWord(token, candidate)) {
+        partnered = true
+        break
+      }
+    }
+    if (!partnered) return false
+  }
+  return true
+}
+
+function dotOrEnd(owner: string): number {
+  const dot = owner.indexOf(".")
+  return dot < 0 ? owner.length : dot
+}
+
+/** The formulas §3.4 reads, over a token table shared for one matching pass. */
+export interface NameScorer {
+  name(baseName: string, headName: string): number
+  member(baseName: string, headName: string): number
+  ownersCompatible(baseName: string, headName: string): boolean
+}
+
+/**
+ * A scorer that tokenises each distinct name once.
+ *
+ * Stage 4 scores every (base, head) pair in a bucket, so a bucket of K on each side asks for
+ * K² similarities over 2K distinct names — tokenising on every call splits the same strings
+ * into the same tokens hundreds of thousands of times. The table lives for one call rather
+ * than for the process, because the names it holds are only the ones that pass is comparing.
+ */
+export function createNameScorer(): NameScorer {
+  const sets = new Map<string, ReadonlySet<string>>()
+  const setOf: TokenSetOf = (value) => {
+    const cached = sets.get(value)
+    if (cached !== undefined) return cached
+    const built: ReadonlySet<string> = new Set(tokenizeName(value))
+    sets.set(value, built)
+    return built
+  }
+  const owners = new Map<string, string>()
+  const ownerOf = (qname: string): string => {
+    const cached = owners.get(qname)
+    if (cached !== undefined) return cached
+    const built = extractOwner(qname)
+    owners.set(qname, built)
+    return built
+  }
+  /**
+   * §3.4.6's gate, with the segment split and the matching skipped where they can be.
+   *
+   * The gate is the costliest thing stage 4 asks per candidate — it splits both owners into
+   * segments, tokenises each and runs an augmenting-path matching. Two shortcuts, in order of
+   * how much they save:
+   *
+   * - **Identical owners.** A bucket of methods on one class asks about the same owner over
+   *   and over, and `ownerGate` already answers those on a string compare; doing it here skips
+   *   the two `extractOwner` calls as well, which are substrings on every candidate.
+   * - **A different first segment.** Segments correspond positionally, so owners whose first
+   *   segments cannot be the same word are incompatible whatever the rest says. The check is
+   *   one `sameWord` over two token sets the table already holds, against a full matching over
+   *   every segment.
+   *
+   * Not memoised on the pair. A bulk rename produces as many distinct owner pairs as
+   * candidates, so a table keyed on them grows with the cross-product — enough to exceed
+   * V8's Map limit on the very input this is meant to make fast.
+   */
+  const compatible = (baseName: string, headName: string): boolean => {
+    const baseOwner = ownerOf(baseName)
+    const headOwner = ownerOf(headName)
+    if (baseOwner === headOwner) return true
+    if (baseOwner === "" || headOwner === "") return false
+    if (!firstSegmentsCouldAgree(baseOwner, headOwner, setOf)) return false
+    return segmentsCorrespond(baseOwner, headOwner, setOf)
+  }
+  return {
+    name: (baseName, headName) => nameFormula(setOf, baseName, headName),
+    member: (baseName, headName) => memberFormula(setOf, baseName, headName),
+    ownersCompatible: compatible,
+  }
 }
 
 function extractOwner(qname: string): string {

package/src/slice.ts

@@ -278,6 +278,12 @@ function nodeIdOf(change: SymbolChange): SymbolId | null {
     case "removed":
       // §4.1 — removed uses base-side id because there is no head symbol.
       return change.symbol.id
+    case "unknown":
+      // The one side that exists carries the id, as for added / removed. An unknown entry
+      // is a thing a reviewer has to look at, so it belongs in the cluster its neighbours
+      // are in — a Symbol whose file went missing is most useful read beside the callers
+      // that still reference it.
+      return change.symbol.id
     case "changed":
     case "moved+changed":
     case "dropped-toggled":