stride-align 0.6.0

data/ext/stride_align/vendor/stride_align/levenshtein_prepared.hpp

@@ -0,0 +1,193 @@
+#pragma once
+
+// Persistent prepared state for Hyyrö's bit-parallel Levenshtein
+// (Myers' algorithm). Pre-builds the pattern_match (PEQ) table once
+// per query so each ``distance(target)`` call skips the per-call PEQ
+// rebuild — the dominant non-kernel cost for the streaming-against-
+// fixed-query workload.
+//
+// Today only the bytes / 1-byte unicode fast path has a prepared
+// variant; wider tokens still go through the per-call hash-map PEQ
+// build inside ``myers_distance<Token>``. Adding a wide prepared
+// state is straightforward (cache the ``unordered_map<Token,
+// vector<uint64_t>>`` instead of the flat 256*B vector) but isn't
+// required to fix the original ``1v1`` bench bug.
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <span>
+#include <vector>
+
+#include "stride_align/levenshtein.hpp"
+
+namespace stride_align::levenshtein {
+
+struct PreparedLevenshteinScoreU8 {
+  // Owning copy of the query bytes. We keep it so callers can
+  // reflect/debug the prepared state and so the kernel can verify
+  // text length math (mirrors the existing kernels' use of m).
+  std::vector<std::uint8_t> pattern;
+  // m = |pattern|. 0 means empty-query (returns |text|).
+  std::size_t m = 0;
+  // B = ceil(m / 64), the number of 64-bit blocks the PEQ spans.
+  // Always 1 in the single_word case (m <= 64); m/64 rounded up
+  // in the multi-word case.
+  std::size_t B = 0;
+  // single_word == true when m <= 64 and the single-word kernel
+  // applies. PEQ is then 256 uint64s (one per byte value) with the
+  // pattern positions bit-packed into the single word.
+  // Otherwise the PEQ is 256*B uint64s laid out [c*B + block].
+  bool single_word = false;
+  // Bit-vector PEQ. Indexed by byte value (0..255) for the single-
+  // word case, and by (byte * B + block_index) for the multi-word
+  // case.
+  std::vector<std::uint64_t> peq;
+  // Multi-word only: the top bit of the last block (m mod 64
+  // accounting). Mirrors the value computed inline in
+  // myers_multi_word_u8.
+  std::uint64_t top_bit_last = 0;
+};
+
+inline PreparedLevenshteinScoreU8 prepare_levenshtein_score_u8(
+    std::span<const std::uint8_t> pattern) {
+  PreparedLevenshteinScoreU8 prepared;
+  prepared.pattern.assign(pattern.begin(), pattern.end());
+  prepared.m = pattern.size();
+  if (prepared.m == 0) {
+    return prepared;
+  }
+  constexpr std::size_t kWord = 64U;
+  const std::uint64_t one = 1;
+  if (prepared.m <= kWord) {
+    prepared.single_word = true;
+    prepared.B = 1;
+    prepared.peq.assign(256, 0);
+    for (std::size_t i = 0; i < prepared.m; ++i) {
+      prepared.peq[pattern[i]] |= one << i;
+    }
+  } else {
+    prepared.single_word = false;
+    prepared.B = (prepared.m + kWord - 1U) / kWord;
+    prepared.peq.assign(static_cast<std::size_t>(256) * prepared.B, 0);
+    for (std::size_t i = 0; i < prepared.m; ++i) {
+      prepared.peq[static_cast<std::size_t>(pattern[i]) * prepared.B + (i / kWord)] |=
+          one << (i % kWord);
+    }
+    const std::size_t last_bits = prepared.m - (prepared.B - 1U) * kWord;
+    prepared.top_bit_last = std::uint64_t{1} << (last_bits - 1U);
+  }
+  return prepared;
+}
+
+// Single-word kernel running against a pre-built PEQ. Mirrors the
+// inner loop of ``myers_single_word_u8`` exactly; only the PEQ build
+// is hoisted out.
+inline std::size_t myers_single_word_u8_with_peq(
+    const std::uint64_t* peq,
+    std::size_t m,
+    std::span<const std::uint8_t> text,
+    std::size_t cutoff = kNoCutoff) noexcept {
+  const std::uint64_t one = 1;
+  const std::size_t n = text.size();
+  if (m == 0U) return n;
+  if (n == 0U) return m;
+  if (cutoff != kNoCutoff) {
+    const std::size_t len_diff = m > n ? m - n : n - m;
+    if (len_diff > cutoff) return cutoff + 1U;
+  }
+
+  const std::size_t shift = m - 1U;
+  std::uint64_t vp = (m == 64U) ? ~std::uint64_t{0} : ((one << m) - 1);
+  std::uint64_t vn = 0;
+  std::size_t score = m;
+
+  if (cutoff == kNoCutoff) {
+    for (const std::uint8_t c : text) {
+      const std::uint64_t eq = peq[c];
+      const std::uint64_t x = eq | vn;
+      const std::uint64_t d0 = (((x & vp) + vp) ^ vp) | x;
+      const std::uint64_t hp = vn | ~(d0 | vp);
+      const std::uint64_t hn = d0 & vp;
+      score += static_cast<std::size_t>((hp >> shift) & 1U);
+      score -= static_cast<std::size_t>((hn >> shift) & 1U);
+      const std::uint64_t hp_shift = (hp << 1) | one;
+      const std::uint64_t hn_shift = (hn << 1);
+      vp = hn_shift | ~(d0 | hp_shift);
+      vn = d0 & hp_shift;
+    }
+    return score;
+  }
+
+  std::size_t k = 0;
+  for (const std::uint8_t c : text) {
+    const std::uint64_t eq = peq[c];
+    const std::uint64_t x = eq | vn;
+    const std::uint64_t d0 = (((x & vp) + vp) ^ vp) | x;
+    const std::uint64_t hp = vn | ~(d0 | vp);
+    const std::uint64_t hn = d0 & vp;
+    score += static_cast<std::size_t>((hp >> shift) & 1U);
+    score -= static_cast<std::size_t>((hn >> shift) & 1U);
+    const std::uint64_t hp_shift = (hp << 1) | one;
+    const std::uint64_t hn_shift = (hn << 1);
+    vp = hn_shift | ~(d0 | hp_shift);
+    vn = d0 & hp_shift;
+    ++k;
+    if (score > cutoff + (n - k)) {
+      return cutoff + 1U;
+    }
+  }
+  return score;
+}
+
+inline std::size_t levenshtein_score_prepared_u8(
+    const PreparedLevenshteinScoreU8& prepared,
+    std::span<const std::uint8_t> text,
+    std::size_t cutoff = kNoCutoff) {
+  if (prepared.m == 0) {
+    return text.size();
+  }
+  if (text.empty()) {
+    return prepared.m;
+  }
+  if (cutoff != kNoCutoff) {
+    const std::size_t n = text.size();
+    const std::size_t len_diff =
+        prepared.m > n ? prepared.m - n : n - prepared.m;
+    if (len_diff > cutoff) {
+      return cutoff + 1U;
+    }
+  }
+  if (prepared.single_word) {
+    return myers_single_word_u8_with_peq(
+        prepared.peq.data(), prepared.m, text, cutoff);
+  }
+  // Multi-word: reuse thread-local vp/vn scratch (amortised across
+  // prepared 1-vs-N calls) and the existing detail::myers_inner with
+  // our cached PEQ.
+  const std::size_t B = prepared.B;
+  MultiWordLevScratch& scr = multi_word_lev_scratch();
+  scr.resize_for(B);
+  std::fill_n(scr.vp.data(), B, ~std::uint64_t{0});
+  std::fill_n(scr.vn.data(), B, std::uint64_t{0});
+  constexpr std::size_t kWord = 64U;
+  const std::size_t last_bits = prepared.m - (B - 1U) * kWord;
+  if (last_bits < kWord) {
+    scr.vp[B - 1U] = (std::uint64_t{1} << last_bits) - 1U;
+  }
+  return detail::myers_inner(
+      prepared.m,
+      B,
+      text,
+      [&prepared, B](std::uint8_t c) {
+        return std::span<const std::uint64_t>(
+            prepared.peq.data() + static_cast<std::size_t>(c) * B, B);
+      },
+      scr.vp.data(),
+      scr.vn.data(),
+      prepared.top_bit_last,
+      prepared.m,
+      cutoff);
+}
+
+}  // namespace stride_align::levenshtein

data/ext/stride_align/vendor/stride_align/match_rating.hpp

@@ -0,0 +1,168 @@
+#pragma once
+
+// Match Rating Approach (Moore, Western Airlines, 1977).
+//
+// Two operations:
+//
+//   1. ``match_rating_codex(name)`` — encode a name into a short
+//      phonetic code (typically 3-6 ASCII letters).
+//   2. ``match_rating_compare(name1, name2)`` — return ``true`` if
+//      the pair is similar enough per MRA's matching rules.
+//
+// MRA pairs an encoder and a rule-based comparator rather than
+// being a one-shot distance. Two names produce the same codex
+// (and so trivially match) when their consonant skeleton is the
+// same; the comparator also tolerates one of the two having a
+// few more characters by enforcing a length-dependent minimum
+// "rating".
+//
+// Codex rules:
+//   * Drop non-ASCII letters; upper-case the rest.
+//   * Drop vowels (AEIOU) **except** when the vowel is the first
+//     letter of the name (Y is treated as a consonant).
+//   * Collapse consecutive duplicate letters.
+//   * If the result is longer than 6 characters, keep the first 3
+//     and the last 3 (e.g. ``ROBERTO`` -> ``RBRT`` -> 4 chars,
+//     unchanged; ``CHRISTOPHER`` -> ``CHRSTPHR`` -> ``CHRPHR``).
+//
+// Comparison rules:
+//   * Compute the codex of both inputs. If they differ in length
+//     by more than 3 letters, return ``false``.
+//   * Combined length determines the minimum rating:
+//         total <= 4  -> need at least 5
+//         5 <= total <= 7  -> need at least 4
+//         8 <= total <= 11 -> need at least 3
+//         total >= 12 -> need at least 2
+//   * Strip identical characters position-wise left-to-right.
+//   * Strip identical characters position-wise right-to-left
+//     (on the reversed remainder).
+//   * Rating = 6 - (length of the longer remainder after both
+//     stripping passes).
+//   * Match iff rating >= minimum.
+//
+// Sources:
+//   * Moore, G. (Western Airlines), 1977 — original MRA paper.
+//   * Apache Commons Codec MatchRatingApproachEncoder.
+//   * https://en.wikipedia.org/wiki/Match_rating_approach
+//
+// Cross-checked against ``jellyfish.match_rating_codex`` and
+// ``jellyfish.match_rating_comparison``.
+
+#include <algorithm>
+#include <cstddef>
+#include <string>
+#include <string_view>
+
+namespace stride_align::phonetic {
+
+namespace mra_detail {
+
+inline constexpr bool is_vowel(char c) noexcept {
+  return c == 'A' || c == 'E' || c == 'I' || c == 'O' || c == 'U';
+}
+
+inline constexpr char to_upper_ascii(char c) noexcept {
+  return (c >= 'a' && c <= 'z') ? static_cast<char>(c - ('a' - 'A')) : c;
+}
+
+inline constexpr bool is_upper_alpha(char c) noexcept {
+  return c >= 'A' && c <= 'Z';
+}
+
+}  // namespace mra_detail
+
+inline std::string match_rating_codex(std::string_view input) {
+  using namespace mra_detail;
+
+  // Pre-pass: keep ASCII letters, upper-case.
+  std::string w;
+  w.reserve(input.size());
+  for (char c : input) {
+    const char uc = to_upper_ascii(c);
+    if (is_upper_alpha(uc)) w.push_back(uc);
+  }
+  if (w.empty()) return {};
+
+  // Step 1: drop vowels except the first letter.
+  std::string consonants;
+  consonants.reserve(w.size());
+  consonants.push_back(w.front());
+  for (std::size_t i = 1; i < w.size(); ++i) {
+    if (!is_vowel(w[i])) consonants.push_back(w[i]);
+  }
+
+  // Step 2: collapse consecutive duplicates.
+  std::string deduped;
+  deduped.reserve(consonants.size());
+  for (char c : consonants) {
+    if (!deduped.empty() && deduped.back() == c) continue;
+    deduped.push_back(c);
+  }
+
+  // Step 3: if longer than 6, keep first 3 + last 3.
+  if (deduped.size() > 6) {
+    deduped = deduped.substr(0, 3) + deduped.substr(deduped.size() - 3, 3);
+  }
+  return deduped;
+}
+
+namespace mra_detail {
+
+// Strip identical characters position-wise, left-to-right, from
+// the heads of ``a`` and ``b`` in place. Returns nothing — both
+// are shortened.
+inline void strip_matching_left_to_right(std::string& a, std::string& b) {
+  std::string a2, b2;
+  a2.reserve(a.size());
+  b2.reserve(b.size());
+  std::size_t i = 0;
+  const std::size_t shared = std::min(a.size(), b.size());
+  for (; i < shared; ++i) {
+    if (a[i] == b[i]) continue;
+    a2.push_back(a[i]);
+    b2.push_back(b[i]);
+  }
+  // Tail of the longer string (if any) is preserved.
+  for (; i < a.size(); ++i) a2.push_back(a[i]);
+  for (std::size_t j = shared; j < b.size(); ++j) b2.push_back(b[j]);
+  a.swap(a2);
+  b.swap(b2);
+}
+
+}  // namespace mra_detail
+
+inline bool match_rating_compare(
+    std::string_view a, std::string_view b) {
+  std::string ca = match_rating_codex(a);
+  std::string cb = match_rating_codex(b);
+
+  // Empty codex on either side: no meaningful comparison.
+  if (ca.empty() || cb.empty()) return false;
+
+  // Length-difference gate.
+  const std::size_t la = ca.size();
+  const std::size_t lb = cb.size();
+  const std::size_t diff = la > lb ? la - lb : lb - la;
+  if (diff > 3) return false;
+
+  // Length-sum -> minimum rating threshold.
+  const std::size_t total = la + lb;
+  int min_rating;
+  if (total <= 4) min_rating = 5;
+  else if (total <= 7) min_rating = 4;
+  else if (total <= 11) min_rating = 3;
+  else min_rating = 2;
+
+  // Strip identical characters position-wise left-to-right.
+  mra_detail::strip_matching_left_to_right(ca, cb);
+  // Then right-to-left: reverse, strip, reverse back.
+  std::reverse(ca.begin(), ca.end());
+  std::reverse(cb.begin(), cb.end());
+  mra_detail::strip_matching_left_to_right(ca, cb);
+  // (No need to reverse back — only sizes matter from here.)
+
+  const int rating = 6 - static_cast<int>(std::max(ca.size(), cb.size()));
+  return rating >= min_rating;
+}
+
+}  // namespace stride_align::phonetic

data/ext/stride_align/vendor/stride_align/metaphone.hpp

@@ -0,0 +1,291 @@
+#pragma once
+
+// Metaphone (Lawrence Philips, 1990).
+//
+// Transforms a word into a phonetic code representing how it sounds.
+// Unlike Soundex (which keeps the leading letter verbatim and emits
+// digits), Metaphone transforms the entire word into letters,
+// keeping the leading vowel only if the word starts with one.
+//
+// Code letters used: ``A B F H J K L M N P R S T W X Y 0`` (zero is
+// the "theta" sound from ``TH``).
+//
+// Reference implementation: this follows the **published Philips
+// 1990 spec** as faithfully reproduced in Apache Commons Codec's
+// Metaphone class. Some popular Python implementations (notably
+// jellyfish) deviate from the spec on a couple of rules — the
+// most visible being CH-after-S (spec: K, jellyfish: X) and GH
+// at end of word (spec: silent, jellyfish: keeps as "KH"). We
+// follow the spec.
+//
+// A standing caveat from Michael Kuhn's 1991 C port (and quoted
+// by the aspell maintainer): when Kuhn transcribed the algorithm
+// from the printed BASIC listing in the 1990 article, he noted
+// "there were discrepancies between the BASIC code and the verbal
+// description. The discrepancies look like they could have been
+// caused by typing errors in the article." So "the published
+// spec" is itself a moving target — the verbal description and
+// the printed code disagree in places. We track the verbal
+// description / Apache Commons Codec branch of the family,
+// which is the most widely cited.
+//
+// Sources:
+//   * Philips, L. "Hanging on the Metaphone." Computer Language
+//     Magazine 7(12), December 1990, pp. 39-44.
+//   * Apache Commons Codec Metaphone:
+//     https://commons.apache.org/proper/commons-codec/apidocs/src-html/org/apache/commons/codec/language/Metaphone.html
+//   * Lawrence Philips' Metaphone overview (with Kuhn's caveat):
+//     http://aspell.net/metaphone/
+
+#include <cstddef>
+#include <string>
+#include <string_view>
+
+namespace stride_align::phonetic {
+
+// Metaphone has two long-running flavours in the wild — the published
+// Philips 1990 spec (as reproduced by Apache Commons Codec) and the
+// jellyfish Python library's interpretation, which deviates on two
+// well-known rules (CH-after-S and GH-at-end-of-word). We expose
+// both as an enum so callers porting from jellyfish can stay
+// bit-exact without giving up the spec for everyone else.
+enum class MetaphoneVariant {
+  kPhilips = 0,    // published 1990 spec; default
+  kJellyfish = 1,  // jellyfish-compat (SCH -> SX, GH at end -> KH)
+};
+
+namespace metaphone_detail {
+
+inline constexpr bool is_vowel(char c) noexcept {
+  return c == 'A' || c == 'E' || c == 'I' || c == 'O' || c == 'U';
+}
+
+inline constexpr char to_upper_ascii(char c) noexcept {
+  return (c >= 'a' && c <= 'z') ? static_cast<char>(c - ('a' - 'A')) : c;
+}
+
+inline constexpr bool is_upper_alpha(char c) noexcept {
+  return c >= 'A' && c <= 'Z';
+}
+
+}  // namespace metaphone_detail
+
+inline std::string metaphone(std::string_view input,
+                             MetaphoneVariant variant = MetaphoneVariant::kPhilips) {
+  using namespace metaphone_detail;
+  const bool jellyfish = variant == MetaphoneVariant::kJellyfish;
+
+  // Pre-pass: keep ASCII letters only, upper-cased, with
+  // adjacent-duplicate collapse (except C — Metaphone deliberately
+  // preserves doubled C for the rule that handles "ACCE" etc.).
+  // Collapsing before the rule pass lets patterns like "TTH" fall
+  // out naturally as "TH" → "0" instead of being split into "T"
+  // and an orphan "H".
+  std::string w;
+  w.reserve(input.size());
+  for (char c : input) {
+    const char uc = to_upper_ascii(c);
+    if (!is_upper_alpha(uc)) continue;
+    if (!w.empty() && w.back() == uc && uc != 'C') continue;
+    w.push_back(uc);
+  }
+  if (w.empty()) return {};
+
+  const std::size_t n = w.size();
+  std::string out;
+  out.reserve(n);
+
+  auto get = [&](std::size_t pos) -> char {
+    return pos < n ? w[pos] : '\0';
+  };
+
+  // Initial-letter pair simplifications.
+  std::size_t i = 0;
+  if (n >= 2) {
+    const char a = w[0], b = w[1];
+    if ((a == 'A' && b == 'E') ||
+        (a == 'G' && b == 'N') ||
+        (a == 'K' && b == 'N') ||
+        (a == 'P' && b == 'N') ||
+        (a == 'W' && b == 'R')) {
+      i = 1;  // drop first letter
+    } else if (a == 'W' && b == 'H') {
+      out.push_back('W');
+      i = 2;
+    } else if (a == 'X') {
+      out.push_back('S');
+      i = 1;
+    }
+  }
+
+  for (; i < n; ++i) {
+    const char c = w[i];
+    const char prev = i > 0 ? w[i - 1] : '\0';
+    const char nx = get(i + 1);
+    const char nx2 = get(i + 2);
+
+    switch (c) {
+      case 'A': case 'E': case 'I': case 'O': case 'U':
+        if (out.empty()) out.push_back(c);
+        break;
+      case 'B':
+        // Drop B at end of word after M (e.g. "lamb").
+        if (i + 1 == n && prev == 'M') break;
+        out.push_back('B');
+        break;
+      case 'C':
+        if (nx == 'I' && nx2 == 'A') {
+          out.push_back('X');
+        } else if (nx == 'H') {
+          // SCH → SK in Philips; SX in jellyfish. Plain CH → X
+          // in both.
+          if (prev == 'S' && !jellyfish) {
+            out.push_back('K');
+          } else {
+            out.push_back('X');
+          }
+          ++i;  // consume H
+        } else if (nx == 'I' || nx == 'E' || nx == 'Y') {
+          // SCI / SCE / SCY: drop the C.
+          if (prev == 'S') break;
+          out.push_back('S');
+        } else {
+          out.push_back('K');
+        }
+        break;
+      case 'D':
+        if (nx == 'G' && (nx2 == 'E' || nx2 == 'Y' || nx2 == 'I')) {
+          out.push_back('J');
+          ++i;  // consume G
+        } else {
+          out.push_back('T');
+        }
+        break;
+      case 'F':
+        out.push_back('F');
+        break;
+      case 'G':
+        if (nx == 'H') {
+          const bool h_at_end = (i + 1 == n - 1);
+          const bool h_before_vowel = (i + 2 < n) && is_vowel(nx2);
+          if (h_at_end) {
+            // jellyfish: GH at end emits K, leaves H to follow.
+            // Philips: GH at end is silent.
+            if (jellyfish) {
+              out.push_back('K');
+              // Don't consume H; H rule handles it next iteration.
+            } else {
+              ++i;  // drop both
+            }
+            break;
+          }
+          if (!h_before_vowel) {
+            // GH not at end, H before a consonant → silent in both.
+            ++i;
+            break;
+          }
+          // GH before a vowel: G emits K, leave H for next loop.
+          out.push_back('K');
+        } else if (nx == 'N') {
+          // GN at end of word, or GNED at end → drop the G.
+          if (i + 1 == n - 1) {
+            // GN at end: drop both
+            ++i;
+          } else if (i + 3 < n && nx2 == 'E' && w[i + 3] == 'D' &&
+                     i + 3 == n - 1) {
+            // GNED at end of word: drop the G.
+            // Fall through to emit nothing for G, the rest is handled normally.
+          } else {
+            out.push_back('K');
+          }
+        } else if (nx == 'I' || nx == 'E' || nx == 'Y') {
+          if (prev == 'G') break;  // GG case
+          out.push_back('J');
+        } else {
+          out.push_back('K');
+        }
+        break;
+      case 'H':
+        // Drop H after a vowel and not before a vowel.
+        if (prev != '\0' && is_vowel(prev) && !is_vowel(nx)) break;
+        out.push_back('H');
+        break;
+      case 'J':
+        out.push_back('J');
+        break;
+      case 'K':
+        // CK collapses to K — drop K when preceded by C.
+        if (prev == 'C') break;
+        out.push_back('K');
+        break;
+      case 'L':
+        out.push_back('L');
+        break;
+      case 'M':
+        out.push_back('M');
+        break;
+      case 'N':
+        out.push_back('N');
+        break;
+      case 'P':
+        if (nx == 'H') {
+          out.push_back('F');
+          ++i;
+        } else {
+          out.push_back('P');
+        }
+        break;
+      case 'Q':
+        out.push_back('K');
+        break;
+      case 'R':
+        out.push_back('R');
+        break;
+      case 'S':
+        if (nx == 'H') {
+          out.push_back('X');
+          ++i;
+        } else if (nx == 'I' && (nx2 == 'A' || nx2 == 'O')) {
+          out.push_back('X');
+        } else {
+          out.push_back('S');
+        }
+        break;
+      case 'T':
+        if (nx == 'H') {
+          out.push_back('0');  // theta
+          ++i;
+        } else if (nx == 'I' && (nx2 == 'A' || nx2 == 'O')) {
+          out.push_back('X');
+        } else if (nx == 'C' && nx2 == 'H') {
+          // T is silent before CH (e.g. "match").
+          break;
+        } else {
+          out.push_back('T');
+        }
+        break;
+      case 'V':
+        out.push_back('F');
+        break;
+      case 'W':
+        // W emits only when followed by a vowel (e.g. "Williamson").
+        if (is_vowel(nx)) out.push_back('W');
+        break;
+      case 'X':
+        out.push_back('K');
+        out.push_back('S');
+        break;
+      case 'Y':
+        // Y emits only when followed by a vowel.
+        if (is_vowel(nx)) out.push_back('Y');
+        break;
+      case 'Z':
+        out.push_back('S');
+        break;
+    }
+  }
+
+  return out;
+}
+
+}  // namespace stride_align::phonetic