stride-align 0.6.0

data/ext/stride_align/vendor/stride_align/ngram.hpp

@@ -0,0 +1,176 @@
+#pragma once
+
+// N-gram set similarity — Phase D.1.
+//
+// Four metrics over the multiset of character n-grams of each input:
+//
+//   * **Jaccard.**   J(A, B) = |A ∩ B| / |A ∪ B|
+//   * **Sørensen-Dice.** D(A, B) = 2·|A ∩ B| / (|A| + |B|)
+//   * **Cosine.**    cos(A, B) = ⟨A, B⟩ / (‖A‖·‖B‖) over the
+//                                 multiset frequency vectors.
+//   * **Overlap.**   O(A, B) = |A ∩ B| / min(|A|, |B|)
+//
+// All four are computed over MULTISETS (each n-gram counted with
+// multiplicity). For ``a = "abab"`` and ``n = 2`` the multiset is
+// ``{"ab": 2, "ba": 1}``; for ``a = "aaaa"`` and ``n = 2`` it is
+// ``{"aa": 3}``. Set-form behaviour falls out for inputs whose n-grams
+// are all distinct.
+//
+// Identity / edge cases:
+//
+//   * Two empty inputs (or both inputs shorter than ``n``) give 1.0
+//     — vacuously identical, matches the rapidfuzz / scikit-learn
+//     convention.
+//   * One empty and one non-empty gives 0.0.
+//   * ``n == 0`` is rejected at the dispatch boundary.
+//
+// Engine works in codepoint space throughout. The dispatch wrapper
+// widens Python ``str`` storage straight out of ``PyUnicode_DATA``
+// into ``std::vector<Codepoint>`` — same convention as the other
+// stride-align codepoint-engine entry points.
+//
+// Source: standard textbook formulas. The C++ here is original.
+
+#include <algorithm>
+#include <cmath>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace stride_align::ngram {
+
+using Codepoint = std::uint32_t;
+
+// One key per n-gram: ``n`` consecutive codepoints packed as a binary
+// ``std::string`` (``n * 4`` bytes). For default ``n = 2`` and typical
+// ``n = 3 / 4`` the key fits in libstdc++'s small-string-optimisation
+// buffer (15 bytes inline) and the multiset insert allocates only the
+// hash-map node, not the key.
+using NGramKey = std::string;
+using NGramMultiset = std::unordered_map<NGramKey, std::uint32_t>;
+
+// Construct the n-gram multiset of ``input``. When ``input.size() < n``
+// the multiset is empty (the "shorter than n" edge case folds into the
+// empty-input branch of every metric formula).
+inline NGramMultiset build_multiset(const std::vector<Codepoint>& input,
+                                     std::size_t n) {
+  NGramMultiset m;
+  if (n == 0 || input.size() < n) return m;
+  const std::size_t count = input.size() - n + 1;
+  m.reserve(count);
+  NGramKey key(n * sizeof(Codepoint), '\0');
+  for (std::size_t i = 0; i < count; ++i) {
+    std::memcpy(key.data(), input.data() + i, n * sizeof(Codepoint));
+    ++m[key];
+  }
+  return m;
+}
+
+// Aggregated stats from a one-pass comparison of two multisets. Enough
+// information here to compute all four metrics without re-walking the
+// inputs.
+struct ComparisonStats {
+  std::size_t size_a = 0;       // sum of count_a (i.e. |A| with multiplicity)
+  std::size_t size_b = 0;
+  std::size_t intersection = 0; // sum over keys of ``min(count_a, count_b)``
+  std::uint64_t dot_product = 0;// sum over keys of ``count_a · count_b``
+  std::uint64_t norm_a_sq = 0;  // sum over keys of ``count_a²``
+  std::uint64_t norm_b_sq = 0;
+};
+
+inline ComparisonStats compare(const NGramMultiset& a, const NGramMultiset& b) {
+  ComparisonStats s;
+  // One pass over ``a`` covers size_a, norm_a_sq, intersection, dot.
+  // A separate pass over ``b`` picks up size_b and norm_b_sq; keys
+  // present in both sets are visited twice (once from each side) but
+  // contribute only to the corresponding side's totals.
+  for (const auto& [key, count_a] : a) {
+    s.size_a += count_a;
+    const auto ca = static_cast<std::uint64_t>(count_a);
+    s.norm_a_sq += ca * ca;
+    auto it = b.find(key);
+    if (it != b.end()) {
+      const auto count_b = it->second;
+      s.intersection += std::min(count_a, count_b);
+      s.dot_product += ca * static_cast<std::uint64_t>(count_b);
+    }
+  }
+  for (const auto& [_, count_b] : b) {
+    s.size_b += count_b;
+    const auto cb = static_cast<std::uint64_t>(count_b);
+    s.norm_b_sq += cb * cb;
+  }
+  return s;
+}
+
+// Individual metric formulas. All four follow the same identity
+// convention: two empty inputs → 1.0, one empty → 0.0.
+
+inline double jaccard_from_stats(const ComparisonStats& s) {
+  if (s.size_a == 0 && s.size_b == 0) return 1.0;
+  const std::size_t union_size = s.size_a + s.size_b - s.intersection;
+  if (union_size == 0) return 0.0;
+  return static_cast<double>(s.intersection)
+       / static_cast<double>(union_size);
+}
+
+inline double dice_from_stats(const ComparisonStats& s) {
+  if (s.size_a == 0 && s.size_b == 0) return 1.0;
+  const std::size_t total = s.size_a + s.size_b;
+  if (total == 0) return 0.0;
+  return 2.0 * static_cast<double>(s.intersection)
+       / static_cast<double>(total);
+}
+
+inline double cosine_from_stats(const ComparisonStats& s) {
+  if (s.size_a == 0 && s.size_b == 0) return 1.0;
+  if (s.norm_a_sq == 0 || s.norm_b_sq == 0) return 0.0;
+  const double denom = std::sqrt(static_cast<double>(s.norm_a_sq)
+                                 * static_cast<double>(s.norm_b_sq));
+  return static_cast<double>(s.dot_product) / denom;
+}
+
+inline double overlap_from_stats(const ComparisonStats& s) {
+  if (s.size_a == 0 && s.size_b == 0) return 1.0;
+  const std::size_t min_size = std::min(s.size_a, s.size_b);
+  if (min_size == 0) return 0.0;
+  return static_cast<double>(s.intersection)
+       / static_cast<double>(min_size);
+}
+
+// Convenience scalar entry points. ``n`` defaults to 2 (character
+// bigrams), the rapidfuzz / scikit-learn default for character-n-gram
+// similarities.
+
+inline double jaccard(const std::vector<Codepoint>& a,
+                       const std::vector<Codepoint>& b,
+                       std::size_t n = 2) {
+  return jaccard_from_stats(compare(build_multiset(a, n),
+                                     build_multiset(b, n)));
+}
+
+inline double dice(const std::vector<Codepoint>& a,
+                    const std::vector<Codepoint>& b,
+                    std::size_t n = 2) {
+  return dice_from_stats(compare(build_multiset(a, n),
+                                  build_multiset(b, n)));
+}
+
+inline double cosine(const std::vector<Codepoint>& a,
+                      const std::vector<Codepoint>& b,
+                      std::size_t n = 2) {
+  return cosine_from_stats(compare(build_multiset(a, n),
+                                    build_multiset(b, n)));
+}
+
+inline double overlap(const std::vector<Codepoint>& a,
+                       const std::vector<Codepoint>& b,
+                       std::size_t n = 2) {
+  return overlap_from_stats(compare(build_multiset(a, n),
+                                     build_multiset(b, n)));
+}
+
+}  // namespace stride_align::ngram

data/ext/stride_align/vendor/stride_align/nysiis.hpp

@@ -0,0 +1,199 @@
+#pragma once
+
+// NYSIIS — New York State Identification and Intelligence System
+// phonetic encoder (Taft, 1970).
+//
+// Developed for cross-referencing person records, NYSIIS tends to
+// produce more discriminative codes than Soundex for English-
+// language names. Returns a code up to 6 characters (the classic
+// length); does not truncate when the input is shorter.
+//
+// Algorithm steps (canonical formulation):
+//
+//   1. Translate the first letters:
+//        MAC -> MCC,  KN -> NN,  K -> C,  PH -> FF,  PF -> FF,
+//        SCH -> SSS
+//   2. Translate the last letters:
+//        EE -> Y,  IE -> Y,  DT/RT/RD/NT/ND -> D
+//   3. First character of the key is the first character of the
+//      transformed name.
+//   4. Walk the remaining characters; for each:
+//        EV -> AF
+//        any vowel (A/E/I/O/U) -> A
+//        Q -> G,  Z -> S,  M -> N
+//        KN -> N,  K -> C
+//        SCH -> SSS,  PH -> FF
+//        H -> if neighbour is non-vowel, replace with the previous
+//             key character (effectively "skip")
+//        W -> if previous is a vowel, replace with previous letter
+//   5. Skip consecutive duplicate letters in the key.
+//   6. If the key ends with S, drop it.
+//   7. If the key ends with AY, replace with Y.
+//   8. If the key ends with A, drop it.
+//   9. Truncate to length 6.
+//
+// Sources:
+//   * Taft, R. L. "Name Search Techniques." New York State
+//     Identification and Intelligence System, 1970.
+//   * Apache Commons Codec Nysiis class.
+//   * https://en.wikipedia.org/wiki/New_York_State_Identification_and_Intelligence_System
+
+#include <cstddef>
+#include <string>
+#include <string_view>
+
+namespace stride_align::phonetic {
+
+namespace nysiis_detail {
+
+inline constexpr bool is_vowel(char c) noexcept {
+  return c == 'A' || c == 'E' || c == 'I' || c == 'O' || c == 'U';
+}
+
+inline constexpr char to_upper_ascii(char c) noexcept {
+  return (c >= 'a' && c <= 'z') ? static_cast<char>(c - ('a' - 'A')) : c;
+}
+
+inline constexpr bool is_upper_alpha(char c) noexcept {
+  return c >= 'A' && c <= 'Z';
+}
+
+inline bool starts_with(std::string_view s, std::string_view prefix) noexcept {
+  return s.size() >= prefix.size() &&
+         s.compare(0, prefix.size(), prefix) == 0;
+}
+
+inline bool ends_with(std::string_view s, std::string_view suffix) noexcept {
+  return s.size() >= suffix.size() &&
+         s.compare(s.size() - suffix.size(), suffix.size(), suffix) == 0;
+}
+
+}  // namespace nysiis_detail
+
+inline std::string nysiis(std::string_view input) {
+  using namespace nysiis_detail;
+
+  // Pre-pass: keep ASCII letters only, upper-cased. No duplicate
+  // collapse here — NYSIIS handles that at the key-building stage.
+  std::string w;
+  w.reserve(input.size());
+  for (char c : input) {
+    const char uc = to_upper_ascii(c);
+    if (is_upper_alpha(uc)) w.push_back(uc);
+  }
+  if (w.empty()) return {};
+
+  // Step 1: first-letter translations.
+  if (starts_with(w, "MAC"))      w.replace(0, 3, "MCC");
+  else if (starts_with(w, "KN"))  w.replace(0, 2, "NN");
+  else if (w.front() == 'K')      w.replace(0, 1, "C");
+  else if (starts_with(w, "PH"))  w.replace(0, 2, "FF");
+  else if (starts_with(w, "PF"))  w.replace(0, 2, "FF");
+  else if (starts_with(w, "SCH")) w.replace(0, 3, "SSS");
+
+  // Step 2: last-letter translations.
+  if (ends_with(w, "EE") || ends_with(w, "IE")) {
+    w.replace(w.size() - 2, 2, "Y");
+  } else if (ends_with(w, "DT") || ends_with(w, "RT") ||
+             ends_with(w, "RD") || ends_with(w, "NT") ||
+             ends_with(w, "ND")) {
+    w.replace(w.size() - 2, 2, "D");
+  }
+
+  // Step 3: key starts with the first character.
+  std::string key;
+  key.reserve(w.size());
+  key.push_back(w.front());
+
+  // Steps 4-5: walk remaining characters, apply translations, skip
+  // consecutive duplicates in the key. Multi-letter transforms
+  // (EV→AF, KN→N, SCH→SSS, PH→FF) consume more than one input
+  // letter; we track that explicitly so the input cursor advances
+  // past every consumed letter rather than re-scanning them.
+  const std::size_t n = w.size();
+  for (std::size_t i = 1; i < n; ++i) {
+    const char c = w[i];
+    const char prev_in_word = w[i - 1];
+    const char nx = (i + 1 < n) ? w[i + 1] : '\0';
+
+    std::string translated;  // 0-3 chars
+    std::size_t consumed = 1;
+
+    if (c == 'E' && nx == 'V') {
+      translated = "AF";
+      consumed = 2;
+    } else if (is_vowel(c)) {
+      translated = "A";
+    } else if (c == 'Q') {
+      translated = "G";
+    } else if (c == 'Z') {
+      translated = "S";
+    } else if (c == 'M') {
+      translated = "N";
+    } else if (c == 'K' && nx == 'N') {
+      translated = "N";
+      consumed = 2;
+    } else if (c == 'K') {
+      translated = "C";
+    } else if (c == 'S' && nx == 'C' && i + 2 < n && w[i + 2] == 'H') {
+      translated = "SSS";
+      consumed = 3;
+    } else if (c == 'P' && nx == 'H') {
+      translated = "FF";
+      consumed = 2;
+    } else if (c == 'H') {
+      // Modern NYSIIS interpretation (matches jellyfish): both
+      // neighbours vowel → keep H; both non-vowel (or end of word)
+      // → replace with previous letter (effectively silent via
+      // key-dedup); one of each → drop H entirely.
+      const bool prev_v = is_vowel(prev_in_word);
+      const bool next_v = (nx != '\0') && is_vowel(nx);
+      if (prev_v && next_v) {
+        translated.push_back('H');
+      } else if (!prev_v && !next_v) {
+        translated.push_back(prev_in_word);
+      }
+      // else: one-of-each, drop H (translated stays empty).
+    } else if (c == 'W') {
+      // If previous is a vowel, replace with the previous letter
+      // (which then dedupes against the previous key character).
+      if (is_vowel(prev_in_word)) {
+        translated.push_back(prev_in_word);
+      } else {
+        translated.push_back('W');
+      }
+    } else {
+      translated.push_back(c);
+    }
+
+    // Step 5: append to key, skipping consecutive duplicates of the
+    // last key character. Internal duplicates inside ``translated``
+    // (e.g. the SSS in SCH→SSS) also collapse — both the leading
+    // SCH→SSS path and the mid-word case do the right thing.
+    for (char tc : translated) {
+      if (!key.empty() && key.back() == tc) continue;
+      key.push_back(tc);
+    }
+
+    // Advance i by (consumed - 1) since the for-loop ++i covers
+    // the first letter on its own.
+    i += consumed - 1;
+  }
+
+  // Steps 6-8: trailing-letter trim.
+  if (!key.empty() && key.back() == 'S') key.pop_back();
+  if (key.size() >= 2 && key[key.size() - 2] == 'A' && key.back() == 'Y') {
+    key.replace(key.size() - 2, 2, "Y");
+  }
+  if (!key.empty() && key.back() == 'A') key.pop_back();
+
+  // Truncation policy: the classical Taft 1970 paper truncates to
+  // six characters, but modern reference implementations (notably
+  // jellyfish, which is the de facto Python reference) don't. We
+  // follow the modern convention so cross-checks against jellyfish
+  // are bit-exact; callers wanting the classical six-character
+  // form can just ``[:6]`` the result themselves.
+  return key;
+}
+
+}  // namespace stride_align::phonetic