stride-align 0.6.0

data/ext/stride_align/vendor/stride_align/beider_morse.hpp

@@ -0,0 +1,121 @@
+#pragma once
+
+// Beider-Morse Phonetic Matching (BMPM) — Alexander Beider & Stephen P.
+// Morse, 2008.
+//
+// Transforms a word (typically a family name) into one or more phonetic
+// codes by recognising letter-cluster patterns and emitting alternative
+// pronunciations conditional on plausible source languages. The output
+// is a ``|``-separated string of distinct codes; an exact-match search
+// against the BMPM codes of a candidate list will surface names spelled
+// differently but pronounced similarly.
+//
+// Stride-align ships the **GENERIC** ``NameType`` only — the broad
+// "general-purpose name or word" rule set. The ``ASHKENAZI`` and
+// ``SEPHARDIC`` name types from the upstream Apache Commons Codec
+// distribution are not included.
+//
+// Pipeline (per Beider-Morse, ported from the algorithm description in
+// the upstream Apache Commons Codec 1.18 BMPM classes):
+//
+//   1. The dispatch wrapper hands the engine a ``vector<Codepoint>``
+//      widened straight out of ``PyUnicode_DATA`` (zero UTF-8 round-
+//      trip on the input side).
+//   2. Lower-case input (ASCII-only fold), replace ``-`` with space,
+//      trim.
+//   3. Handle generic name prefixes (``d'``, ``van``, ``von``, ``de``,
+//      etc.): for each recognised prefix, emit ``(encoded_remainder)-
+//      (encoded_combined)`` and stop.
+//   4. Walk the input. At each position, the rule set's Aho-Corasick
+//      trie finds the longest-prefix-matching pattern whose left and
+//      right context predicates also fire. Apply that rule's
+//      phoneme expression to the running PhonemeBuilder set, advance
+//      past the pattern. If no rule matches, drop the character.
+//   5. Apply ``common`` final rules then language-specific final rules
+//      to convert language-conditional phonemes into a language-
+//      independent representation.
+//   6. Emit ``|``-joined unique phonemes as UTF-8 (the BMPM phoneme
+//      alphabet contains Cyrillic / Greek / Polish glyphs).
+//
+// Performance:
+//   * One Aho-Corasick trie per ``(RuleType, language)`` shares a single
+//     scan over each position.
+//   * Left/right context predicates are pre-classified at static-init
+//     into ``ContextPredKind`` (empty, anchor, single char-class,
+//     literal prefix/suffix, anchored variants, regex-fallback).
+//     ``std::regex`` is only used for the few patterns the classifier
+//     rejects.
+//   * A per-call bump arena hosts the PhonemeBuilder set so the hot
+//     loop does not call ``malloc`` / ``free``.
+//   * The rule trie + final-rule maps + Lang regexes are parsed once
+//     into a single immutable ``BmpmTables`` singleton on the first
+//     ``beider_morse()`` call and reused thereafter (``std::call_once``
+//     -guarded).
+//
+// Source attribution:
+//   * Beider, A. & Morse, S.P. "Phonetic Matching: A Better Soundex"
+//     (2008). https://stevemorse.org/phonetics/bmpm.htm
+//   * Apache Commons Codec ``org.apache.commons.codec.language.bm.*``
+//     (Apache 2.0) for the algorithm structure and the rule data
+//     files. The vendored upstream rule files live in
+//     ``src/stride_align/bmpm_data/`` with their original ASF headers
+//     intact.
+//
+// The C++ port and the Aho-Corasick / arena / predicate-classifier
+// machinery are original.
+
+#include <cstddef>
+#include <cstdint>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+namespace stride_align::phonetic {
+
+// The engine works in codepoint space end-to-end. Python ``str`` storage
+// is fixed-width per string (``PyUnicode_KIND`` returns 1/2/4 bytes per
+// codepoint, ``PyUnicode_DATA`` is the raw codepoint array); the
+// dispatch wrapper widens that storage straight into
+// ``std::vector<Codepoint>`` without UTF-8 round-tripping on the input
+// side. UTF-8 only appears on the output side (the returned phonetic
+// codes) and inside the ``std::regex`` predicate fallback, which
+// lazy-encodes the codepoint slice only when a regex predicate is
+// about to fire.
+using Codepoint = std::uint32_t;
+
+enum class BmpmRuleType : int {
+  kApprox = 0,
+  kExact  = 1,
+};
+
+// Register the Apache Commons Codec BMPM resource files with the engine.
+// ``resources`` maps the basename (e.g. ``"gen_rules_english"``, no
+// ``.txt`` suffix) to the file's textual content. Safe to call multiple
+// times — the first registration wins and the engine is built lazily on
+// the first ``beider_morse()`` call after registration.
+//
+// Throws ``std::runtime_error`` if a required resource is missing or
+// malformed.
+void bmpm_register_resources(
+    const std::unordered_map<std::string, std::string>& resources);
+
+// Encode ``input`` into one or more BMPM phonetic codes joined with
+// ``|``. Returns an empty string when the engine has not been
+// registered or when ``input`` contains no encodable characters.
+//
+// ``rule_type`` picks the rule family: ``kApprox`` produces a broader
+// phonetic spread; ``kExact`` a tighter one.
+//
+// ``concat`` controls multi-word names: when true, the encoded form is
+// produced for the concatenated whole; when false, each word is
+// encoded separately and the per-word codes are joined with ``-``.
+//
+// ``max_phonemes`` caps the PhonemeBuilder set size (default 20,
+// matching the upstream Commons Codec default).
+std::string beider_morse(
+    const std::vector<Codepoint>& input,
+    BmpmRuleType rule_type = BmpmRuleType::kApprox,
+    bool concat = true,
+    std::size_t max_phonemes = 20);
+
+}  // namespace stride_align::phonetic

data/ext/stride_align/vendor/stride_align/caverphone.hpp

@@ -0,0 +1,222 @@
+#pragma once
+
+// Caverphone 2.0 (David Hood, University of Otago, 2004).
+//
+// Phonetic encoder originally designed to match names from late-19th-
+// century New Zealand electoral rolls; produces a fixed-length
+// 10-character code right-padded with the digit ``1``. Caverphone 2
+// supersedes the original 1.0 algorithm; we ship v2 only.
+//
+// The algorithm is a cascade of ~40 substitutions applied in a
+// specific order. Order matters — most rules depend on the
+// previous rules having already fired. The reference is Hood's
+// "Caverphone Revisited" paper. We follow the rule list as
+// reproduced in Apache Commons Codec Caverphone2.
+//
+// Sources:
+//   * Hood, D. "Caverphone Revisited." Technical Report, University
+//     of Otago, 2004.
+//   * Apache Commons Codec Caverphone2 class.
+//   * https://en.wikipedia.org/wiki/Caverphone
+
+#include <cstddef>
+#include <string>
+#include <string_view>
+
+namespace stride_align::phonetic {
+
+namespace caverphone_detail {
+
+inline constexpr char to_lower_ascii(char c) noexcept {
+  return (c >= 'A' && c <= 'Z') ? static_cast<char>(c + ('a' - 'A')) : c;
+}
+
+inline constexpr bool is_lower_alpha(char c) noexcept {
+  return c >= 'a' && c <= 'z';
+}
+
+// In-place find/replace-all of a single substring.
+inline void replace_all(std::string& s,
+                        std::string_view from,
+                        std::string_view to) {
+  if (from.empty()) return;
+  std::size_t pos = 0;
+  while ((pos = s.find(from, pos)) != std::string::npos) {
+    s.replace(pos, from.size(), to);
+    pos += to.size();
+  }
+}
+
+inline bool starts_with(std::string_view s, std::string_view prefix) noexcept {
+  return s.size() >= prefix.size() &&
+         s.compare(0, prefix.size(), prefix) == 0;
+}
+
+inline bool ends_with(std::string_view s, std::string_view suffix) noexcept {
+  return s.size() >= suffix.size() &&
+         s.compare(s.size() - suffix.size(), suffix.size(), suffix) == 0;
+}
+
+}  // namespace caverphone_detail
+
+inline std::string caverphone(std::string_view input) {
+  using namespace caverphone_detail;
+
+  // Step 0: lower-case ASCII only.
+  std::string w;
+  w.reserve(input.size());
+  for (char c : input) {
+    const char lc = to_lower_ascii(c);
+    if (is_lower_alpha(lc)) w.push_back(lc);
+  }
+  if (w.empty()) return std::string(10, '1');
+
+  // Initial "-ough" word replacements: only if the entire word starts
+  // with the listed prefix.
+  if (starts_with(w, "cough"))  w.replace(0, 5, "cou2f");
+  else if (starts_with(w, "rough")) w.replace(0, 5, "rou2f");
+  else if (starts_with(w, "tough")) w.replace(0, 5, "tou2f");
+  else if (starts_with(w, "enough")) w.replace(0, 6, "enou2f");
+  else if (starts_with(w, "trough")) w.replace(0, 6, "trou2f");
+
+  // Initial gn -> 2n.
+  if (starts_with(w, "gn")) w.replace(0, 2, "2n");
+  // Final mb -> m2.
+  if (ends_with(w, "mb")) w.replace(w.size() - 2, 2, "m2");
+
+  // Plain substring substitutions (order-sensitive — each rule
+  // assumes the preceding ones have already fired).
+  replace_all(w, "cq", "2q");
+  replace_all(w, "ci", "si");
+  replace_all(w, "ce", "se");
+  replace_all(w, "cy", "sy");
+  replace_all(w, "tch", "2ch");
+  replace_all(w, "c", "k");
+  replace_all(w, "q", "k");
+  replace_all(w, "x", "k");
+  replace_all(w, "v", "f");
+  replace_all(w, "dg", "2g");
+  replace_all(w, "tio", "sio");
+  replace_all(w, "tia", "sia");
+  replace_all(w, "d", "t");
+  replace_all(w, "ph", "fh");
+  replace_all(w, "b", "p");
+  replace_all(w, "sh", "s2");
+  replace_all(w, "z", "s");
+
+  // Initial vowel -> A; remaining vowels -> 3.
+  if (!w.empty()) {
+    const char c0 = w.front();
+    if (c0 == 'a' || c0 == 'e' || c0 == 'i' || c0 == 'o' || c0 == 'u') {
+      w[0] = 'A';
+    }
+  }
+  for (char& c : w) {
+    if (c == 'a' || c == 'e' || c == 'i' || c == 'o' || c == 'u') c = '3';
+  }
+
+  // GH context.
+  replace_all(w, "3gh3", "3kh3");
+  replace_all(w, "gh", "22");
+  replace_all(w, "g", "k");
+
+  // Consonant-run collapse: each rule maps consecutive same-letter
+  // sequences to a single upper-case marker, then back to lower.
+  // Order matters; we follow the published rule order.
+  auto collapse_runs = [&](char ch, char marker) {
+    // Replace runs of 2+ instances of ``ch`` with a single ``marker``.
+    std::string out;
+    out.reserve(w.size());
+    std::size_t i = 0;
+    while (i < w.size()) {
+      if (w[i] == ch) {
+        std::size_t j = i;
+        while (j < w.size() && w[j] == ch) ++j;
+        out.push_back(j - i >= 2 ? marker : ch);
+        i = j;
+      } else {
+        out.push_back(w[i]);
+        ++i;
+      }
+    }
+    w.swap(out);
+    for (char& c : w) {
+      if (c == marker) c = ch;
+    }
+  };
+  collapse_runs('s', 'S');
+  collapse_runs('t', 'T');
+  collapse_runs('p', 'P');
+  collapse_runs('k', 'K');
+  collapse_runs('f', 'F');
+  collapse_runs('m', 'M');
+  collapse_runs('n', 'N');
+
+  // W rules.
+  replace_all(w, "w3", "W3");
+  replace_all(w, "wh3", "Wh3");
+  if (ends_with(w, "w")) w.back() = '3';
+  replace_all(w, "w", "2");
+
+  // H rules.
+  if (!w.empty() && w.front() == 'h') w[0] = 'A';
+  replace_all(w, "h", "2");
+
+  // R rules.
+  replace_all(w, "r3", "R3");
+  if (ends_with(w, "r")) w.back() = '3';
+  replace_all(w, "r", "2");
+
+  // L rules. The liquid L is preserved only when it sits between
+  // two vowel markers — "Cailean" (k33L33n) keeps L because both
+  // neighbours are 3; "hold" (A3lt) and "able" (Apl3) drop L
+  // because at least one neighbour is a consonant.
+  replace_all(w, "3l3", "3L3");
+  if (ends_with(w, "l")) w.back() = '3';
+  replace_all(w, "l", "2");
+
+  // J / Y rules.
+  replace_all(w, "j", "y");
+  if (starts_with(w, "y3")) w.replace(0, 2, "Y3");
+  if (!w.empty() && w.front() == 'y') w[0] = 'A';
+  replace_all(w, "y", "3");
+
+  // Strip 2s first, then handle the "trailing 3 → A" rule, then
+  // strip the remaining 3s. Order matters — the trailing 3 needs to
+  // survive past the 2-strip to be picked up here.
+  {
+    std::string out;
+    out.reserve(w.size());
+    for (char c : w) {
+      if (c != '2') out.push_back(c);
+    }
+    w.swap(out);
+  }
+  // Trailing 3 → A (gives names like "Peter" their final "A").
+  if (!w.empty() && w.back() == '3') w.back() = 'A';
+  // Strip remaining 3s.
+  {
+    std::string out;
+    out.reserve(w.size());
+    for (char c : w) {
+      if (c != '3') out.push_back(c);
+    }
+    w.swap(out);
+  }
+
+  // Lower-case the marker letters back to plain ASCII.
+  for (char& c : w) {
+    if (c >= 'A' && c <= 'Z') c = static_cast<char>(c + ('a' - 'A'));
+  }
+
+  // Pad with '1' to length 10, or truncate to 10.
+  if (w.size() >= 10) w.resize(10);
+  else                w.append(10 - w.size(), '1');
+  // Upper-case for the final code (Apache Commons Codec convention).
+  for (char& c : w) {
+    if (c >= 'a' && c <= 'z') c = static_cast<char>(c - ('a' - 'A'));
+  }
+  return w;
+}
+
+}  // namespace stride_align::phonetic

data/ext/stride_align/vendor/stride_align/cologne_phonetic.hpp

@@ -0,0 +1,202 @@
+#pragma once
+
+// Cologne Phonetic (Kölner Phonetik) — Hans Joachim Postel, 1969.
+//
+// German-language phonetic encoder. Maps letters / letter-pairs to
+// digits 0-8 with context-sensitive rules for C, X, D, T, P; then
+// collapses adjacent duplicate digits; then drops all '0' digits
+// except a leading one. Result is a digit string of variable length.
+//
+// Designed for German names but widely applied to general European
+// surnames. Distinguishes pairs like "Müller" / "Maier" that
+// Soundex / Metaphone (English-oriented) tend to collapse the same.
+//
+// Sources:
+//   * Postel, H.-J. "Die Kölner Phonetik" (1969).
+//   * https://en.wikipedia.org/wiki/Cologne_phonetics
+//   * https://de.wikipedia.org/wiki/K%C3%B6lner_Phonetik
+//   * Apache Commons Codec ColognePhonetic class.
+//
+// Cross-checked against the Apache Commons Codec reference vectors
+// and the Wikipedia examples (Wikipedia → 3412, Breschnew → 17863,
+// Müller → 657).
+
+#include <cstddef>
+#include <cstdint>
+#include <string>
+#include <vector>
+
+namespace stride_align::phonetic {
+
+// The engine runs in codepoint space. Python ``str`` storage is
+// fixed-width per string (``PyUnicode_KIND`` is 1/2/4 bytes per
+// codepoint, ``PyUnicode_DATA`` is the raw codepoint array); the
+// dispatch wrapper widens that storage straight into
+// ``std::vector<Codepoint>`` without UTF-8 round-tripping.
+using Codepoint = std::uint32_t;
+
+namespace cologne_detail {
+
+inline constexpr char to_upper_ascii(char c) noexcept {
+  return (c >= 'a' && c <= 'z') ? static_cast<char>(c - ('a' - 'A')) : c;
+}
+
+inline constexpr bool is_upper_alpha(char c) noexcept {
+  return c >= 'A' && c <= 'Z';
+}
+
+// Apache Commons Codec preprocesses the German umlauts and ß into
+// their Latin-letter equivalents before running the digit table; we
+// follow that so callers don't have to NFKD-fold themselves. The
+// fold is now keyed on the codepoint (the natural Unicode value)
+// rather than on UTF-8 byte pairs:
+//   Ä U+00C4 / ä U+00E4 -> A
+//   Ö U+00D6 / ö U+00F6 -> O
+//   Ü U+00DC / ü U+00FC -> U
+//   ß U+00DF            -> SS
+inline std::string preprocess_to_ascii_letters(
+    const std::vector<Codepoint>& input) {
+  std::string out;
+  out.reserve(input.size());
+  for (const auto cp : input) {
+    switch (cp) {
+      case 0x00C4: case 0x00E4: out.push_back('A');     continue;  // Ä / ä
+      case 0x00D6: case 0x00F6: out.push_back('O');     continue;  // Ö / ö
+      case 0x00DC: case 0x00FC: out.push_back('U');     continue;  // Ü / ü
+      case 0x00DF:              out.append("SS");       continue;  // ß
+      default: break;
+    }
+    if (cp >= 128U) continue;  // any other non-ASCII codepoint is dropped
+    const char uc = to_upper_ascii(static_cast<char>(cp));
+    if (is_upper_alpha(uc)) out.push_back(uc);
+  }
+  return out;
+}
+
+}  // namespace cologne_detail
+
+inline std::string cologne_phonetic(const std::vector<Codepoint>& input) {
+  using namespace cologne_detail;
+
+  const std::string w = preprocess_to_ascii_letters(input);
+  if (w.empty()) return {};
+
+  const std::size_t n = w.size();
+  auto at = [&](std::size_t i) noexcept -> char {
+    return i < n ? w[i] : '\0';
+  };
+
+  // Step 1: emit digits per rule. H is silent (no digit), but it
+  // still counts as the previous letter for context. We track the
+  // previous SOURCE letter (not the previous emitted digit) so the
+  // "after S/Z" check for C looks at the right thing.
+  std::string codes;
+  codes.reserve(n);
+  for (std::size_t i = 0; i < n; ++i) {
+    const char ch = w[i];
+    const char next = at(i + 1);
+    const char prev = (i > 0) ? w[i - 1] : '\0';
+
+    switch (ch) {
+      case 'A': case 'E': case 'I': case 'J': case 'O': case 'U': case 'Y':
+        codes.push_back('0');
+        break;
+      case 'H':
+        // Silent — emit no digit. Postel's original rule: H is a
+        // "modifier" letter; it can appear in context lookups but
+        // never as an output digit on its own.
+        break;
+      case 'B':
+        codes.push_back('1');
+        break;
+      case 'P':
+        // P -> 3 before H ("Philip" -> 3...); P -> 1 elsewhere.
+        codes.push_back(next == 'H' ? '3' : '1');
+        break;
+      case 'D': case 'T':
+        // D/T -> 8 before C/S/Z (the affricate); -> 2 elsewhere.
+        codes.push_back(
+            (next == 'C' || next == 'S' || next == 'Z') ? '8' : '2');
+        break;
+      case 'F': case 'V': case 'W':
+        codes.push_back('3');
+        break;
+      case 'G': case 'K': case 'Q':
+        codes.push_back('4');
+        break;
+      case 'C': {
+        // C is the most context-sensitive letter. Rules:
+        //   * Word-initial: -> 4 before A, H, K, L, O, Q, R, U, X;
+        //                    -> 8 otherwise.
+        //   * Non-initial:  -> 4 before A, H, K, O, Q, U, X
+        //                      and NOT immediately after S or Z;
+        //                    -> 8 otherwise.
+        // The L / R exception only applies at start of word.
+        const bool before_hard_initial =
+            next == 'A' || next == 'H' || next == 'K' || next == 'L' ||
+            next == 'O' || next == 'Q' || next == 'R' || next == 'U' ||
+            next == 'X';
+        const bool before_hard_inner =
+            next == 'A' || next == 'H' || next == 'K' || next == 'O' ||
+            next == 'Q' || next == 'U' || next == 'X';
+        if (i == 0) {
+          codes.push_back(before_hard_initial ? '4' : '8');
+        } else if (before_hard_inner && prev != 'S' && prev != 'Z') {
+          codes.push_back('4');
+        } else {
+          codes.push_back('8');
+        }
+        break;
+      }
+      case 'X':
+        // X -> 8 after C / K / Q (since the C/K/Q already emitted the
+        // /k/ part); X -> 48 elsewhere (representing /ks/).
+        if (prev == 'C' || prev == 'K' || prev == 'Q') {
+          codes.push_back('8');
+        } else {
+          codes.push_back('4');
+          codes.push_back('8');
+        }
+        break;
+      case 'L':
+        codes.push_back('5');
+        break;
+      case 'M': case 'N':
+        codes.push_back('6');
+        break;
+      case 'R':
+        codes.push_back('7');
+        break;
+      case 'S': case 'Z':
+        codes.push_back('8');
+        break;
+      default:
+        // Already filtered to A-Z in preprocess_to_ascii_letters.
+        break;
+    }
+  }
+
+  // Step 2: collapse adjacent duplicate digits.
+  std::string collapsed;
+  collapsed.reserve(codes.size());
+  for (char c : codes) {
+    if (collapsed.empty() || collapsed.back() != c) {
+      collapsed.push_back(c);
+    }
+  }
+
+  // Step 3: drop all '0' digits except a leading one (when present).
+  // "Mayer" -> 607 -> after dropping internal zeros -> 67 (no leading
+  // 0 to preserve). "Aaron" -> 00706 -> collapse -> 0706 -> keep the
+  // leading 0, drop the rest -> 076.
+  std::string result;
+  result.reserve(collapsed.size());
+  for (std::size_t i = 0; i < collapsed.size(); ++i) {
+    if (i == 0 || collapsed[i] != '0') {
+      result.push_back(collapsed[i]);
+    }
+  }
+  return result;
+}
+
+}  // namespace stride_align::phonetic