stride-align 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (190) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE +201 -0
  3. data/NOTICE +255 -0
  4. data/README.md +107 -0
  5. data/data/bmpm_data/gen_approx_any.txt +131 -0
  6. data/data/bmpm_data/gen_approx_arabic.txt +26 -0
  7. data/data/bmpm_data/gen_approx_common.txt +233 -0
  8. data/data/bmpm_data/gen_approx_cyrillic.txt +18 -0
  9. data/data/bmpm_data/gen_approx_czech.txt +18 -0
  10. data/data/bmpm_data/gen_approx_dutch.txt +18 -0
  11. data/data/bmpm_data/gen_approx_english.txt +47 -0
  12. data/data/bmpm_data/gen_approx_french.txt +25 -0
  13. data/data/bmpm_data/gen_approx_german.txt +73 -0
  14. data/data/bmpm_data/gen_approx_greek.txt +18 -0
  15. data/data/bmpm_data/gen_approx_greeklatin.txt +20 -0
  16. data/data/bmpm_data/gen_approx_hebrew.txt +18 -0
  17. data/data/bmpm_data/gen_approx_hungarian.txt +18 -0
  18. data/data/bmpm_data/gen_approx_italian.txt +18 -0
  19. data/data/bmpm_data/gen_approx_polish.txt +84 -0
  20. data/data/bmpm_data/gen_approx_portuguese.txt +18 -0
  21. data/data/bmpm_data/gen_approx_romanian.txt +18 -0
  22. data/data/bmpm_data/gen_approx_russian.txt +48 -0
  23. data/data/bmpm_data/gen_approx_spanish.txt +21 -0
  24. data/data/bmpm_data/gen_approx_turkish.txt +18 -0
  25. data/data/bmpm_data/gen_exact_any.txt +40 -0
  26. data/data/bmpm_data/gen_exact_approx_common.txt +79 -0
  27. data/data/bmpm_data/gen_exact_arabic.txt +18 -0
  28. data/data/bmpm_data/gen_exact_common.txt +32 -0
  29. data/data/bmpm_data/gen_exact_cyrillic.txt +18 -0
  30. data/data/bmpm_data/gen_exact_czech.txt +18 -0
  31. data/data/bmpm_data/gen_exact_dutch.txt +18 -0
  32. data/data/bmpm_data/gen_exact_english.txt +18 -0
  33. data/data/bmpm_data/gen_exact_french.txt +18 -0
  34. data/data/bmpm_data/gen_exact_german.txt +18 -0
  35. data/data/bmpm_data/gen_exact_greek.txt +18 -0
  36. data/data/bmpm_data/gen_exact_greeklatin.txt +18 -0
  37. data/data/bmpm_data/gen_exact_hebrew.txt +18 -0
  38. data/data/bmpm_data/gen_exact_hungarian.txt +18 -0
  39. data/data/bmpm_data/gen_exact_italian.txt +18 -0
  40. data/data/bmpm_data/gen_exact_polish.txt +23 -0
  41. data/data/bmpm_data/gen_exact_portuguese.txt +18 -0
  42. data/data/bmpm_data/gen_exact_romanian.txt +18 -0
  43. data/data/bmpm_data/gen_exact_russian.txt +19 -0
  44. data/data/bmpm_data/gen_exact_spanish.txt +19 -0
  45. data/data/bmpm_data/gen_exact_turkish.txt +18 -0
  46. data/data/bmpm_data/gen_hebrew_common.txt +113 -0
  47. data/data/bmpm_data/gen_lang.txt +295 -0
  48. data/data/bmpm_data/gen_languages.txt +36 -0
  49. data/data/bmpm_data/gen_rules_any.txt +367 -0
  50. data/data/bmpm_data/gen_rules_arabic.txt +76 -0
  51. data/data/bmpm_data/gen_rules_cyrillic.txt +99 -0
  52. data/data/bmpm_data/gen_rules_czech.txt +67 -0
  53. data/data/bmpm_data/gen_rules_dutch.txt +78 -0
  54. data/data/bmpm_data/gen_rules_english.txt +113 -0
  55. data/data/bmpm_data/gen_rules_french.txt +114 -0
  56. data/data/bmpm_data/gen_rules_german.txt +129 -0
  57. data/data/bmpm_data/gen_rules_greek.txt +97 -0
  58. data/data/bmpm_data/gen_rules_greeklatin.txt +118 -0
  59. data/data/bmpm_data/gen_rules_hebrew.txt +62 -0
  60. data/data/bmpm_data/gen_rules_hungarian.txt +83 -0
  61. data/data/bmpm_data/gen_rules_italian.txt +77 -0
  62. data/data/bmpm_data/gen_rules_polish.txt +185 -0
  63. data/data/bmpm_data/gen_rules_portuguese.txt +105 -0
  64. data/data/bmpm_data/gen_rules_romanian.txt +64 -0
  65. data/data/bmpm_data/gen_rules_russian.txt +142 -0
  66. data/data/bmpm_data/gen_rules_spanish.txt +85 -0
  67. data/data/bmpm_data/gen_rules_turkish.txt +50 -0
  68. data/data/keyboard_data/qwerty.npy +0 -0
  69. data/data/matrix_data/BLOSUM100 +31 -0
  70. data/data/matrix_data/BLOSUM30 +31 -0
  71. data/data/matrix_data/BLOSUM35 +31 -0
  72. data/data/matrix_data/BLOSUM40 +31 -0
  73. data/data/matrix_data/BLOSUM45 +25 -0
  74. data/data/matrix_data/BLOSUM50 +25 -0
  75. data/data/matrix_data/BLOSUM55 +31 -0
  76. data/data/matrix_data/BLOSUM60 +31 -0
  77. data/data/matrix_data/BLOSUM62 +25 -0
  78. data/data/matrix_data/BLOSUM65 +31 -0
  79. data/data/matrix_data/BLOSUM70 +31 -0
  80. data/data/matrix_data/BLOSUM75 +31 -0
  81. data/data/matrix_data/BLOSUM80 +25 -0
  82. data/data/matrix_data/BLOSUM85 +31 -0
  83. data/data/matrix_data/BLOSUM90 +25 -0
  84. data/data/matrix_data/NUC.4.4 +25 -0
  85. data/data/matrix_data/PAM10 +34 -0
  86. data/data/matrix_data/PAM100 +34 -0
  87. data/data/matrix_data/PAM110 +34 -0
  88. data/data/matrix_data/PAM120 +34 -0
  89. data/data/matrix_data/PAM130 +34 -0
  90. data/data/matrix_data/PAM140 +34 -0
  91. data/data/matrix_data/PAM150 +34 -0
  92. data/data/matrix_data/PAM160 +34 -0
  93. data/data/matrix_data/PAM170 +34 -0
  94. data/data/matrix_data/PAM180 +34 -0
  95. data/data/matrix_data/PAM190 +34 -0
  96. data/data/matrix_data/PAM20 +34 -0
  97. data/data/matrix_data/PAM200 +34 -0
  98. data/data/matrix_data/PAM210 +34 -0
  99. data/data/matrix_data/PAM220 +34 -0
  100. data/data/matrix_data/PAM230 +34 -0
  101. data/data/matrix_data/PAM240 +34 -0
  102. data/data/matrix_data/PAM250 +25 -0
  103. data/data/matrix_data/PAM260 +34 -0
  104. data/data/matrix_data/PAM270 +34 -0
  105. data/data/matrix_data/PAM280 +34 -0
  106. data/data/matrix_data/PAM290 +34 -0
  107. data/data/matrix_data/PAM30 +25 -0
  108. data/data/matrix_data/PAM300 +34 -0
  109. data/data/matrix_data/PAM310 +34 -0
  110. data/data/matrix_data/PAM320 +34 -0
  111. data/data/matrix_data/PAM330 +34 -0
  112. data/data/matrix_data/PAM340 +34 -0
  113. data/data/matrix_data/PAM350 +34 -0
  114. data/data/matrix_data/PAM360 +34 -0
  115. data/data/matrix_data/PAM370 +34 -0
  116. data/data/matrix_data/PAM380 +34 -0
  117. data/data/matrix_data/PAM390 +34 -0
  118. data/data/matrix_data/PAM40 +34 -0
  119. data/data/matrix_data/PAM400 +34 -0
  120. data/data/matrix_data/PAM410 +34 -0
  121. data/data/matrix_data/PAM420 +34 -0
  122. data/data/matrix_data/PAM430 +34 -0
  123. data/data/matrix_data/PAM440 +34 -0
  124. data/data/matrix_data/PAM450 +34 -0
  125. data/data/matrix_data/PAM460 +34 -0
  126. data/data/matrix_data/PAM470 +34 -0
  127. data/data/matrix_data/PAM480 +34 -0
  128. data/data/matrix_data/PAM490 +34 -0
  129. data/data/matrix_data/PAM50 +34 -0
  130. data/data/matrix_data/PAM500 +34 -0
  131. data/data/matrix_data/PAM60 +34 -0
  132. data/data/matrix_data/PAM70 +25 -0
  133. data/data/matrix_data/PAM80 +34 -0
  134. data/data/matrix_data/PAM90 +34 -0
  135. data/ext/stride_align/backend_avx2.cpp +2 -0
  136. data/ext/stride_align/backend_avx512bwvl.cpp +2 -0
  137. data/ext/stride_align/backend_generic.cpp +3 -0
  138. data/ext/stride_align/backend_impl.hpp +983 -0
  139. data/ext/stride_align/backend_lasx.cpp +2 -0
  140. data/ext/stride_align/backend_lsx.cpp +2 -0
  141. data/ext/stride_align/backend_neon.cpp +2 -0
  142. data/ext/stride_align/backend_rvv.cpp +2 -0
  143. data/ext/stride_align/backend_sse41.cpp +2 -0
  144. data/ext/stride_align/backend_sve.cpp +2 -0
  145. data/ext/stride_align/backend_sve2.cpp +2 -0
  146. data/ext/stride_align/backend_vsx.cpp +2 -0
  147. data/ext/stride_align/beider_morse_impl.cpp +5 -0
  148. data/ext/stride_align/cpu_detect.cpp +172 -0
  149. data/ext/stride_align/cpu_detect.hpp +6 -0
  150. data/ext/stride_align/extconf.rb +114 -0
  151. data/ext/stride_align/target_profile.hpp +82 -0
  152. data/ext/stride_align/vendor/beider_morse_impl.cpp +1467 -0
  153. data/ext/stride_align/vendor/stride_align/alignment.hpp +199 -0
  154. data/ext/stride_align/vendor/stride_align/batch.hpp +812 -0
  155. data/ext/stride_align/vendor/stride_align/beider_morse.hpp +121 -0
  156. data/ext/stride_align/vendor/stride_align/caverphone.hpp +222 -0
  157. data/ext/stride_align/vendor/stride_align/cologne_phonetic.hpp +202 -0
  158. data/ext/stride_align/vendor/stride_align/core.hpp +731 -0
  159. data/ext/stride_align/vendor/stride_align/daitch_mokotoff.hpp +631 -0
  160. data/ext/stride_align/vendor/stride_align/double_metaphone.hpp +796 -0
  161. data/ext/stride_align/vendor/stride_align/dtw.hpp +300 -0
  162. data/ext/stride_align/vendor/stride_align/encoded.hpp +235 -0
  163. data/ext/stride_align/vendor/stride_align/hamming.hpp +55 -0
  164. data/ext/stride_align/vendor/stride_align/indel.hpp +1200 -0
  165. data/ext/stride_align/vendor/stride_align/jaro.hpp +517 -0
  166. data/ext/stride_align/vendor/stride_align/lcs.hpp +159 -0
  167. data/ext/stride_align/vendor/stride_align/levenshtein.hpp +1247 -0
  168. data/ext/stride_align/vendor/stride_align/levenshtein_prepared.hpp +193 -0
  169. data/ext/stride_align/vendor/stride_align/match_rating.hpp +168 -0
  170. data/ext/stride_align/vendor/stride_align/metaphone.hpp +291 -0
  171. data/ext/stride_align/vendor/stride_align/ngram.hpp +176 -0
  172. data/ext/stride_align/vendor/stride_align/nysiis.hpp +199 -0
  173. data/ext/stride_align/vendor/stride_align/pairwise_alignment.hpp +465 -0
  174. data/ext/stride_align/vendor/stride_align/partial_ratio.hpp +486 -0
  175. data/ext/stride_align/vendor/stride_align/ratcliff_obershelp.hpp +101 -0
  176. data/ext/stride_align/vendor/stride_align/soundex.hpp +108 -0
  177. data/ext/stride_align/vendor/stride_align/token_ratios.hpp +445 -0
  178. data/ext/stride_align/vendor/stride_align/types.hpp +16 -0
  179. data/ext/stride_align/vendor/stride_align/utf8.hpp +512 -0
  180. data/ext/stride_align/vendor/stride_align/wratio.hpp +363 -0
  181. data/lib/stride_align/algorithms.rb +296 -0
  182. data/lib/stride_align/alignment_path.rb +217 -0
  183. data/lib/stride_align/backend.rb +47 -0
  184. data/lib/stride_align/batch.rb +705 -0
  185. data/lib/stride_align/core.rb +180 -0
  186. data/lib/stride_align/keyboard.rb +200 -0
  187. data/lib/stride_align/matrices.rb +403 -0
  188. data/lib/stride_align/version.rb +5 -0
  189. data/lib/stride_align.rb +87 -0
  190. metadata +231 -0
@@ -0,0 +1,517 @@
1
+ #pragma once
2
+
3
+ // Jaro similarity (Jaro 1989) and Jaro-Winkler (Winkler 1990).
4
+ //
5
+ // Jaro counts how many characters of two strings can be paired up within
6
+ // a sliding window of floor(max(|a|, |b|) / 2) - 1 positions, plus a
7
+ // half-transposition penalty for matches that occur out of order. The
8
+ // resulting similarity is in [0, 1]; 1.0 = identical, 0.0 = no matches
9
+ // inside the window.
10
+ //
11
+ // Jaro-Winkler adds a length-of-common-prefix bonus (capped at 4 chars
12
+ // by convention) when the base Jaro score crosses a threshold (0.7 by
13
+ // convention, matching rapidfuzz). The bonus magnifies similarity for
14
+ // strings that agree at the start — useful for surname/place-name
15
+ // matching, less useful for arbitrary text.
16
+ //
17
+ // Neither metric has a DP recurrence; bit-parallel implementations live
18
+ // in src/cpp/jaro_simd.hpp and use a PEQ-style position bitmap of the
19
+ // query alphabet to find matches in O(1) per target character.
20
+ // This header carries the scalar reference + the threshold/weight
21
+ // defaults used by the bindings layer.
22
+
23
+ #include <algorithm>
24
+ #include <array>
25
+ #include <bit>
26
+ #include <cstddef>
27
+ #include <cstdint>
28
+ #include <span>
29
+ #include <unordered_map>
30
+ #include <vector>
31
+
32
+ #include "stride_align/alignment.hpp"
33
+
34
+ namespace stride_align::jaro {
35
+
36
+ // Defaults match rapidfuzz's Jaro/JaroWinkler so that
37
+ // stride_align.jaro_winkler_similarity(a, b) == rapidfuzz at machine
38
+ // precision for byte inputs.
39
+ inline constexpr double kDefaultPrefixWeight = 0.1;
40
+ inline constexpr double kDefaultPrefixThreshold = 0.7;
41
+ inline constexpr std::size_t kDefaultPrefixCap = 4U;
42
+
43
+ // Half the maximum length, minus one, clamped to >= 0. Standard Jaro
44
+ // match window. For max <= 3 the formula underflows in unsigned
45
+ // arithmetic; the clamp gives "same-position matches only", which is
46
+ // what the floor would have produced anyway.
47
+ inline std::size_t match_window(std::size_t n, std::size_t m) noexcept {
48
+ const std::size_t max_len = std::max(n, m);
49
+ return max_len >= 2U ? max_len / 2U - 1U : 0U;
50
+ }
51
+
52
+ // Scalar Jaro similarity in [0, 1].
53
+ //
54
+ // Caller guarantees both spans use the same token width (templated, no
55
+ // implicit conversion). For empty inputs the convention is:
56
+ // * both empty → 1.0 (vacuously identical)
57
+ // * one empty → 0.0 (no possible match)
58
+ // This matches rapidfuzz.distance.Jaro.similarity.
59
+ template <typename Token>
60
+ inline double jaro_scalar(
61
+ std::span<const Token> a,
62
+ std::span<const Token> b) {
63
+ const std::size_t n = a.size();
64
+ const std::size_t m = b.size();
65
+ if (n == 0U && m == 0U) {
66
+ return 1.0;
67
+ }
68
+ if (n == 0U || m == 0U) {
69
+ return 0.0;
70
+ }
71
+
72
+ const std::size_t window = match_window(n, m);
73
+
74
+ std::vector<std::uint8_t> a_matched(n, 0U);
75
+ std::vector<std::uint8_t> b_matched(m, 0U);
76
+ std::size_t matches = 0U;
77
+
78
+ for (std::size_t i = 0; i < n; ++i) {
79
+ const std::size_t lo = i > window ? i - window : 0U;
80
+ const std::size_t hi = std::min(i + window + 1U, m);
81
+ for (std::size_t j = lo; j < hi; ++j) {
82
+ if (b_matched[j] != 0U) {
83
+ continue;
84
+ }
85
+ if (a[i] != b[j]) {
86
+ continue;
87
+ }
88
+ a_matched[i] = 1U;
89
+ b_matched[j] = 1U;
90
+ ++matches;
91
+ break;
92
+ }
93
+ }
94
+
95
+ if (matches == 0U) {
96
+ return 0.0;
97
+ }
98
+
99
+ // Half-transposition count: walk matched chars of a and b in order;
100
+ // each position where they differ contributes 1 to half_trans (which
101
+ // is then divided by 2 for the formula).
102
+ std::size_t half_trans = 0U;
103
+ std::size_t k = 0U;
104
+ for (std::size_t i = 0; i < n; ++i) {
105
+ if (a_matched[i] == 0U) {
106
+ continue;
107
+ }
108
+ while (b_matched[k] == 0U) {
109
+ ++k;
110
+ }
111
+ if (a[i] != b[k]) {
112
+ ++half_trans;
113
+ }
114
+ ++k;
115
+ }
116
+
117
+ // half_trans / 2 with INTEGER division — matches rapidfuzz exactly.
118
+ // When the match permutation contains a cycle of odd length (say a 3-
119
+ // cycle u→t→k→u), half_trans is odd (3 here), and the standard Jaro
120
+ // formula's "transpositions = matches_out_of_order / 2" underestimates
121
+ // the true transposition count from the cycle decomposition. The
122
+ // literature and every common implementation accept that rounding-
123
+ // down convention; matching it keeps our values bit-equivalent to
124
+ // rapidfuzz's so users can swap one for the other.
125
+ const double matches_d = static_cast<double>(matches);
126
+ const double trans_d = static_cast<double>(half_trans / 2U);
127
+ return (matches_d / static_cast<double>(n)
128
+ + matches_d / static_cast<double>(m)
129
+ + (matches_d - trans_d) / matches_d)
130
+ / 3.0;
131
+ }
132
+
133
+ // ---------------------------------------------------------------------
134
+ // Bit-parallel Jaro (single 64-bit word per side).
135
+ //
136
+ // Replaces the O(n * window) scalar inner loop with an iteration over
137
+ // `b` where each step is an O(1) bitwise update: for j-th char of b,
138
+ // `peq[b[j]] & window_mask & ~used_a` is the set of unused matching
139
+ // positions in a; the lowest set bit picks the leftmost. After the
140
+ // scan, `used_a` and `b_matched` are popcount-able bitmaps, and
141
+ // transpositions fall out of a parallel walk over set bits.
142
+ //
143
+ // Constraint: both n and m must fit in 64 bits. The caller checks
144
+ // this before dispatching here; the scalar reference handles the
145
+ // long-string fallback.
146
+ // ---------------------------------------------------------------------
147
+
148
+ // Build PEQ over `a` for the byte alphabet (256 entries). Bit i of
149
+ // peq[c] is set iff a[i] == c. Returned by value because the caller
150
+ // usually owns it on the stack.
151
+ inline std::array<std::uint64_t, 256> build_peq_byte_64(
152
+ std::span<const std::uint8_t> a) noexcept {
153
+ std::array<std::uint64_t, 256> peq{};
154
+ const std::size_t n = std::min<std::size_t>(a.size(), 64U);
155
+ for (std::size_t i = 0; i < n; ++i) {
156
+ peq[a[i]] |= std::uint64_t{1} << i;
157
+ }
158
+ return peq;
159
+ }
160
+
161
+ // Bits [lo, hi) of a 64-bit word. Caller guarantees 0 <= lo <= hi <= 64.
162
+ inline constexpr std::uint64_t bit_range(std::size_t lo, std::size_t hi) noexcept {
163
+ // Build the [0, hi) half as a right-shift of all-ones; that
164
+ // sidesteps the UB of `1ULL << 64`. The empty range hi==0 collapses
165
+ // to zero because the high_bits term becomes ~0 >> 64 (= 0 under
166
+ // unsigned arithmetic via the special case below).
167
+ const std::uint64_t high_bits =
168
+ (hi >= 64U) ? ~std::uint64_t{0} : (~std::uint64_t{0} >> (64U - hi));
169
+ const std::uint64_t low_bits = (lo == 0U) ? 0U : ((std::uint64_t{1} << lo) - 1U);
170
+ return high_bits & ~low_bits;
171
+ }
172
+
173
+ // One-shot bit-parallel Jaro for byte inputs with n <= 64 and m <= 64.
174
+ // Returns the (already-divided-by-3) similarity in [0, 1].
175
+ inline double jaro_bp_byte_64(
176
+ std::span<const std::uint8_t> a,
177
+ std::span<const std::uint8_t> b) noexcept {
178
+ const std::size_t n = a.size();
179
+ const std::size_t m = b.size();
180
+ if (n == 0U && m == 0U) {
181
+ return 1.0;
182
+ }
183
+ if (n == 0U || m == 0U) {
184
+ return 0.0;
185
+ }
186
+
187
+ const std::size_t window = match_window(n, m);
188
+ const auto peq = build_peq_byte_64(a);
189
+
190
+ std::uint64_t used_a = 0;
191
+ std::uint64_t b_matched = 0;
192
+ for (std::size_t j = 0; j < m; ++j) {
193
+ const std::size_t lo = j > window ? j - window : 0U;
194
+ const std::size_t hi = std::min(j + window + 1U, n);
195
+ if (lo >= hi) {
196
+ continue;
197
+ }
198
+ const std::uint64_t window_mask = bit_range(lo, hi);
199
+ const std::uint64_t candidate =
200
+ peq[b[j]] & window_mask & ~used_a;
201
+ if (candidate == 0U) {
202
+ continue;
203
+ }
204
+ // Lowest set bit (= leftmost unused match position in a).
205
+ const std::uint64_t lowest = candidate & (~candidate + 1U);
206
+ used_a |= lowest;
207
+ b_matched |= std::uint64_t{1} << j;
208
+ }
209
+
210
+ const std::size_t matches =
211
+ static_cast<std::size_t>(std::popcount(used_a));
212
+ if (matches == 0U) {
213
+ return 0.0;
214
+ }
215
+
216
+ // Transposition count: walk set bits of used_a and b_matched in
217
+ // ascending order and count mismatched pairs.
218
+ std::uint64_t a_bits = used_a;
219
+ std::uint64_t b_bits = b_matched;
220
+ std::size_t half_trans = 0;
221
+ while (a_bits != 0U) {
222
+ const std::size_t i = static_cast<std::size_t>(std::countr_zero(a_bits));
223
+ const std::size_t k = static_cast<std::size_t>(std::countr_zero(b_bits));
224
+ if (a[i] != b[k]) {
225
+ ++half_trans;
226
+ }
227
+ a_bits &= a_bits - 1U;
228
+ b_bits &= b_bits - 1U;
229
+ }
230
+
231
+ const double matches_d = static_cast<double>(matches);
232
+ const double trans_d = static_cast<double>(half_trans / 2U);
233
+ return (matches_d / static_cast<double>(n)
234
+ + matches_d / static_cast<double>(m)
235
+ + (matches_d - trans_d) / matches_d)
236
+ / 3.0;
237
+ }
238
+
239
+ // Arbitrary-length bit-parallel Jaro for byte inputs (no 64-char cap).
240
+ // A faithful bit-parallel transcription of jaro_scalar: it iterates `a`
241
+ // and, for each a[i], takes the leftmost not-yet-used position in b's
242
+ // window — the identical greedy, so the result is bit-for-bit equal to
243
+ // jaro_scalar at every length. The per-a-char match step is O(ceil(m/64))
244
+ // word ops instead of O(window) scalar compares, making the whole
245
+ // function O(n * ceil(m/64)) ~ O(n*m/64): 64x cheaper than the scalar
246
+ // O(n*window) path it replaces, and with no length cliff. PEQ is built
247
+ // over `b`; the W = ceil(m/64) word count is a runtime value.
248
+ inline double jaro_bp_byte_multiword(
249
+ std::span<const std::uint8_t> a,
250
+ std::span<const std::uint8_t> b) {
251
+ const std::size_t n = a.size();
252
+ const std::size_t m = b.size();
253
+ if (n == 0U && m == 0U) {
254
+ return 1.0;
255
+ }
256
+ if (n == 0U || m == 0U) {
257
+ return 0.0;
258
+ }
259
+
260
+ const std::size_t window = match_window(n, m);
261
+ const std::size_t wb = (m + 63U) / 64U; // words spanning b's positions
262
+ const std::size_t wa = (n + 63U) / 64U; // words spanning a's positions
263
+
264
+ // Thread-local scratch — amortise 256*wb PEQ + bitmap allocations
265
+ // across long-string Jaro calls (same pattern as multi-word Indel).
266
+ struct MultiWordJaroScratch {
267
+ std::vector<std::uint64_t> peq;
268
+ std::vector<std::uint64_t> used_b;
269
+ std::vector<std::uint64_t> a_matched;
270
+ std::vector<std::uint8_t> peq_dirty;
271
+ std::array<std::uint32_t, 256> peq_touch{};
272
+ std::uint32_t peq_gen = 0;
273
+ std::size_t peq_layout_wb = 0;
274
+ };
275
+ thread_local MultiWordJaroScratch scr;
276
+
277
+ if (scr.peq.size() < 256U * wb) {
278
+ scr.peq.assign(256U * wb, 0U);
279
+ scr.peq_dirty.clear();
280
+ scr.peq_layout_wb = wb;
281
+ scr.peq_touch.fill(0);
282
+ scr.peq_gen = 0;
283
+ } else if (scr.peq_layout_wb != wb) {
284
+ std::fill_n(scr.peq.data(), 256U * wb, std::uint64_t{0});
285
+ scr.peq_dirty.clear();
286
+ scr.peq_layout_wb = wb;
287
+ } else {
288
+ for (const std::uint8_t s : scr.peq_dirty) {
289
+ std::fill_n(
290
+ scr.peq.data() + static_cast<std::size_t>(s) * wb, wb,
291
+ std::uint64_t{0});
292
+ }
293
+ scr.peq_dirty.clear();
294
+ }
295
+ if (++scr.peq_gen == 0U) {
296
+ scr.peq_touch.fill(0);
297
+ scr.peq_gen = 1U;
298
+ }
299
+
300
+ // PEQ over b: bit j of peq row b[j] is set. 256 rows of `wb` words.
301
+ for (std::size_t j = 0; j < m; ++j) {
302
+ const std::uint8_t c = b[j];
303
+ if (scr.peq_touch[c] != scr.peq_gen) {
304
+ scr.peq_touch[c] = scr.peq_gen;
305
+ scr.peq_dirty.push_back(c);
306
+ }
307
+ scr.peq[static_cast<std::size_t>(c) * wb + (j >> 6U)] |=
308
+ std::uint64_t{1} << (j & 63U);
309
+ }
310
+
311
+ if (scr.used_b.size() < wb) scr.used_b.resize(wb);
312
+ if (scr.a_matched.size() < wa) scr.a_matched.resize(wa);
313
+ std::fill_n(scr.used_b.data(), wb, std::uint64_t{0});
314
+ std::fill_n(scr.a_matched.data(), wa, std::uint64_t{0});
315
+ std::size_t matches = 0U;
316
+
317
+ for (std::size_t i = 0; i < n; ++i) {
318
+ const std::size_t lo = i > window ? i - window : 0U;
319
+ const std::size_t hi = std::min(i + window + 1U, m);
320
+ if (lo >= hi) {
321
+ continue;
322
+ }
323
+ const std::uint64_t* row =
324
+ scr.peq.data() + static_cast<std::size_t>(a[i]) * wb;
325
+ // Leftmost unused b-position in [lo, hi) matching a[i]: scan words
326
+ // low->high; the first word holding a candidate has the lowest
327
+ // position, and its lowest set bit is the leftmost one.
328
+ const std::size_t w_lo = lo >> 6U;
329
+ const std::size_t w_hi = (hi - 1U) >> 6U;
330
+ for (std::size_t w = w_lo; w <= w_hi; ++w) {
331
+ const std::size_t base = w * 64U;
332
+ const std::size_t blo = lo > base ? lo - base : 0U;
333
+ const std::size_t bhi = std::min<std::size_t>(64U, hi - base);
334
+ const std::uint64_t cand =
335
+ row[w] & bit_range(blo, bhi) & ~scr.used_b[w];
336
+ if (cand != 0U) {
337
+ scr.used_b[w] |= cand & (~cand + 1U);
338
+ scr.a_matched[i >> 6U] |= std::uint64_t{1} << (i & 63U);
339
+ ++matches;
340
+ break;
341
+ }
342
+ }
343
+ }
344
+
345
+ if (matches == 0U) {
346
+ return 0.0;
347
+ }
348
+
349
+ // Transposition: lockstep ascending walk over the matched positions of
350
+ // a (a_matched) and b (used_b), counting pairs whose chars differ.
351
+ std::size_t half_trans = 0U;
352
+ std::size_t wa_i = 0U;
353
+ std::size_t wb_i = 0U;
354
+ std::uint64_t a_bits = scr.a_matched[0];
355
+ std::uint64_t b_bits = scr.used_b[0];
356
+ for (std::size_t c = 0; c < matches; ++c) {
357
+ while (a_bits == 0U) {
358
+ a_bits = scr.a_matched[++wa_i];
359
+ }
360
+ while (b_bits == 0U) {
361
+ b_bits = scr.used_b[++wb_i];
362
+ }
363
+ const std::size_t i =
364
+ wa_i * 64U + static_cast<std::size_t>(std::countr_zero(a_bits));
365
+ const std::size_t k =
366
+ wb_i * 64U + static_cast<std::size_t>(std::countr_zero(b_bits));
367
+ if (a[i] != b[k]) {
368
+ ++half_trans;
369
+ }
370
+ a_bits &= a_bits - 1U;
371
+ b_bits &= b_bits - 1U;
372
+ }
373
+
374
+ const double matches_d = static_cast<double>(matches);
375
+ const double trans_d = static_cast<double>(half_trans / 2U);
376
+ return (matches_d / static_cast<double>(n)
377
+ + matches_d / static_cast<double>(m)
378
+ + (matches_d - trans_d) / matches_d)
379
+ / 3.0;
380
+ }
381
+
382
+ // Arbitrary-length, arbitrary-alphabet bit-parallel Jaro for wider tokens
383
+ // (UCS-2 / UCS-4 / object sequences). Same a-outer greedy as jaro_scalar
384
+ // — bit-identical at every length — but the PEQ is a hash map keyed by
385
+ // token value (the alphabet is sparse and can be huge, so a dense table
386
+ // is out). O(n * ceil(m/64)) with O(1) amortized map lookups; replaces
387
+ // the O(n*window) scalar reference on the wide-token path.
388
+ template <typename Token>
389
+ inline double jaro_bp_token_multiword(
390
+ std::span<const Token> a,
391
+ std::span<const Token> b) {
392
+ const std::size_t n = a.size();
393
+ const std::size_t m = b.size();
394
+ if (n == 0U && m == 0U) {
395
+ return 1.0;
396
+ }
397
+ if (n == 0U || m == 0U) {
398
+ return 0.0;
399
+ }
400
+
401
+ const std::size_t window = match_window(n, m);
402
+ const std::size_t wb = (m + 63U) / 64U;
403
+ const std::size_t wa = (n + 63U) / 64U;
404
+
405
+ // PEQ over b, keyed by token value: row[token] has bit j set for each
406
+ // j where b[j] == token. `wb` words per distinct token.
407
+ std::unordered_map<Token, std::vector<std::uint64_t>> peq;
408
+ for (std::size_t j = 0; j < m; ++j) {
409
+ auto& row = peq[b[j]];
410
+ if (row.empty()) {
411
+ row.assign(wb, 0U);
412
+ }
413
+ row[j >> 6U] |= std::uint64_t{1} << (j & 63U);
414
+ }
415
+
416
+ std::vector<std::uint64_t> used_b(wb, 0U);
417
+ std::vector<std::uint64_t> a_matched(wa, 0U);
418
+ std::size_t matches = 0U;
419
+
420
+ for (std::size_t i = 0; i < n; ++i) {
421
+ const std::size_t lo = i > window ? i - window : 0U;
422
+ const std::size_t hi = std::min(i + window + 1U, m);
423
+ if (lo >= hi) {
424
+ continue;
425
+ }
426
+ const auto it = peq.find(a[i]);
427
+ if (it == peq.end()) {
428
+ continue;
429
+ }
430
+ const std::uint64_t* row = it->second.data();
431
+ const std::size_t w_lo = lo >> 6U;
432
+ const std::size_t w_hi = (hi - 1U) >> 6U;
433
+ for (std::size_t w = w_lo; w <= w_hi; ++w) {
434
+ const std::size_t base = w * 64U;
435
+ const std::size_t blo = lo > base ? lo - base : 0U;
436
+ const std::size_t bhi = std::min<std::size_t>(64U, hi - base);
437
+ const std::uint64_t cand = row[w] & bit_range(blo, bhi) & ~used_b[w];
438
+ if (cand != 0U) {
439
+ used_b[w] |= cand & (~cand + 1U);
440
+ a_matched[i >> 6U] |= std::uint64_t{1} << (i & 63U);
441
+ ++matches;
442
+ break;
443
+ }
444
+ }
445
+ }
446
+
447
+ if (matches == 0U) {
448
+ return 0.0;
449
+ }
450
+
451
+ std::size_t half_trans = 0U;
452
+ std::size_t wa_i = 0U;
453
+ std::size_t wb_i = 0U;
454
+ std::uint64_t a_bits = a_matched[0];
455
+ std::uint64_t b_bits = used_b[0];
456
+ for (std::size_t c = 0; c < matches; ++c) {
457
+ while (a_bits == 0U) {
458
+ a_bits = a_matched[++wa_i];
459
+ }
460
+ while (b_bits == 0U) {
461
+ b_bits = used_b[++wb_i];
462
+ }
463
+ const std::size_t i =
464
+ wa_i * 64U + static_cast<std::size_t>(std::countr_zero(a_bits));
465
+ const std::size_t k =
466
+ wb_i * 64U + static_cast<std::size_t>(std::countr_zero(b_bits));
467
+ if (a[i] != b[k]) {
468
+ ++half_trans;
469
+ }
470
+ a_bits &= a_bits - 1U;
471
+ b_bits &= b_bits - 1U;
472
+ }
473
+
474
+ const double matches_d = static_cast<double>(matches);
475
+ const double trans_d = static_cast<double>(half_trans / 2U);
476
+ return (matches_d / static_cast<double>(n)
477
+ + matches_d / static_cast<double>(m)
478
+ + (matches_d - trans_d) / matches_d)
479
+ / 3.0;
480
+ }
481
+
482
+ // Common-prefix length up to `cap` chars (inclusive of cap). Used by
483
+ // the Jaro-Winkler bonus; the standard literature uses cap = 4.
484
+ template <typename Token>
485
+ inline std::size_t common_prefix(
486
+ std::span<const Token> a,
487
+ std::span<const Token> b,
488
+ std::size_t cap) noexcept {
489
+ const std::size_t limit = std::min({a.size(), b.size(), cap});
490
+ std::size_t i = 0;
491
+ while (i < limit && a[i] == b[i]) {
492
+ ++i;
493
+ }
494
+ return i;
495
+ }
496
+
497
+ // Jaro-Winkler similarity in [0, 1]. If the base Jaro score is at least
498
+ // `prefix_threshold`, add `L * prefix_weight * (1 - jaro)` where L is
499
+ // the common prefix length (capped at `prefix_cap`). Otherwise return
500
+ // the base Jaro score unchanged — this matches rapidfuzz's behavior
501
+ // (where below threshold no bonus is applied).
502
+ template <typename Token>
503
+ inline double jaro_winkler_scalar(
504
+ std::span<const Token> a,
505
+ std::span<const Token> b,
506
+ double prefix_weight = kDefaultPrefixWeight,
507
+ double prefix_threshold = kDefaultPrefixThreshold,
508
+ std::size_t prefix_cap = kDefaultPrefixCap) {
509
+ const double jaro = jaro_scalar<Token>(a, b);
510
+ if (jaro < prefix_threshold) {
511
+ return jaro;
512
+ }
513
+ const std::size_t L = common_prefix<Token>(a, b, prefix_cap);
514
+ return jaro + static_cast<double>(L) * prefix_weight * (1.0 - jaro);
515
+ }
516
+
517
+ } // namespace stride_align::jaro
@@ -0,0 +1,159 @@
1
+ #pragma once
2
+
3
+ // Longest Common Subsequence (LCS) and Longest Common Substring (LCSubstr).
4
+ //
5
+ // Two distinct algorithms with confusable names:
6
+ //
7
+ // * **Subsequence** — characters need not be contiguous. ``ABCBDAB`` and
8
+ // ``BDCAB`` have LCS ``BCAB`` (length 4). The recurrence is the same
9
+ // one Indel distance uses (``indel = |a| + |b| - 2·LCS``).
10
+ // * **Substring** — characters must be contiguous. The same two
11
+ // strings have LCSubstr ``AB`` (length 2). Different DP:
12
+ // ``dp[i][j] = dp[i-1][j-1] + 1`` on match, ``0`` on mismatch.
13
+ //
14
+ // Both run in ``O(m·n)`` time with two rolling rows for ``O(min(m,n))``
15
+ // space. The substring DP additionally tracks the running maximum and
16
+ // the position at which it occurred so the substring itself can be
17
+ // recovered by a single slice (no traceback table).
18
+ //
19
+ // Public API takes ``std::vector<Codepoint>`` — the dispatch wrapper
20
+ // widens Python ``str`` storage straight out of ``PyUnicode_DATA``
21
+ // into codepoints, same convention as the other stride-align entry
22
+ // points that work in codepoint space.
23
+ //
24
+ // Source: textbook dynamic programming (Hirschberg 1975 and Wagner-
25
+ // Fischer 1974). The C++ here is original.
26
+
27
+ #include <algorithm>
28
+ #include <cstddef>
29
+ #include <cstdint>
30
+ #include <utility>
31
+ #include <vector>
32
+
33
+ namespace stride_align::lcs {
34
+
35
+ using Codepoint = std::uint32_t;
36
+
37
+ // Length of the longest common SUBSEQUENCE of ``a`` and ``b``.
38
+ //
39
+ // The two-row DP is standard: ``dp[i][j]`` is the LCS length of the
40
+ // first ``i`` characters of ``a`` and the first ``j`` characters of
41
+ // ``b``. The recurrence is ``dp[i-1][j-1] + 1`` on a character match
42
+ // and ``max(dp[i-1][j], dp[i][j-1])`` otherwise. We index ``a`` along
43
+ // the columns (shorter side) so the rows are as short as possible.
44
+ inline std::size_t lcs_length(const std::vector<Codepoint>& a,
45
+ const std::vector<Codepoint>& b) {
46
+ if (a.empty() || b.empty()) return 0;
47
+ // Put the shorter sequence on the column axis so the rolling rows
48
+ // hold ``min(|a|, |b|) + 1`` cells.
49
+ const auto& cols = a.size() <= b.size() ? a : b;
50
+ const auto& rows = a.size() <= b.size() ? b : a;
51
+ const std::size_t M = cols.size();
52
+ // Thread-local rolling rows — Ratcliff-Obershelp calls LCS/substring
53
+ // DP many times; amortise allocation across those passes.
54
+ thread_local std::vector<std::size_t> prev;
55
+ thread_local std::vector<std::size_t> curr;
56
+ if (prev.size() < M + 1U) {
57
+ prev.resize(M + 1U);
58
+ curr.resize(M + 1U);
59
+ }
60
+ std::fill_n(prev.data(), M + 1U, std::size_t{0});
61
+ for (std::size_t i = 1; i <= rows.size(); ++i) {
62
+ curr[0] = 0;
63
+ for (std::size_t j = 1; j <= M; ++j) {
64
+ if (rows[i - 1] == cols[j - 1]) {
65
+ curr[j] = prev[j - 1] + 1;
66
+ } else {
67
+ curr[j] = std::max(prev[j], curr[j - 1]);
68
+ }
69
+ }
70
+ prev.swap(curr);
71
+ }
72
+ return prev[M];
73
+ }
74
+
75
+ // Result of the substring DP: ``length`` is the LCSubstr length;
76
+ // ``end_a`` and ``end_b`` are the one-past-the-last indices of an
77
+ // occurrence of that substring in ``a`` and ``b`` respectively, so
78
+ // ``a[end_a - length .. end_a) == b[end_b - length .. end_b)``. When
79
+ // the inputs share no character, all three fields are 0.
80
+ //
81
+ // Tiebreak when several substrings achieve the maximum length:
82
+ // smallest ``end_a`` first (earliest occurrence in ``a``), then
83
+ // smallest ``end_b`` — matches Python's
84
+ // ``difflib.SequenceMatcher.find_longest_match`` convention.
85
+ struct LcsSubstringInfo {
86
+ std::size_t length = 0;
87
+ std::size_t end_a = 0;
88
+ std::size_t end_b = 0;
89
+ };
90
+
91
+ // Length and end-positions of the longest common SUBSTRING over the
92
+ // half-open ranges ``a[a_lo, a_hi)`` and ``b[b_lo, b_hi)``. Returned
93
+ // positions are absolute indices into the underlying vectors.
94
+ //
95
+ // The recurrence ``dp[i][j] = dp[i-1][j-1] + 1`` on match and ``0``
96
+ // on mismatch lets us collapse to two rolling rows along the ``b``
97
+ // axis. The running maximum is tracked so the substring's bounds in
98
+ // both inputs can be recovered with two slices.
99
+ inline LcsSubstringInfo lcs_substring_info_range(
100
+ const std::vector<Codepoint>& a, std::size_t a_lo, std::size_t a_hi,
101
+ const std::vector<Codepoint>& b, std::size_t b_lo, std::size_t b_hi) {
102
+ LcsSubstringInfo r;
103
+ if (a_hi <= a_lo || b_hi <= b_lo) return r;
104
+ const std::size_t M = a_hi - a_lo;
105
+ const std::size_t N = b_hi - b_lo;
106
+ // Separate TL pair from ``lcs_length`` so nested/overlapping use is
107
+ // safe if both ever share a call stack.
108
+ thread_local std::vector<std::size_t> prev;
109
+ thread_local std::vector<std::size_t> curr;
110
+ if (prev.size() < N + 1U) {
111
+ prev.resize(N + 1U);
112
+ curr.resize(N + 1U);
113
+ }
114
+ std::fill_n(prev.data(), N + 1U, std::size_t{0});
115
+ for (std::size_t i = 1; i <= M; ++i) {
116
+ curr[0] = 0;
117
+ for (std::size_t j = 1; j <= N; ++j) {
118
+ if (a[a_lo + i - 1] == b[b_lo + j - 1]) {
119
+ curr[j] = prev[j - 1] + 1;
120
+ if (curr[j] > r.length) {
121
+ r.length = curr[j];
122
+ r.end_a = a_lo + i;
123
+ r.end_b = b_lo + j;
124
+ }
125
+ } else {
126
+ curr[j] = 0;
127
+ }
128
+ }
129
+ prev.swap(curr);
130
+ }
131
+ return r;
132
+ }
133
+
134
+ // Convenience overload that scans the whole of both inputs.
135
+ inline LcsSubstringInfo lcs_substring_info(
136
+ const std::vector<Codepoint>& a,
137
+ const std::vector<Codepoint>& b) {
138
+ return lcs_substring_info_range(a, 0, a.size(), b, 0, b.size());
139
+ }
140
+
141
+ // Length of the longest common SUBSTRING.
142
+ inline std::size_t lcs_substring_length(const std::vector<Codepoint>& a,
143
+ const std::vector<Codepoint>& b) {
144
+ return lcs_substring_info(a, b).length;
145
+ }
146
+
147
+ // The longest common SUBSTRING itself, as a codepoint vector taken
148
+ // from ``a``. When multiple substrings tie at the maximum length, the
149
+ // FIRST occurrence in ``a`` (smallest ``end_a``) is returned —
150
+ // matching ``std::search`` / ``str.find`` conventions.
151
+ inline std::vector<Codepoint> lcs_substring(const std::vector<Codepoint>& a,
152
+ const std::vector<Codepoint>& b) {
153
+ const auto info = lcs_substring_info(a, b);
154
+ if (info.length == 0) return {};
155
+ return std::vector<Codepoint>(a.begin() + info.end_a - info.length,
156
+ a.begin() + info.end_a);
157
+ }
158
+
159
+ } // namespace stride_align::lcs