phonetics 3.2.0 → 4.0.0

data/ext/phonetics_ruby/vendor/phonetics/src/cross_class.rs

@@ -0,0 +1,56 @@
+//! Cross-class bridge: consonant ↔ vowel distance.
+//!
+//! English perception treats /j/, /w/, /ɹ/, /ɰ/ as non-syllabic versions
+//! of /i/, /u/, /ɝ/, /ɯ/ respectively — Mad Gab's "yes" ≈ "Es" depends
+//! on this bridge. Glottals are mostly an air pulse, also nearer to
+//! vowels than to a true stop or fricative. Everything else uses
+//! [`CROSS_CLASS_DEFAULT`](crate::symbols::CROSS_CLASS_DEFAULT).
+
+use crate::symbols::{CROSS_CLASS_DEFAULT, CROSS_CLASS_NEAR_BRIDGE};
+
+/// Returns the bridge cost when `consonant` is an approximant with
+/// listed vowel-distance entries. Vowels not specifically listed under
+/// a bridge consonant get `CROSS_CLASS_NEAR_BRIDGE`. Non-bridge
+/// consonants return `None`.
+fn approximant_bridge(consonant: &str, vowel: &str) -> Option<f64> {
+    let cost = match (consonant, vowel) {
+        ("j", "i") => 0.10,
+        ("j", "ɪ") => 0.14,
+        ("j", "y") => 0.18,
+        ("j", "e") => 0.22,
+        ("w", "u") => 0.10,
+        ("w", "ʊ") => 0.14,
+        ("w", "o") => 0.22,
+        ("w", "ɔ") => 0.30,
+        ("w", "ɯ") => 0.20,
+        ("ɹ", "ɝ") => 0.08,
+        ("ɹ", "ə") => 0.25,
+        ("ɰ", "ɯ") => 0.10,
+        ("ɰ", "u") => 0.20,
+        // Bridge consonants without a specific vowel entry.
+        ("j" | "w" | "ɹ" | "ɰ", _) => CROSS_CLASS_NEAR_BRIDGE,
+        _ => return None,
+    };
+    Some(cost)
+}
+
+/// Glottal-bridge cost; glottals are nearly vowel-like everywhere.
+fn glottal_bridge(consonant: &str) -> Option<f64> {
+    Some(match consonant {
+        "h" | "ɦ" => 0.50,
+        "ʔ"       => 0.55,
+        _ => return None,
+    })
+}
+
+/// Look up the cross-class distance for a (consonant, vowel) pair.
+/// Falls back to `CROSS_CLASS_DEFAULT` when neither bridge fires.
+pub fn distance(consonant: &str, vowel: &str) -> f64 {
+    if let Some(c) = approximant_bridge(consonant, vowel) {
+        return c;
+    }
+    if let Some(c) = glottal_bridge(consonant) {
+        return c;
+    }
+    CROSS_CLASS_DEFAULT
+}

data/ext/phonetics_ruby/vendor/phonetics/src/diacritics.rs

@@ -0,0 +1,113 @@
+//! Suprasegmental and modifier diacritics.
+//!
+//! Each diacritic character either attaches to the preceding base
+//! phoneme (length, aspiration, palatalization, etc.) or to the
+//! following base phoneme (stress marks, which in IPA precede the
+//! stressed syllable). The tokenizer absorbs them into the same token;
+//! the distance metric splits them back out via [`decompose`] and
+//! charges a small additive cost when modifier sets differ.
+
+/// Kinds of suprasegmental modifier recognised by the metric.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum Diacritic {
+    /// `ː` — full length.
+    Long,
+    /// `ˑ` — half-length.
+    HalfLong,
+    /// `ʰ` — aspiration.
+    Aspirated,
+    /// `ʲ` — palatalization.
+    Palatalized,
+    /// `ˤ` — pharyngealization.
+    Pharyngealized,
+    /// `ˠ` — velarization.
+    Velarized,
+    /// Combining tilde above (U+0303) — nasalization.
+    Nasalized,
+    /// `ˈ` — primary stress.
+    PrimaryStress,
+    /// `ˌ` — secondary stress.
+    SecondaryStress,
+}
+
+impl Diacritic {
+    /// Additive cost when this modifier is present on one phoneme but
+    /// not the other.
+    pub fn penalty(self) -> f64 {
+        match self {
+            Self::Long             => 0.05,
+            Self::HalfLong         => 0.025,
+            Self::Aspirated        => 0.04,
+            Self::Palatalized      => 0.06,
+            Self::Pharyngealized   => 0.07,
+            Self::Velarized        => 0.07,
+            Self::Nasalized        => 0.06,
+            Self::PrimaryStress    => 0.05,
+            Self::SecondaryStress  => 0.03,
+        }
+    }
+
+    /// Classify a single Unicode character as a diacritic. The combining
+    /// tilde is a multi-byte sequence in UTF-8 but one Unicode scalar.
+    pub fn from_char(c: char) -> Option<Self> {
+        Some(match c {
+            'ː' => Self::Long,
+            'ˑ' => Self::HalfLong,
+            'ʰ' => Self::Aspirated,
+            'ʲ' => Self::Palatalized,
+            'ˤ' => Self::Pharyngealized,
+            'ˠ' => Self::Velarized,
+            '\u{0303}' => Self::Nasalized,
+            'ˈ' => Self::PrimaryStress,
+            'ˌ' => Self::SecondaryStress,
+            _ => return None,
+        })
+    }
+
+    /// True if this diacritic attaches to the *following* segment
+    /// (stress marks); false for the preceding-attached ones.
+    pub fn is_leading(self) -> bool {
+        matches!(self, Self::PrimaryStress | Self::SecondaryStress)
+    }
+}
+
+/// Generic fallback diacritic cost when an unrecognised diacritic-like
+/// character ends up in a modifier set somehow.
+pub const DEFAULT_PENALTY: f64 = 0.03;
+
+/// Split a phoneme token into its base symbol and the set of diacritic
+/// kinds it carries. Unrecognised characters stay in the base so a
+/// misspelt input doesn't silently lose information.
+pub fn decompose(token: &str) -> (String, Vec<Diacritic>) {
+    let mut base = String::new();
+    let mut mods: Vec<Diacritic> = Vec::new();
+    for c in token.chars() {
+        if let Some(d) = Diacritic::from_char(c) {
+            if !mods.contains(&d) {
+                mods.push(d);
+            }
+        } else {
+            base.push(c);
+        }
+    }
+    (base, mods)
+}
+
+/// Symmetric-difference cost between two diacritic sets.
+pub fn distance(mods1: &[Diacritic], mods2: &[Diacritic]) -> f64 {
+    if mods1.is_empty() && mods2.is_empty() {
+        return 0.0;
+    }
+    let mut total = 0.0;
+    for d in mods1 {
+        if !mods2.contains(d) {
+            total += d.penalty();
+        }
+    }
+    for d in mods2 {
+        if !mods1.contains(d) {
+            total += d.penalty();
+        }
+    }
+    total
+}

data/ext/phonetics_ruby/vendor/phonetics/src/distance.rs

@@ -0,0 +1,183 @@
+//! Top-level distance dispatch.
+//!
+//! `distance(p1, p2)` returns the acoustic distance between two phoneme
+//! tokens, scaled to [0, 1]. The dispatch order matters:
+//!
+//!   1. Identity → 0.
+//!   2. Boundary token (`#`) → 0 vs boundary, else `BOUNDARY_VS_PHONEME`.
+//!   3. Diacritic decomposition: split the token into base + modifier
+//!      set; recurse on the base; add a per-modifier mismatch cost.
+//!   4. Compound expansion: if either side is a diphthong or affricate,
+//!      compare component-by-component (padded by repeating the last
+//!      segment), averaging.
+//!   5. Class-based dispatch: vowel-vowel, consonant-consonant, or
+//!      cross-class bridge.
+
+use crate::{compounds, consonants, cross_class, diacritics, symbols, vowels};
+
+/// Distance between two phoneme tokens, scaled to [0, 1].
+pub fn distance(p1: &str, p2: &str) -> f64 {
+    if p1 == p2 {
+        return 0.0;
+    }
+
+    if p1 == symbols::BOUNDARY_TOKEN || p2 == symbols::BOUNDARY_TOKEN {
+        return symbols::BOUNDARY_VS_PHONEME;
+    }
+
+    let (base1, mods1) = diacritics::decompose(p1);
+    let (base2, mods2) = diacritics::decompose(p2);
+
+    // If either side carried any diacritics, strip them and recompute
+    // on the bases, then add a per-modifier mismatch cost.
+    if !mods1.is_empty() || !mods2.is_empty() {
+        let base_dist = base_pair_distance(&base1, &base2);
+        return (base_dist + diacritics::distance(&mods1, &mods2)).min(1.0);
+    }
+
+    base_pair_distance(p1, p2)
+}
+
+/// Distance between two bare base phonemes (no diacritics on either
+/// side). Handles compounds, class dispatch, and cross-class bridges.
+fn base_pair_distance(p1: &str, p2: &str) -> f64 {
+    if p1 == p2 {
+        return 0.0;
+    }
+
+    let comp1 = compounds::components(p1);
+    let comp2 = compounds::components(p2);
+    if comp1.is_some() || comp2.is_some() {
+        let a: Vec<&str> = comp1.map_or_else(|| vec![p1], <[&str]>::to_vec);
+        let b: Vec<&str> = comp2.map_or_else(|| vec![p2], <[&str]>::to_vec);
+        return compound_distance(&a, &b);
+    }
+
+    let is_vowel_1 = vowels::lookup(p1).is_some();
+    let is_vowel_2 = vowels::lookup(p2).is_some();
+    let is_cons_1  = consonants::lookup(p1).is_some();
+    let is_cons_2  = consonants::lookup(p2).is_some();
+
+    if is_vowel_1 && is_vowel_2 {
+        return vowels::distance(p1, p2).unwrap_or(1.0);
+    }
+    if is_cons_1 && is_cons_2 {
+        return consonants::distance(p1, p2).unwrap_or(1.0);
+    }
+    if is_cons_1 && is_vowel_2 {
+        return cross_class::distance(p1, p2);
+    }
+    if is_vowel_1 && is_cons_2 {
+        return cross_class::distance(p2, p1);
+    }
+    1.0
+}
+
+/// Pairwise component-mean distance for compound (or
+/// compound-and-simple) phonemes. The shorter side is padded by
+/// repeating its last segment so /aɪ/ vs /a/ charges half a phoneme
+/// distance rather than nothing.
+fn compound_distance(c1: &[&str], c2: &[&str]) -> f64 {
+    let n = c1.len().max(c2.len());
+
+    // Pad the shorter side by repeating its last entry.
+    fn pad<'a>(v: &[&'a str], n: usize) -> Vec<&'a str> {
+        let mut out: Vec<&'a str> = v.to_vec();
+        if let Some(&last) = out.last() {
+            while out.len() < n {
+                out.push(last);
+            }
+        }
+        out
+    }
+    let a = pad(c1, n);
+    let b = pad(c2, n);
+
+    let total: f64 = a
+        .iter()
+        .zip(b.iter())
+        .map(|(x, y)| if x == y { 0.0 } else { distance(x, y) })
+        .sum();
+    total / n as f64
+}
+
+#[cfg(test)]
+mod tests {
+    use super::distance;
+
+    const EPS: f64 = 1e-12;
+
+    #[test]
+    fn matches_ruby_dispatch_cases() {
+        // Reference values produced by the Ruby implementation across
+        // every dispatch branch: cross-class bridges, glottals, the
+        // default cross-class, compound diphthongs and affricates,
+        // compound-vs-simple averaging, diacritics, and the boundary
+        // token.
+        let cases: &[(&str, &str, f64)] = &[
+            // Approximant↔vowel bridge
+            ("j", "i", 0.10),
+            ("j", "ɪ", 0.14),
+            ("w", "u", 0.10),
+            ("w", "o", 0.22),
+            ("ɹ", "ɝ", 0.08),
+            ("ɰ", "ɯ", 0.10),
+            // Glottal bridge
+            ("h", "a", 0.50),
+            ("ʔ", "i", 0.55),
+            ("ɦ", "ɛ", 0.50),
+            // Default cross-class
+            ("k", "i", 0.85),
+            ("s", "u", 0.85),
+            ("m", "a", 0.85),
+            // Diphthongs
+            ("aɪ", "ɑɪ", 0.107_288_248_772_162_7),
+            ("aɪ", "eɪ", 0.130_229_442_812_957_87),
+            ("aɪ", "a",  0.144_904_524_019_594_53),
+            ("oʊ", "o",  0.067_465_504_570_433_46),
+            // Affricates
+            ("tʃ", "ʃ",  0.124_358_541_225_631_43),
+            ("tʃ", "dʒ", 0.15),
+            ("tʃ", "t",  0.124_358_541_225_631_43),
+            ("dʒ", "ʒ",  0.124_358_541_225_631_43),
+            // Compound vs unrelated → averages to the default
+            ("aɪ", "k", 0.85),
+            // Diacritics
+            ("pʰ", "p",  0.04),
+            ("uː", "u",  0.05),
+            ("ˈp", "p",  0.05),
+            ("pʰ", "bʰ", 0.15),
+            // Boundary token
+            ("#", "a", 0.95),
+            ("#", "#", 0.0),
+        ];
+
+        for (a, b, expected) in cases {
+            let got = distance(a, b);
+            assert!(
+                (got - expected).abs() < EPS,
+                "distance({a:?}, {b:?}) = {got}, expected {expected}",
+            );
+        }
+    }
+
+    #[test]
+    fn identity_is_zero_across_classes() {
+        for s in ["i", "p", "aɪ", "tʃ", "pʰ", "#"] {
+            assert_eq!(distance(s, s), 0.0);
+        }
+    }
+
+    #[test]
+    fn diphthong_against_its_nucleus_is_half_a_phoneme() {
+        // /aɪ/ vs /a/ should be roughly distance(ɪ, a) / 2 since the
+        // first component matches and only the second contributes.
+        let direct = crate::vowels::distance("ɪ", "a").unwrap();
+        let compound = distance("aɪ", "a");
+        assert!(
+            (compound - direct / 2.0).abs() < EPS,
+            "compound={compound}, direct/2={}",
+            direct / 2.0
+        );
+    }
+}

data/ext/phonetics_ruby/vendor/phonetics/src/levenshtein.rs

@@ -0,0 +1,146 @@
+//! Strict phonetic edit distance.
+//!
+//! Damerau-Levenshtein DP with INDEL_COST = 1.0 (one inserted or
+//! deleted phoneme costs exactly one indel, regardless of its
+//! neighbours — the operation costs what the operation costs, not what
+//! happens to sit next to it). Adjacent transpositions cost
+//! TRANSPOSE_COST < 2 * INDEL_COST because in casual speech swapping
+//! adjacent phonemes is a real and cheap thing speakers do.
+//!
+//! Substitution cost is the per-phoneme acoustic distance returned by
+//! [`crate::distance`], so improvements to the acoustic model land here
+//! automatically.
+
+use crate::tokenizer;
+
+/// Cost of inserting or deleting one phoneme.
+pub const INDEL_COST: f64 = 1.0;
+
+/// Cost of an adjacent-pair transposition (the Damerau extension).
+pub const TRANSPOSE_COST: f64 = 0.8;
+
+/// Edit distance between two IPA strings under the strict acoustic
+/// metric. Non-IPA characters are tokenised out before the DP runs.
+pub fn distance(a: &str, b: &str) -> f64 {
+    let ta = tokenizer::tokens(a, false);
+    let tb = tokenizer::tokens(b, false);
+    distance_from_tokens(&ta, &tb)
+}
+
+/// Edit distance over pre-tokenised phoneme sequences.
+pub fn distance_from_tokens<S: AsRef<str>>(a: &[S], b: &[S]) -> f64 {
+    let m = a.len();
+    let n = b.len();
+    if m == 0 && n == 0 {
+        return 0.0;
+    }
+
+    // d[i][j] flattened; width = n + 1.
+    let width = n + 1;
+    let mut d = vec![0.0_f64; (m + 1) * width];
+
+    // Seed: matching the empty string against the first i phonemes of a
+    // costs i indels; symmetric for b.
+    for i in 0..=m {
+        d[i * width] = i as f64 * INDEL_COST;
+    }
+    #[allow(clippy::needless_range_loop)]
+    for j in 0..=n {
+        d[j] = j as f64 * INDEL_COST;
+    }
+
+    for i in 1..=m {
+        for j in 1..=n {
+            let ai = a[i - 1].as_ref();
+            let bj = b[j - 1].as_ref();
+            let sub_cost = crate::distance(ai, bj);
+
+            let delete = d[(i - 1) * width + j] + INDEL_COST;
+            let insert = d[i * width + (j - 1)] + INDEL_COST;
+            let substitute = d[(i - 1) * width + (j - 1)] + sub_cost;
+
+            let mut best = delete.min(insert).min(substitute);
+
+            // Damerau adjacent-transposition.
+            if i > 1
+                && j > 1
+                && a[i - 1].as_ref() == b[j - 2].as_ref()
+                && a[i - 2].as_ref() == b[j - 1].as_ref()
+            {
+                let transpose = d[(i - 2) * width + (j - 2)] + TRANSPOSE_COST;
+                if transpose < best {
+                    best = transpose;
+                }
+            }
+
+            d[i * width + j] = best;
+        }
+    }
+
+    d[m * width + n]
+}
+
+#[cfg(test)]
+mod tests {
+    use super::distance;
+
+    const EPS: f64 = 1e-12;
+
+    #[test]
+    fn matches_ruby_reference_distances() {
+        // Reference values produced by Ruby's Phonetics::RubyLevenshtein.
+        let cases: &[(&str, &str, f64)] = &[
+            ("kæt", "kæt", 0.0),
+            ("dɪsug", "ɪsug", 1.0),
+            ("izok", "ɪsug", 0.425_001_067_076_172_47),
+            ("kæt", "", 3.0),
+            ("kæt", "kæɪt", 1.0),
+            ("kæt", "kʌt", 0.145_085_455_502_268_37),
+            ("ɪtsdʒʌstəstupɪdgeɪm",
+             "hɪtsdʒʌstɪsduphɪdkeɪm",
+             2.469_519_814_165_789_5),
+            ("mɔop", "sinkœ", 3.025_788_981_175_774),
+            ("bæd", "ben", 0.510_984_626_268_258_8),
+        ];
+
+        for (a, b, expected) in cases {
+            let got = distance(a, b);
+            assert!(
+                (got - expected).abs() < EPS,
+                "distance({a:?}, {b:?}) = {got}, expected {expected}",
+            );
+        }
+    }
+
+    #[test]
+    fn empty_pair_is_zero() {
+        assert_eq!(distance("", ""), 0.0);
+    }
+
+    #[test]
+    fn one_indel_costs_INDEL_COST() {
+        // Inserting or deleting one phoneme costs exactly the indel.
+        assert!((distance("kæt", "kæte") - super::INDEL_COST).abs() < 1e-6);
+    }
+
+    #[test]
+    fn identity_strings_are_zero() {
+        for s in ["", "kæt", "stupɪdgeɪm", "ɪtsdʒʌstəstupɪdgeɪm"] {
+            assert_eq!(distance(s, s), 0.0);
+        }
+    }
+
+    #[test]
+    fn symmetric() {
+        let pairs = [
+            ("kæt", "kʌt"),
+            ("dɪsug", "ɪsug"),
+            ("stupɪdgeɪm", "stupɪdli"),
+        ];
+        for (a, b) in pairs {
+            let d_ab = distance(a, b);
+            let d_ba = distance(b, a);
+            assert!((d_ab - d_ba).abs() < EPS, "asymmetric: {a}/{b}");
+        }
+    }
+}

data/ext/phonetics_ruby/vendor/phonetics/src/lib.rs

@@ -0,0 +1,44 @@
+//! Phonetics: IPA-based phonetic distance.
+//!
+//! Two-tier API, same as the Ruby reference implementation:
+//!
+//! * [`distance`] — strict per-phoneme acoustic distance, fed to
+//!   [`levenshtein`] for whole-string edit distance. The right call for
+//!   accent clustering, dialect work, and ASR error analysis.
+//!
+//! * [`Confusion::distance`](confusion::Confusion::distance) — listener-
+//!   confusion distance, calibrated against Mad Gab puzzle data. Uses
+//!   Gotoh's affine-gap DP plus a weak-phoneme indel discount and an
+//!   empirical-confusion overlay. The right call for Mad Gab solving,
+//!   pun detection, and mishearing modelling.
+//!
+//! Both tiers share the same per-phoneme cost basis. Improvements to the
+//! acoustic model propagate to both metrics automatically.
+//!
+//! ```
+//! use phonetics::distance;
+//! // Tense /i/ versus lax /ɪ/ — close in Bark space.
+//! assert!((distance("i", "ɪ") - 0.060_056).abs() < 1e-3);
+//! // The same vowel twice is exactly zero.
+//! assert_eq!(distance("ə", "ə"), 0.0);
+//! ```
+
+#![doc(html_root_url = "https://docs.rs/phonetics/0.1.0")]
+
+pub mod compounds;
+pub mod confusion;
+pub mod consonants;
+pub mod cross_class;
+pub mod diacritics;
+pub mod levenshtein;
+pub mod symbols;
+pub mod tokenizer;
+pub mod vowels;
+
+mod distance;
+
+pub use confusion::distance as confusion;
+pub use confusion::similarity;
+pub use distance::distance;
+pub use levenshtein::distance as levenshtein;
+pub use tokenizer::tokens;

data/ext/phonetics_ruby/vendor/phonetics/src/symbols.rs

@@ -0,0 +1,21 @@
+//! Shared phoneme symbol metadata.
+//!
+//! These constants are used by the distance dispatch and the tokenizer.
+
+/// Synthetic phoneme token representing a word boundary. Used by the
+/// Confusion metric to model re-syllabification cheaply.
+pub const BOUNDARY_TOKEN: &str = "#";
+
+/// Cost of substituting a word boundary against a real phoneme. Set high
+/// enough that the Confusion algorithm prefers indeling the boundary
+/// (via the cheap boundary-indel tier) over substituting it.
+pub const BOUNDARY_VS_PHONEME: f64 = 0.95;
+
+/// Default cross-class (consonant↔vowel) distance when no bridge applies.
+/// Lower than 1.0 (the indel cost) on purpose: a consonant against a
+/// vowel is more like a strong substitution than a categorical break.
+pub const CROSS_CLASS_DEFAULT: f64 = 0.85;
+
+/// Cross-class cost when the consonant is in the approximant bridge but
+/// the specific vowel isn't enumerated.
+pub const CROSS_CLASS_NEAR_BRIDGE: f64 = 0.55;