phonetics 3.2.0 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ddee58e3870d471e1d5f03399be12dac1d1a4c9132c28000715796e80914a493
-  data.tar.gz: 502596f055d85a9c7206b5ea8b33fcb18ac9229dd00014551428e6fee498fc01
+  metadata.gz: c174999459702990b87d739037c9fbd2d99d3be2bda047b9b381d9d2432e9670
+  data.tar.gz: 89416b41cf47c0db96186575ac9e133d49e5d81fec15ebec02e864526a0a77af
 SHA512:
-  metadata.gz: 5aef31036df0eb866f036757f024e0c4296c1b958b69f4f1aca901a2c68a63fd84334ae18d1a44ccb8467ed16bdc5069e40d81b5bd1cba22f04da089b490b2dd
-  data.tar.gz: 9bf842a914628aed7fbe0c8cc3ee3ddf49745f35769c11e5650645cae5a59e234edb5c760a8165c2041ff72a516fdf32b7e9230adad418860dfd54e90f4077d6
+  metadata.gz: 6b5bd8339e1e3aba8cbc4599f97588ff046e074832ae84657aba4988d4fe6ef1e06f56a4a8b9c73cb54c130702d459d59257cbdb967396aef9eee8bcd2b12279
+  data.tar.gz: ba950dfbd243de733de5230336a61b6f03a1726371c21112e9765f76d854929084a151d97d8c49d68f593ef7a787c91a82bae70a29becd0297ebc84a7b730e26

data/.gitignore

@@ -10,8 +10,23 @@
 # rspec failure tracking
 .rspec_status
 .ruby-version
-.ruby-version
 *.so
 Gemfile.lock
 *.bundle
-.ruby-version
+*.bundle.dSYM
+
+# Rust extension build artifacts (mkmf-generated)
+ext/phonetics_ruby/Makefile
+ext/phonetics_ruby/Cargo.lock
+ext/phonetics_ruby/target/
+ext/phonetics_ruby/mkmf.log
+
+# Vendored Rust core. Populated by `rake vendor_rust`; shipped in
+# the .gem tarball but not tracked in git (single source of truth
+# lives at <repo>/rust/phonetics).
+ext/phonetics_ruby/vendor/
+
+# Cargo workspace build artifacts (the gem-local workspace at
+# ruby/Cargo.toml puts its target/ at ruby/target).
+target/
+Cargo.lock

data/Cargo.toml

@@ -0,0 +1,27 @@
+# Cargo workspace declaration for the Ruby gem. Exists so that
+# `rb_sys/extensiontask` can find the native extension's manifest by
+# walking up from the gem root, and so cargo resolves the vendored
+# `phonetics` core path dependency.
+#
+# The workspace.package values mirror what the Rust workspace at
+# <repo>/rust/ defines; they're declared here so the vendored copy
+# of the Rust core (which inherits via `.workspace = true`) resolves
+# correctly inside the gem.
+[workspace]
+resolver = "2"
+members = [
+    "ext/phonetics_ruby",
+    "ext/phonetics_ruby/vendor/phonetics",
+]
+exclude = []
+
+[workspace.package]
+version = "0.2.0"
+edition = "2021"
+license = "MIT"
+repository = "https://github.com/JackDanger/phonetics"
+homepage = "https://github.com/JackDanger/phonetics"
+authors = ["Jack Danger <github@jackcanty.com>"]
+
+[workspace.lints.rust]
+unused_must_use = "deny"

data/Rakefile

@@ -1,41 +1,73 @@
 # frozen_string_literal: true
 
+# Rake tasks for the Rust-backed Ruby gem.
+#
+#   rake compile          build the native extension via cargo
+#   rake spec             run rspec
+#   rake build            assemble the source gem (vendor + package)
+#
+# Cross-platform native gem builds use rb-sys-dock; see
+# .github/workflows/release.yml for the published recipe.
+
+require 'fileutils'
 require 'bundler/gem_tasks'
-require 'rake/extensiontask'
-require 'rspec/core/rake_task'
-require 'rubocop/rake_task'
 
-EXT_PATH = 'ext/c_levenshtein'
+# ---------------------------------------------------------------
+# Vendoring of the Rust core into the gem directory.
+#
+# A gem can't ship files from outside its own directory (rubygems
+# policy), and `cargo` can't follow a path dependency that points
+# up into the parent of a `gem install`-extracted source tree. So
+# we keep the canonical Rust core at <repo>/rust/phonetics and
+# copy it into <repo>/ruby/ext/phonetics_ruby/vendor/ for builds
+# that need to ship it (gem-build, gem-install, CI).
+#
+# The vendor copy is git-ignored — single source of truth is the
+# /rust/ tree. It IS bundled into the .gem tarball at build time.
+# ---------------------------------------------------------------
+
+RUST_CORE_SRC    = File.expand_path('../rust/phonetics', __dir__)
+RUST_CORE_VENDOR = File.expand_path('ext/phonetics_ruby/vendor/phonetics', __dir__)
+
+def vendor_rust_core
+  return if File.directory?(RUST_CORE_SRC).nil?
 
-Rake::ExtensionTask.new('c_levenshtein') do |extension|
-  extension.ext_dir = EXT_PATH
-  extension.lib_dir = 'lib/phonetics'
+  unless File.directory?(RUST_CORE_SRC)
+    warn "(skipping vendor: #{RUST_CORE_SRC} not present)" if ENV['VERBOSE']
+    return
+  end
+  FileUtils.rm_rf(RUST_CORE_VENDOR)
+  FileUtils.mkdir_p(File.dirname(RUST_CORE_VENDOR))
+  FileUtils.cp_r(RUST_CORE_SRC, RUST_CORE_VENDOR)
+  # Strip the CLI bin and any stale build artifacts.
+  FileUtils.rm_rf(File.join(RUST_CORE_VENDOR, 'src', 'bin'))
+  FileUtils.rm_rf(File.join(RUST_CORE_VENDOR, 'target'))
 end
 
-PHONETIC_COST_C_EXTENSION = File.join(EXT_PATH, 'phonetic_cost.c')
-NEXT_PHONEME_LENGTH_C_EXTENSION = File.join(EXT_PATH, 'next_phoneme_length.c')
+# The downstream Cargo workspace at ruby/Cargo.toml lists the ext
+# crate as a member, and that crate's Cargo.toml has a path
+# dependency on vendor/phonetics. `cargo metadata` (which
+# `rb_sys/extensiontask` runs at load time) would fail before our
+# tasks even register. Vendor synchronously here, before any
+# require touches cargo.
+vendor_rust_core
 
-require_relative './lib/phonetics/code_generator'
+require 'rb_sys/extensiontask'
+require 'rspec/core/rake_task'
 
-desc 'Write phonetic_cost.c using Phonetic values'
-task PHONETIC_COST_C_EXTENSION do
-  file = File.open(PHONETIC_COST_C_EXTENSION, 'w')
-  Phonetics::CodeGenerator.new(file).generate_phonetic_cost_c_code
-  puts "Wrote #{PHONETIC_COST_C_EXTENSION}"
-end
+desc 'Re-copy the Rust core into ext/phonetics_ruby/vendor/'
+task(:vendor_rust) { vendor_rust_core }
 
-desc 'Write phonemes.c using a lookup table of byte arrays'
-task NEXT_PHONEME_LENGTH_C_EXTENSION do
-  file = File.open(NEXT_PHONEME_LENGTH_C_EXTENSION, 'w')
-  Phonetics::CodeGenerator.new(file).generate_next_phoneme_length_c_code
-  puts "Wrote #{NEXT_PHONEME_LENGTH_C_EXTENSION}"
+GEMSPEC = Gem::Specification.load(File.expand_path('phonetics.gemspec', __dir__))
+
+RbSys::ExtensionTask.new('phonetics_ruby', GEMSPEC) do |ext|
+  ext.lib_dir = 'lib/phonetics'
 end
 
-task compile: PHONETIC_COST_C_EXTENSION
-task compile: NEXT_PHONEME_LENGTH_C_EXTENSION
+# build / compile both depend on a fresh vendor copy.
+task compile: :vendor_rust
+task build:   :vendor_rust
 
 RSpec::Core::RakeTask.new(:spec)
 
-RuboCop::RakeTask.new
-
-task default: [:compile, :rubocop, :spec]
+task default: %i[compile spec]

data/VERSION

@@ -1 +1 @@
-3.2.0
+4.0.0

data/bin/phonetics

@@ -0,0 +1,89 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Phonetics CLI — implementation-agnostic contract.
+#
+# The integration tests in spec/phonetics/cli_spec.rb shell out to this
+# binary (or any other one — set PHONETICS_BIN). What matters is
+# stdin/stdout/exit-code behaviour, not which language fields the
+# Phonetics module. With the Rust port live, this script is a thin
+# layer over the Magnus binding.
+
+$stdout.set_encoding('UTF-8')
+$stderr.set_encoding('UTF-8')
+
+$LOAD_PATH.unshift File.expand_path('../lib', __dir__)
+require 'phonetics'
+
+USAGE = <<~USAGE
+  Usage: phonetics <command> [args...]
+
+  Phrase-level distances (input: IPA strings, possibly with spaces):
+    distance    <ipa1> <ipa2>     Strict phonetic Levenshtein.
+    confusion   <ipa1> <ipa2>     Listener-confusion distance.
+    similarity  <ipa1> <ipa2>     0..1 normalised similarity.
+
+  Phoneme-level distances (input: single phonemes):
+    phoneme         <a> <b>       Acoustic distance.
+    phoneme-conf    <a> <b>       Perceptual distance (with overlay).
+
+  Tokenisation:
+    tokenize    [--boundaries] <ipa>     Phoneme stream, one per line.
+
+  Numeric output is a single line. With --json, output is a JSON object.
+USAGE
+
+def die(message, code = 2)
+  warn message
+  exit code
+end
+
+def format_number(value, json:)
+  return JSON.dump(value: value) if json
+
+  value.to_s
+end
+
+json   = !ARGV.delete('--json').nil?
+bounds = !ARGV.delete('--boundaries').nil?
+
+command, *args = ARGV
+require 'json' if json
+
+case command
+when 'distance'
+  die USAGE unless args.size == 2
+  puts format_number(Phonetics.levenshtein(args[0], args[1]), json: json)
+
+when 'confusion'
+  die USAGE unless args.size == 2
+  puts format_number(Phonetics.confusion(args[0], args[1]), json: json)
+
+when 'similarity'
+  die USAGE unless args.size == 2
+  puts format_number(Phonetics.similarity(args[0], args[1]), json: json)
+
+when 'phoneme'
+  die USAGE unless args.size == 2
+  puts format_number(Phonetics.distance(args[0], args[1]), json: json)
+
+when 'phoneme-conf'
+  die USAGE unless args.size == 2
+  puts format_number(Phonetics.sub_cost(args[0], args[1]), json: json)
+
+when 'tokenize'
+  die USAGE unless args.size == 1
+
+  tokens = Phonetics.tokenize(args[0], boundaries: bounds)
+  if json
+    puts JSON.dump(tokens: tokens)
+  else
+    puts tokens
+  end
+
+when '--help', '-h', nil
+  puts USAGE
+
+else
+  die "phonetics: unknown command #{command.inspect}\n\n#{USAGE}"
+end

data/ext/phonetics_ruby/Cargo.toml

@@ -0,0 +1,36 @@
+# Member of the gem-local workspace at <repo>/ruby/Cargo.toml. The
+# workspace exists purely so `rb_sys` can find this crate via `cargo
+# metadata` from the gem root; the package is otherwise standalone
+# (no shared inheritance from the Rust workspace at <repo>/rust/),
+# which means it builds the same in a checkout and in a `gem
+# install`-extracted source tree.
+[package]
+# Underscored to match the rake extension name and the cdylib that
+# `Init_phonetics_ruby` expects to find. Cargo accepts both forms;
+# this avoids the s/-/_/ rename Cargo otherwise applies to the
+# produced artifact.
+name = "phonetics_ruby"
+description = "Ruby bindings for the phonetics crate, via magnus."
+version = "0.2.0"
+edition = "2021"
+license = "MIT"
+publish = false
+
+[lib]
+# cdylib so Ruby `require` can dlopen it as a native extension.
+crate-type = ["cdylib"]
+
+[dependencies]
+# Vendored copy of the Rust core. Populated by `rake vendor_rust`
+# from <repo>/rust/phonetics; see ruby/Rakefile. The vendor dir is
+# not committed to git but IS shipped in the .gem tarball, so a
+# user running `gem install phonetics` gets a self-contained build.
+#
+# The package on crates.io is `phonetics-rs` (the bare `phonetics`
+# name is taken by an unrelated metaphone library). The `package`
+# alias lets us still write `use phonetics::distance;` in our code.
+phonetics = { path = "vendor/phonetics", package = "phonetics-rs" }
+magnus = "0.8"
+
+[build-dependencies]
+rb-sys-env = "0.2"

data/ext/phonetics_ruby/build.rs

@@ -0,0 +1,24 @@
+fn main() {
+    // rb-sys-env 0.2 reads RUBY_VERSION, but the rb_sys gem's mkmf
+    // Makefile only exports the split form (RBCONFIG_MAJOR/MINOR/TEENY).
+    // Reconstruct it from RBCONFIG_* when needed so both invocation
+    // paths (standalone `cargo build` and gem-extconf-driven `make`)
+    // work without the caller having to know which one to set.
+    if std::env::var_os("RUBY_VERSION").is_none() {
+        if let (Ok(major), Ok(minor), Ok(teeny)) = (
+            std::env::var("RBCONFIG_MAJOR"),
+            std::env::var("RBCONFIG_MINOR"),
+            std::env::var("RBCONFIG_TEENY"),
+        ) {
+            std::env::set_var("RUBY_VERSION", format!("{major}.{minor}.{teeny}"));
+        }
+    }
+
+    let _ = rb_sys_env::activate().expect("phonetics-ruby: cannot locate a Ruby");
+
+    // macOS native-extension linker behaviour: leave Ruby's `rb_*`
+    // symbols unresolved until dlopen time.
+    if cfg!(target_os = "macos") {
+        println!("cargo:rustc-link-arg=-Wl,-undefined,dynamic_lookup");
+    }
+}

data/ext/phonetics_ruby/extconf.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+# Build the Rust-backed native extension that replaces the previous
+# hand-written C codegen.
+#
+# rb_sys/mkmf generates a Makefile that invokes `cargo rustc` against
+# the Cargo.toml sibling to this file. That crate is a workspace
+# member of <repo>/rust/Cargo.toml — the workspace lints, profile, and
+# version come from there, but the build pipeline that produces the
+# Ruby-loadable cdylib lives here where conventional gem tooling
+# expects it.
+require 'mkmf'
+require 'rb_sys/mkmf'
+
+create_rust_makefile('phonetics/phonetics_ruby') do |r|
+  r.profile = :release
+end

data/ext/phonetics_ruby/src/lib.rs

@@ -0,0 +1,56 @@
+//! Magnus bindings around the `phonetics` core crate.
+//!
+//! Loaded into Ruby as `phonetics_ruby.bundle` / `.so`. The
+//! gem-installed library calls `require 'phonetics/phonetics_ruby'`
+//! and gets a populated `Phonetics` module with module functions
+//! matching what the previous hand-written C extension exposed:
+//!
+//!   Phonetics.distance(a, b)              acoustic per-phoneme
+//!   Phonetics.confusion(a, b)             listener-confusion distance
+//!   Phonetics.levenshtein(a, b)           strict edit distance
+//!   Phonetics.similarity(a, b)            normalised 0..1
+//!   Phonetics.sub_cost(a, b)              perceptual per-phoneme
+//!   Phonetics.tokenize(input, boundaries) phoneme stream
+//!
+//! All real work lives in the `phonetics` core crate. This file is
+//! the impedance-matching layer between Rust types and Ruby's ABI.
+
+use magnus::{function, Error, Ruby};
+
+#[magnus::init]
+fn init(ruby: &Ruby) -> Result<(), Error> {
+    let phonetics = ruby.define_module("Phonetics")?;
+
+    phonetics.define_module_function("distance",    function!(distance, 2))?;
+    phonetics.define_module_function("confusion",   function!(confusion, 2))?;
+    phonetics.define_module_function("levenshtein", function!(levenshtein, 2))?;
+    phonetics.define_module_function("similarity",  function!(similarity, 2))?;
+    phonetics.define_module_function("sub_cost",    function!(sub_cost, 2))?;
+    phonetics.define_module_function("_tokenize",   function!(tokenize, 2))?;
+
+    Ok(())
+}
+
+fn distance(a: String, b: String) -> f64 {
+    phonetics::distance(&a, &b)
+}
+
+fn confusion(a: String, b: String) -> f64 {
+    phonetics::confusion(&a, &b)
+}
+
+fn levenshtein(a: String, b: String) -> f64 {
+    phonetics::levenshtein(&a, &b)
+}
+
+fn similarity(a: String, b: String) -> f64 {
+    phonetics::similarity(&a, &b)
+}
+
+fn sub_cost(a: String, b: String) -> f64 {
+    phonetics::confusion::sub_cost(&a, &b)
+}
+
+fn tokenize(input: String, boundaries: bool) -> Vec<String> {
+    phonetics::tokens(&input, boundaries)
+}

data/ext/phonetics_ruby/vendor/phonetics/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+# `phonetics` on crates.io is taken (different project — a metaphone/
+# NYSIIS encoder). We publish under `phonetics-rs`, the conventional
+# fallback when a bare project name is unavailable. The Rust API is
+# still `use phonetics::distance;` because `lib.name` below stays
+# `phonetics`; only the crates.io package name changes.
+name = "phonetics-rs"
+description = "IPA-based phonetic distance metrics: strict edit distance, listener-confusion distance, and per-phoneme acoustic and perceptual scoring. Calibrated against Mad Gab puzzle data; tunable per dialect."
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+homepage.workspace = true
+authors.workspace = true
+readme = "README.md"
+keywords = ["phonetics", "ipa", "linguistics", "edit-distance", "speech"]
+categories = ["text-processing", "science"]
+
+[lints]
+workspace = true
+
+# Keep the Rust API name as `phonetics` even though the registry name
+# is `phonetics-rs`. Downstream code stays `use phonetics::distance;`.
+[lib]
+name = "phonetics"
+
+[dependencies]
+
+[dev-dependencies]
+# Empty for now; will gain criterion + insta as the surface grows.

data/ext/phonetics_ruby/vendor/phonetics/README.md

@@ -0,0 +1,29 @@
+# phonetics
+
+IPA-based phonetic distance metrics for Rust.
+
+Two scores live in this crate:
+
+* **`distance(a, b)`** — strict per-phoneme acoustic distance over IPA
+  phonemes (Bark-space vowel distance, 2D consonant place embedding,
+  approximant–vowel bridge, diphthong/affricate handling).
+* **`Confusion::distance(s1, s2)`** — listener-confusion distance over
+  whole phonemic strings, calibrated against Mad Gab puzzle data and
+  English speech-perception literature.
+
+The split is the point. The first is a claim about the waveform; the
+second is a claim about how a listener parses it. They give different
+answers to different questions.
+
+This is a port of the Ruby gem of the same name; see the parent
+repository for an extended write-up of the metric design.
+
+## Status
+
+Pre-1.0. Core data tables and the per-phoneme acoustic distance are
+parity-tested against the Ruby reference. Levenshtein and Confusion DPs
+are landing module-by-module.
+
+## License
+
+MIT.

data/ext/phonetics_ruby/vendor/phonetics/src/compounds.rs

@@ -0,0 +1,40 @@
+//! Compound phonemes — diphthongs and affricates that are recognised
+//! as single perceptual units even though their IPA notation is two
+//! characters.
+//!
+//! Distance for a compound vs another phoneme is the average of the
+//! pairwise component distances; the shorter side is padded by
+//! repeating its last segment so /aɪ/ vs /a/ charges half a phoneme
+//! distance rather than nothing.
+
+/// Every registered compound phoneme symbol.
+pub const INVENTORY: &[&str] = &[
+    // Diphthongs
+    "aɪ","ɑɪ","aʊ","ɑʊ","ɔɪ","eɪ","oʊ","əʊ","ɪə","ʊə","ɛə",
+    // English affricates
+    "tʃ","dʒ",
+];
+
+/// Decompose a compound phoneme symbol into its components. Returns
+/// `None` for non-compound phonemes.
+pub fn components(symbol: &str) -> Option<&'static [&'static str]> {
+    Some(match symbol {
+        // English-style diphthongs (both /aɪ/-form and /ɑɪ/-form).
+        "aɪ" => &["a", "ɪ"],
+        "ɑɪ" => &["ɑ", "ɪ"],
+        "aʊ" => &["a", "ʊ"],
+        "ɑʊ" => &["ɑ", "ʊ"],
+        "ɔɪ" => &["ɔ", "ɪ"],
+        "eɪ" => &["e", "ɪ"],
+        "oʊ" => &["o", "ʊ"],
+        "əʊ" => &["ə", "ʊ"],
+        "ɪə" => &["ɪ", "ə"],
+        "ʊə" => &["ʊ", "ə"],
+        "ɛə" => &["ɛ", "ə"],
+        // English affricates only. /ts/, /dz/ etc. would mis-tokenise
+        // English plurals (cats, kids) as single phonemes; left out.
+        "tʃ" => &["t", "ʃ"],
+        "dʒ" => &["d", "ʒ"],
+        _ => return None,
+    })
+}