phonemizer 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: dffac8410cd35ee71c8c3532537aaf635f4eee980f302c72fad6ad52c8bff72b
4
+ data.tar.gz: '09cf6f4fda3e28e7f972ae6ac9c99a7912ebb64a2ddfd6498fec047a746ec21d'
5
+ SHA512:
6
+ metadata.gz: 35ccc1998cf9ad0f69d0bf22384a5e02a53e6108df18781addb4e61989557984ccc6fcea78d0544561b45f4a1c4902e516277a4c1ab8a9c66ee4b426d49997b4
7
+ data.tar.gz: d9c57e5abcbfc91491b6b69b41ed6164f11b2ae7bf753ca2728cc342809047652ddd5be276046b0c4b87264b999e1b765e5e090c39427ea6b6694f533eefdbfa
data/CHANGELOG.md ADDED
@@ -0,0 +1,5 @@
1
+ # Changelog
2
+
3
+ ## 0.1.0 - 2024-11-15
4
+
5
+ Initial release.
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 anantone
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,175 @@
1
+ phonemizer
2
+ ============
3
+
4
+ [![Gem Version](https://img.shields.io/gem/v/phonemizer.svg)](https://rubygems.org/gems/phonemizer)
5
+ [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
6
+
7
+ A minimal Ruby wrapper around espeak-ng for simple word-to-IPA phonemization, with caching and parallelism.
8
+
9
+
10
+ Requirements
11
+ ----------------------------------------------------------------
12
+
13
+ - Ruby 3.0+
14
+ - **espeak-ng** command-line tool must be installed:
15
+ ```bash
16
+ apt install espeak-ng # Debian/Ubuntu
17
+ brew install espeak-ng # macOS
18
+ ```
19
+
20
+
21
+ Installation
22
+ ----------------------------------------------------------------
23
+
24
+ Add to your Gemfile:
25
+
26
+ ```ruby
27
+ gem "phonemizer"
28
+ ```
29
+
30
+ Or install directly:
31
+
32
+ ```bash
33
+ gem install phonemizer
34
+ ```
35
+
36
+
37
+ Usage
38
+ ----------------------------------------------------------------
39
+
40
+ ```ruby
41
+ require "phonemizer"
42
+
43
+ # Single word
44
+ Phonemizer.phonemize("manger", language: "fr-fr")
45
+ # => "mɑ̃ʒe"
46
+
47
+ # Batch - pass an Array, get an Array back (same order)
48
+ Phonemizer.phonemize(["docteur", "tableau", "pastille"], language: "fr-fr")
49
+ # => ["dɔktœʁ", "tablo", "pastij"]
50
+ ```
51
+
52
+
53
+ Options
54
+ ----------------------------------------------------------------
55
+
56
+ ```ruby
57
+ Phonemizer.phonemize(
58
+ "les amis",
59
+ language: "fr-fr", # espeak voice (default "en-us")
60
+ phone_sep: "", # separator between phones ("" = contiguous)
61
+ word_sep: " ", # separator between words in one utterance
62
+ strip_stress: true, # remove stress marks and liaison hyphens
63
+ language_switch: :remove # :remove or :keep, for inline (fr)/(en) tags
64
+ )
65
+ ```
66
+
67
+ Separate each phone explicitly:
68
+
69
+ ```ruby
70
+ Phonemizer.phonemize("tableau", language: "fr-fr", phone_sep: " ")
71
+ # => "t a b l o"
72
+ ```
73
+
74
+ Keep the language-switch markers espeak emits mid-utterance:
75
+
76
+ ```ruby
77
+ Phonemizer.phonemize("le célèbre Schwarzenegger", language: "fr-fr", language_switch: :keep)
78
+ "lə selɛbʁ (en)ʃwɔːzənɛɡə(fr)"
79
+ ```
80
+
81
+
82
+ Punctuation
83
+ ----------------------------------------------------------------
84
+
85
+ Punctuation is preserved verbatim and never sent to espeak, so whole sentences pass through cleanly:
86
+
87
+ ```ruby
88
+ Phonemizer.phonemize("Bonjour, comment ça va ?", language: "fr-fr")
89
+ # => "bɔ̃ʒuʁ,kɔmɑ̃ sa va?"
90
+ ```
91
+
92
+ Apostrophes are kept inside the word on purpose, so elisions stay intact and liaison consonants aren't lost:
93
+
94
+ ```ruby
95
+ Phonemizer.phonemize("aujourd'hui", language: "fr-fr")
96
+ # => "oʒuʁdyi"
97
+ ```
98
+
99
+
100
+ Caching
101
+ ----------------------------------------------------------------
102
+
103
+ Results are cached per `(word, language, options)`. Useful when the same forms recur across a large lexicon. Call `clear_cache` to free memory or start fresh:
104
+
105
+ ```ruby
106
+ Phonemizer.clear_cache
107
+ ```
108
+
109
+
110
+ Thread Safety
111
+ ----------------------------------------------------------------
112
+
113
+ This gem is **not thread-safe**. The internal cache uses a plain Hash without synchronization. If you call `phonemize` from multiple threads:
114
+
115
+ - Duplicate espeak-ng calls may occur for the same word
116
+ - Cache reads may see stale state
117
+
118
+ **Workaround:** Wrap your calls in your own `Mutex`:
119
+
120
+ ```ruby
121
+ @mutex = Mutex.new
122
+ @mutex.synchronize { Phonemizer.phonemize(words, language: "fr-fr") }
123
+ ```
124
+
125
+
126
+ API Reference
127
+ ----------------------------------------------------------------
128
+
129
+ ### `Phonemizer.phonemize(words, **options)`
130
+
131
+ Phonemize text to IPA. Returns a String for String input, Array for Array input.
132
+
133
+ **Arguments:**
134
+ - `words` — String or Array of Strings
135
+ - `language` — espeak voice (default: `"en-us"`)
136
+ - `phone_sep` — separator between phones (default: `""`)
137
+ - `word_sep` — separator between words in multi-word input (default: `" "`)
138
+ - `strip_stress` — remove stress marks ˈ ˌ and liaison hyphens (default: `true`)
139
+ - `language_switch` — `:remove` or `:keep` for inline language tags (default: `:remove`)
140
+
141
+ ### `Phonemizer.espeak_raw(text, language, phone_sep: nil)`
142
+
143
+ Low-level: call espeak-ng directly. Returns raw IPA output string.
144
+
145
+ **Arguments:**
146
+ - `text` — text to phonemize
147
+ - `language` — espeak voice
148
+ - `phone_sep` — optional separator (passed as `--sep` to espeak-ng)
149
+
150
+ ### `Phonemizer.clear_cache`
151
+
152
+ Clear the internal phonemization cache. Returns the new empty cache Hash.
153
+
154
+
155
+ Reading from a File
156
+ ----------------------------------------------------------------
157
+
158
+ `phonemize` accepts any Array of Strings, so a wordlist is a one-liner:
159
+
160
+ ```ruby
161
+ lines = File.readlines("words.txt", chomp: true).reject(&:empty?)
162
+ File.write("words.ipa", Phonemizer.phonemize(lines, language: "fr-fr").join("\n"))
163
+ ```
164
+
165
+
166
+ Contributing
167
+ ----------------------------------------------------------------
168
+
169
+ Bug reports and pull requests are welcome on [GitHub](https://github.com/anantone/ruby-phonemizer).
170
+
171
+
172
+ License
173
+ ----------------------------------------------------------------
174
+
175
+ MIT
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Phonemizer
4
+ VERSION = "0.1.0"
5
+ end
data/lib/phonemizer.rb ADDED
@@ -0,0 +1,169 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "phonemizer/version"
4
+ require "open3"
5
+ require "etc"
6
+
7
+ # A minimal wrap of espeak-ng for simple uses (calls espeak CLI directly).
8
+ #
9
+ # apt install espeak-ng # or brew install espeak-ng
10
+ #
11
+ # Backend contract (verified against espeak-ng):
12
+ # espeak-ng -q --ipa -v <lang> --stdin
13
+ # espeak-ng merges ALL of stdin into one text and prints the IPA with
14
+ # words space-separated on one line, in input order. Newlines do NOT
15
+ # isolate utterances, so each piece is sent in its own call (its own
16
+ # stdin line) to guarantee per-word isolation. A piece is usually one
17
+ # word, but a multi-word chunk inside one input element (no internal
18
+ # punctuation) stays whole — words within it DO influence each other
19
+ # (liaisons, pronounced final consonants).
20
+ module Phonemizer
21
+ module_function
22
+
23
+ DEFAULT_LANGUAGE = "en-us"
24
+ DEFAULT_WORKERS = [Etc.nprocessors - 1, 1].max
25
+
26
+ # espeak emits parenthesized language tags on language switches, e.g.
27
+ # (en) / (fr) / (pt-br); the pattern covers any tag espeak may emit.
28
+ LANG_SWITCH = /\((?:[a-z]{2,3}(?:-[a-z0-9]+)*)\)/
29
+
30
+ # Characters espeak would try to pronounce; we split them out instead.
31
+ # Apostrophes are EXCLUDED on purpose: they must stay inside the word
32
+ # they elide (aujourd'hui, l'instar). Splitting on them transcribes the
33
+ # halves separately and drops liaison consonants (oʒuʁdɥi -> oʒuʁ + yi).
34
+ PUNCT = /([[:punct:]&&[^'’‘]]+)/
35
+
36
+ @cache = {}
37
+
38
+ # ---------------------------------------------------------------- public API
39
+
40
+ # Words: a String (one utterance) or Array of words.
41
+ # Returns a String (or Array of Strings, matching the input shape).
42
+ #
43
+ # Options:
44
+ # language: espeak voice, default "en-us"
45
+ # phone_sep: inserted between phones, default "" (mɑ̃ʒe)
46
+ # word_sep: joins multiword utterances, default " "
47
+ # strip_stress: remove ˈ ˌ from output, default true
48
+ # language_switch: :remove (default) or :keep
49
+ def phonemize(words, language: DEFAULT_LANGUAGE, phone_sep: "",
50
+ word_sep: " ", strip_stress: true, language_switch: :remove)
51
+ if words.is_a?(Array)
52
+ phonemize_batch(words, language:, phone_sep:, word_sep:,
53
+ strip_stress:, language_switch:)
54
+ else
55
+ phonemize_batch([words], language:, phone_sep:, word_sep:,
56
+ strip_stress:, language_switch:).first
57
+ end
58
+ end
59
+
60
+ def clear_cache
61
+ @cache = {}
62
+ end
63
+
64
+ # ---------------------------------------------------------------- backend call
65
+
66
+ # One espeak-ng call. Returns the raw output string (may span several
67
+ # words or lines; the caller interprets it).
68
+ def espeak_raw(text, language, phone_sep: nil)
69
+ args = ["espeak-ng", "-q", "--ipa", "-v", language]
70
+ # --sep only when explicitly requested: some espeak-ng builds reject it,
71
+ # and the default contiguous IPA output needs no separator anyway.
72
+ args << "--sep=#{phone_sep}" if phone_sep && !phone_sep.empty?
73
+ args << "--stdin"
74
+
75
+ # NB: text + "\n", not text << "\n" — << would mutate the caller's
76
+ # string, and pieces are hash keys in transcribe_all's ipa_map
77
+ out, err, status = Open3.capture3(*args, stdin_data: text + "\n")
78
+
79
+ raise "espeak-ng failed (#{status.exitstatus}): #{err.strip}" unless status.success?
80
+
81
+ warn "WARNING: espeak-ng returned no output for: #{text[0, 40].inspect}" \
82
+ if out.strip.empty? && !text.strip.empty?
83
+
84
+ out
85
+ end
86
+
87
+ # ---------------------------------------------------------------- pipeline
88
+
89
+ def phonemize_batch(words, language:, phone_sep:, word_sep:,
90
+ strip_stress:, language_switch:)
91
+ # Guard for nil
92
+ words = words.map { |w| w.to_s } # Convert nil to empty string
93
+ # Cache: word forms repeat constantly across a lexicon
94
+ key = ->(w) { [w, language, phone_sep, word_sep, strip_stress] }
95
+ todo = words.reject { |w| @cache.key?(key.call(w)) }.uniq
96
+
97
+ unless todo.empty?
98
+ # Punctuation handling: split utterances on punctuation, transcribe
99
+ # only the clean chunks, splice punctuation back by position
100
+ chunks = todo.map { |w| split_punctuation(w) }
101
+
102
+ clean_words = chunks.flatten.reject { |p| p =~ PUNCT }.uniq
103
+ ipa_map = transcribe_all(clean_words, language, phone_sep)
104
+
105
+ todo.each_with_index do |w, i|
106
+ @cache[key.call(w)] = stitch(chunks[i], ipa_map,
107
+ phone_sep:, word_sep:, strip_stress:,
108
+ language_switch:)
109
+ end
110
+ end
111
+
112
+ words.map { |w| @cache[key.call(w)] }
113
+ end
114
+
115
+ # One espeak-ng call per piece (per-line isolation), drained by a
116
+ # machine-dependent pool of worker threads.
117
+ def transcribe_all(pieces, language, phone_sep, workers: DEFAULT_WORKERS)
118
+ queue = Queue.new
119
+ pieces.each { |p| queue << p }
120
+ workers.times { queue << nil } # poison pills: one per worker
121
+
122
+ results = Array.new(workers) { [] }
123
+ threads = Array.new(workers) do |i|
124
+ Thread.new do
125
+ while (piece = queue.pop)
126
+ results[i] << [piece, espeak_raw(piece, language, phone_sep:).strip]
127
+ end
128
+ end
129
+ end
130
+ threads.each(&:join)
131
+
132
+ results.each_with_object({}) { |pairs, map| map.merge!(pairs.to_h) }
133
+ end
134
+
135
+ # utterance -> [punct, word, punct, word, ...]
136
+ def split_punctuation(word)
137
+ word.split(PUNCT).reject(&:empty?)
138
+ end
139
+
140
+ # Reassemble one utterance: punctuation back in place, separators applied.
141
+ # ipa_map maps each chunk piece to its raw IPA (looked up by content).
142
+ def stitch(parts, ipa_map, phone_sep:, word_sep:, strip_stress:,
143
+ language_switch:)
144
+ parts.map do |part|
145
+ next part if part =~ PUNCT # punctuation: verbatim
146
+ t = ipa_map[part] || ""
147
+ t = t.gsub(LANG_SWITCH, "") if language_switch == :remove
148
+ # Strip stress marks always; strip the liaison hyphen only in contiguous
149
+ # mode, where espeak emits it (e.g. "les amis" -> "le-z ami"). In --sep
150
+ # mode a "-" is a user-chosen phone separator, never a liaison marker,
151
+ # so it must survive.
152
+ if strip_stress
153
+ t = t.delete("ˈˌ")
154
+ t = t.delete("-") if phone_sep.empty?
155
+ end
156
+ if phone_sep.empty?
157
+ # contiguous phones: a plain space is a word boundary
158
+ t = t.gsub(" ", word_sep)
159
+ else
160
+ # --sep was requested: single spaces are phone separators, double
161
+ # spaces are word boundaries. Protect the boundaries first, otherwise
162
+ # the single-space substitution consumes one half of every double
163
+ # space and the word boundary is lost.
164
+ t = t.gsub(" ", "\x00").gsub(" ", phone_sep).gsub("\x00", word_sep)
165
+ end
166
+ t
167
+ end.join
168
+ end
169
+ end
metadata ADDED
@@ -0,0 +1,73 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: phonemizer
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - "[anantone]"
8
+ bindir: bin
9
+ cert_chain: []
10
+ date: 1980-01-02 00:00:00.000000000 Z
11
+ dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: rake
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - ">="
17
+ - !ruby/object:Gem::Version
18
+ version: '0'
19
+ type: :development
20
+ prerelease: false
21
+ version_requirements: !ruby/object:Gem::Requirement
22
+ requirements:
23
+ - - ">="
24
+ - !ruby/object:Gem::Version
25
+ version: '0'
26
+ - !ruby/object:Gem::Dependency
27
+ name: rspec
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - "~>"
31
+ - !ruby/object:Gem::Version
32
+ version: '3.13'
33
+ type: :development
34
+ prerelease: false
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '3.13'
40
+ description: A thin Ruby wrapper around espeak-ng CLI for simple word-to-IPA phonemization
41
+ with caching and parallelism.
42
+ email: "[self@anantone.me]"
43
+ executables: []
44
+ extensions: []
45
+ extra_rdoc_files: []
46
+ files:
47
+ - CHANGELOG.md
48
+ - LICENSE
49
+ - README.md
50
+ - lib/phonemizer.rb
51
+ - lib/phonemizer/version.rb
52
+ homepage: https://github.com/anantone/ruby-phonemizer
53
+ licenses:
54
+ - MIT
55
+ metadata: {}
56
+ rdoc_options: []
57
+ require_paths:
58
+ - lib
59
+ required_ruby_version: !ruby/object:Gem::Requirement
60
+ requirements:
61
+ - - ">="
62
+ - !ruby/object:Gem::Version
63
+ version: '3.0'
64
+ required_rubygems_version: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ requirements: []
70
+ rubygems_version: 4.0.24
71
+ specification_version: 4
72
+ summary: Minimal espeak-ng wrapper for phonemization
73
+ test_files: []