translit_kit 0.9

data/lib/phoneme_maps/single.json

@@ -0,0 +1,40 @@
+{
+  "א": [""],
+  "ב": ["v"],
+  "בּ": ["b"],
+  "ג": ["g"],
+  "ד": ["d"],
+  "ה": ["h"],
+  "ו": ["v"],
+  "ז": ["z"],
+  "ח": ["ch"],
+  "חַ": ["ach"],
+  "ט": ["t"],
+  "י": ["y"],
+  "כ": ["ch"],
+  "כּ": ["k"],
+  "ל": ["l"],
+  "מ": ["m"],
+  "נ": ["n"],
+  "ס": ["s"],
+  "ע": ["a"],
+  "פ": ["f"],
+  "פּ": ["p"],
+  "צ": ["tz"],
+  "ק": ["k"],
+  "ר": ["r"],
+  "שׁ": ["sh"],
+  "ש": ["s"],
+  "ת": ["s"],
+  "תּ": ["t"],
+  "ָ": ["o"],
+  "ַ": ["a"],
+  "ֵ": ["ei"],
+  "ֶ": ["e"],
+  "ִ": ["i"],
+  "ֹ": ["o"],
+  "וֹ": ["o"],
+  "וּ": ["u"],
+  "ֻ": ["u"],
+  "ְ": ["e"]
+}

data/lib/phonemizer.rb

@@ -0,0 +1,170 @@
+=begin
+  Phonemizer.rb
+
+  Takes a raw Hebrew word (with nekudos)
+  and returns an array of phonemes.
+
+  Behavior:
+  * Letters and nekudos are seperated.
+  * Strips spaces
+  * Normalizes CHATAF nekudos
+  * Normalizes final letters
+  * The DAGESH is joined to its letter
+  * The SHIN's dot is attached to the SHIN
+  * MALEI nekudos are stripped of their extra YUD
+
+=end
+
+
+require 'permuter'
+require 'phoneme_maps'
+
+# Constants
+
+# English and Hebrew Unicode have different space (' ') characters
+ENGLISH_SPACE = [160].pack "U"
+HEBREW_SPACE = [32].pack "U"
+
+# Edge-case characters
+DAGESH   = "ּ"
+SHIN_DOT = "ׁ"
+
+# Nekudos that have special cases
+CHOLOM   = "ֹ"
+PATACH   = "ַ"
+CHIRIK   = "ִ"
+TZEIREI  = "ֵ"
+
+# Letters that have special cases
+SIN      = "ש"
+VAV      = "ו"
+CHES     = "ח"
+YUD      = "י"
+SHIN_WITH_DOT = "שׁ"
+
+
+# Regexes
+LETTER       = /[אבגדהוזחטיכלמנסעפקרשתםןץףךצ]/
+FINAL_LETTER = /[םןךףץ]/
+CHATAF       = ['ֲ','ֳ','ֱ']
+DAGESH_WHITELIST = /[בוכפת]/
+
+
+# Breaks a Hebrew string into its discrete phonemes
+class Phonemizer
+
+  def initialize word
+    @hebword = word
+  end
+
+  # Returns the unedited Hebrew string
+  def raw
+    @hebword
+  end
+
+  # Breaks the word down into its discrete phonemes
+  # “ם’’ ,“וּ“ ,“כּ“ ,“ע“] = "עַכּוּם]
+  #
+  # No arguments; returns an array
+  #
+  # This function depends heavily on the workings of Hebrew grammer,
+  # so it gets a bit complicated. If you have a more elegant solution, I'd gladly take it.
+  # This thing was a hornet's nest full of bugs, so watch that test suite when editing!
+  def phonemes
+        @completed = []
+
+        # For each raw character :
+        @hebword.chars.each_with_index do |char,i|
+
+          # Skip whitespace
+          if char == ENGLISH_SPACE || char == HEBREW_SPACE
+            next
+
+          # If it's a final letter, normalize it to its standard form (מ –> ם)
+          elsif char =~ FINAL_LETTER
+            @completed << normalize_final_letter(char)
+
+          # If it's a CHATAF, normalize it to it's standard form
+          elsif CHATAF.include? char
+            @completed << deCHATAFize(char)
+
+          # If it's a SHIN_DOT, find the previous SIN and replace it with SHIN_WITH_DOT
+          elsif char == SHIN_DOT
+            @completed[@completed.rindex(SIN)] = SHIN_WITH_DOT
+
+          # If it's a DAGESH:
+          # 1. Find the previous letter
+          # 2. Check if it's on the list of DAGESH-compatible letters
+          # 3. If it is, add it
+          # 4. If it's not, implicitly fall through to the `else` case
+          elsif char == DAGESH
+            previous_letter = previous_letter_index(i, @completed)
+            if previous_letter.nil? then raise "Orphaned DAGESH: DAGESH at position #{i} is not preceded by a letter.(Word: \"#{@hebword}\")"; end
+            if DAGESH_WHITELIST =~ @completed[previous_letter]
+              @completed[previous_letter] += DAGESH
+            end
+
+          # Skip the VAV of a CHOLOM MALEI, otherwise add it
+          elsif char == VAV
+            @hebword[i + 1] == CHOLOM ? next : @completed << VAV
+
+          # Skip the YUD of a CHIRIK MALEI and TZEIREI MALEI, otherwise add them
+          elsif char == YUD
+            (@completed.last == CHIRIK ||
+             @completed.last == TZEIREI) ?
+                next : @completed << YUD
+
+          # Append a PATACH to a final CHES  ( חַ )
+          elsif char == PATACH &&          # It's a PATACH
+              @completed.last == CHES &&   # Proceeded by a CHES
+              (i == @hebword.length - 1)   # At the end of the word
+           @completed[@completed.length - 1] += PATACH
+
+          # Otherwise, pass the letter or nekuda unchanged
+          else
+            @completed << char
+          end
+
+        end # end loop
+        @completed
+
+  end
+
+  private
+
+
+    # Normalize final letters to standard forms
+    def normalize_final_letter char
+      case char
+        when "ם" then return "מ"
+        when "ן" then return "נ"
+        when "ץ" then return "צ"
+        when "ף" then return "פ"
+        when "ך" then return "כ"
+        else
+          raise "#{char} is not a final letter\nSuggested test snippet: #{FINAL_LETTER} =~ #{char}\n"
+      end
+    end
+
+    # Normalize CHATAF nekudos to standard forms
+    # Raises a `RuntimeError` if the character is not one of ['ֲ','ֳ','ֱ']
+    def deCHATAFize chataf
+      case chataf
+        when "ֲ" then return "ַ"
+        when "ֳ" then return "ָ"
+        when "ֱ" then return "ֶ"
+      end
+      raise "#{chataf} is not a CHATAF\n\tSuggested test snippet: ['ֲ','ֳ','ֱ'].include?(#{chataf})"
+    end
+
+    # Return the index of the first previous character that is a letter
+    #  * If the index is a letter            -> Ignore it and find the previous one #BugOrFeature?
+    #  * If a previous character is a letter -> return its index
+    #  * If no characters are letters        -> nil
+    def previous_letter_index current_loc, array
+      current_loc.downto(0) do |i|
+        return i if array[i] =~ LETTER
+      end
+      nil
+    end
+end

data/lib/readme.md

@@ -0,0 +1,120 @@
+# How Transliteration Works
+
+LectureLab uses a pile of helper classes to ease mass-editing strings
+
+## The HebrewWord class
+
+`HebrewWord` takes a a Hebrew word (with _nikkud_) and a _phoneme list_, which maps Hebrew phonemes (letters with optional modifiers) unto English characters.
+(If phonemes are not supplied, it loads a default set. See the implementation)
+
+Example:
+```ruby
+@phonemes = {"ב" => ["v"], "בּ" => ["b","bb"]}
+h = new HebrewWord "בָּעוֹמֶר", @phonemes
+h.transliterate
+# => ...
+```
+
+Let's see the implementation:
+```ruby
+def transliterate list_name = nil
+  Transliterator.new(@hebword, list_name).transliterate
+end
+```
+
+`Hebrew` delegates the actual work to the `Transliterator` class.
+
+## The Transliterator class
+
+
+```ruby
+class Transliterator
+  def transliterate
+    @permuter.permutations
+  end
+...
+```
+
+In the initializer:
+```ruby
+@permuter = Permuter.new
+```
+
+So HebrewWord delegates the actual permuting to the _Permuter_ class
+
+## The Permuter class
+
+The `Permuter` class is a general purpose object for generating combinations:
+```ruby
+p = Permuter.new
+3.times { p.add_array [1,2,3] }
+
+
+p.permutations
+# => [1,1,1]
+[1,1,2]
+[1,1,3]
+[1,2,3]
+...
+```
+
+In our case, the arrays are the possible English letters for every Hebrew phoneme:
+
+```ruby
+def setup_permuter
+  heb_letters.each do |heb_letter|
+    @permuter.add_array @possible_english_letters[heb_letter]
+  end
+end
+```
+Suppose that:
+
+```ruby
+@possible_english_letters = {"ב" => ["v"], "בּ" => ["b","bb"]}
+@possible_english_letters["בּ"]
+# => "["b","bb"]"`
+```
+
+If the word contains the letter _'בּ'_, permutations will be generated containing both _'b'_ and _bb_.
+
+###### And how does Permuter work?
+`Permuter` uses a basic recursive strategy to generate the permutations.
+
+From the implementation
+```ruby
+ private
+  # permute (indices)
+  # Recursively generate every permutation of the arrays (Courtesy of Ari Fordsham)
+  #
+  # The classic recursive permutation algorithm:
+  # Imagine picking a combination lock: [0][0][0]
+  # Each cylinder is the index to one of the arrays
+  # On each recursion, we add another cylinder [0], [0][0], [0][0][0]
+  # When we have enough cylinders, we generate the permutation (base case)
+  # and iterate to the next value by dropping a cylinder, [0][0]
+  # iterating the loop in else, and recursing again  [0][0][1]
+  # Simple and elegant
+  def permute indices
+    # Base case
+    if indices.length == @arrays.length
+      build_permutation indices
+    else
+      @arrays[indices.length].each_with_index do |item,i|
+        permute indices.dup << i
+      end
+    end
+  end
+```
+
+`Permuter` now returns to `Transliterator`, which returns to `HebrewWord`, which returns to the user.
+
+## Summary
+
+1. `HebrewWord` is given a string of Hebrew text, with all the necessary vowelisations.
+2. `HebrewWord` passes the string into `Transliterator`
+3. `Transliterator` passes the string into `Phonemizer`
+4. `Phonemizer` digests the string into usable phonemes for mapping, and hands them back to `Transliterator`
+5. `Transliterator` loads the phoneme map from `PhonemeMaps`, and uses the map and string to configure the `Permuter`
+6. `Permuter` generates the transliterations, and hands them back to `Transliterator`
+7. `Transliterator` returns to `HebrewWord`
+8. `HebrewWord` returns to the user

data/lib/translit_kit.rb

@@ -0,0 +1,2 @@
+module TranslitKit
+end

data/lib/translit_kit/version.rb

@@ -0,0 +1,3 @@
+module TranslitKit
+  VERSION = "0.9"
+end

data/lib/transliterator.rb

@@ -0,0 +1,115 @@
+=begin
+  Transliterator.rb
+
+
+=end
+
+require 'permuter'
+require 'phoneme_maps'
+require 'phonemizer'
+
+class Transliterator < String
+
+  # Initializer
+  # Expects a Unicode Hebrew word (i.e. "עַקֵדָה")
+  # and a optional phoneme-mapping list
+  def initialize string, map_name = nil
+    @hebword     = string
+    @phoneme_map = fetch_phoneme_map map_name
+    setup_permuter
+  end
+
+  # Get the raw Hebrew text of the word (Included NIKUD)
+  def raw
+    @hebword
+  end
+
+  # Alias of `raw`
+  def to_s
+    raw
+  end
+
+  def phoneme_map
+    @list_name
+  end
+
+  def phoneme_map= name
+    @phoneme_map = fetch_phoneme_map name
+  end
+  # Returns a `String` of format:
+  # `hebrew_text`: Permutations: `x` single | `y` short | `z` long
+  def inspect
+    "#{@hebword}: Permutations: #{transliterate(:single).length} single | #{transliterate(:short).length} short | #{transliterate(:long).length} long"
+  end
+
+
+  def phonemes
+    Phonemizer.new(@hebword).phonemes
+  end
+
+  # Return an `Array` of all possible transliterations of the word
+  # As defined in the optional `list_name` argument. options: [:long, :short, :single]
+  # Default is `:single`
+  def transliterate list_name = nil
+    self.phoneme_map = list_name
+    setup_permuter()
+    generate_permutations()
+  end
+
+  private
+
+  # #fetch_phoneme_maponeme_map(list_name)
+  # Returns the appropriate `phoneme_map` for transliteration
+  #
+  # If a name is supplied, use that
+  #   options: [:long, :short, :single] (default is :short)
+  #
+  # Following init, if no list is supplied, the one selected in init is used.
+  #
+  # On init:
+  #    >> name -> use name
+  #    >> nil  -> use :short
+  #
+  # After init
+  #    >> name -> use name
+  #    >> nil  -> use what we've already got
+
+  def fetch_phoneme_map list_name = nil
+    if list_name.nil?
+      defined?(@phoneme_map) ? (return @phoneme_map) : list_name = :short
+    end
+
+    map = PhonemeMaps.new.load list_name
+    @list_name = list_name
+    map
+  end
+
+  # Get all permutations for `@hebword`
+  def generate_permutations
+    @permuter.permutations.
+      select do |pr|
+        # Eliminate duplicate chars
+        # At start and end of permutations
+        # i.e. "avrohom"  -> keep
+        #      "avrohomm" -> reject
+        pr[0] != pr[1] &&                      # compare first 2 chars
+        pr[pr.length - 1] != pr[pr.length - 2] # compare last  2 chars
+      end
+  end
+
+  # Configures the versatile Permuter for permuting the word
+  def setup_permuter
+    @permuter = Permuter.new
+
+    # Get the letters of the word
+    heb_letters = self.phonemes
+
+    # For each letter, add the array
+    # of possible english letters to the permuter
+    heb_letters.each do |heb_letter|
+      en_letters = @phoneme_map[heb_letter]
+      if en_letters.nil? then raise "Couldn't find phoneme_map entry for letter ( #{heb_letter.chars} ) in list `#{@list_name}`\nSuggested test snippet: #{@list_name == ":custom" ? @list_name : "require \'phoneme_maps\';PhonemeMaps.new.short"}['#{heb_letter}'].nil?\n" end
+      @permuter.add_array en_letters
+    end
+  end
+end