RubyGems - translit_kit - Versions diffs - 0.9 - Mend

translit_kit 0.9

Files changed (64) hide show

checksums.yaml +7 -0
data/MIT-LICENSE +20 -0
data/README.md +86 -0
data/Rakefile +29 -0
data/lib/hebrewword.rb +60 -0
data/lib/permuter.rb +97 -0
data/lib/phoneme_maps.rb +80 -0
data/lib/phoneme_maps/long.json +41 -0
data/lib/phoneme_maps/short.json +39 -0
data/lib/phoneme_maps/single.json +40 -0
data/lib/phonemizer.rb +170 -0
data/lib/readme.md +120 -0
data/lib/translit_kit.rb +2 -0
data/lib/translit_kit/version.rb +3 -0
data/lib/transliterator.rb +115 -0
data/test/dummy/README.rdoc +28 -0
data/test/dummy/Rakefile +6 -0
data/test/dummy/app/assets/javascripts/application.js +13 -0
data/test/dummy/app/assets/stylesheets/application.css +15 -0
data/test/dummy/app/controllers/application_controller.rb +5 -0
data/test/dummy/app/helpers/application_helper.rb +2 -0
data/test/dummy/app/views/layouts/application.html.erb +14 -0
data/test/dummy/bin/bundle +3 -0
data/test/dummy/bin/rails +4 -0
data/test/dummy/bin/rake +4 -0
data/test/dummy/bin/setup +34 -0
data/test/dummy/bin/update +29 -0
data/test/dummy/config.ru +4 -0
data/test/dummy/config/application.rb +15 -0
data/test/dummy/config/boot.rb +3 -0
data/test/dummy/config/cable.yml +9 -0
data/test/dummy/config/database.yml +25 -0
data/test/dummy/config/environment.rb +5 -0
data/test/dummy/config/environments/development.rb +54 -0
data/test/dummy/config/environments/production.rb +86 -0
data/test/dummy/config/environments/test.rb +42 -0
data/test/dummy/config/initializers/application_controller_renderer.rb +6 -0
data/test/dummy/config/initializers/assets.rb +11 -0
data/test/dummy/config/initializers/backtrace_silencers.rb +7 -0
data/test/dummy/config/initializers/cookies_serializer.rb +5 -0
data/test/dummy/config/initializers/filter_parameter_logging.rb +4 -0
data/test/dummy/config/initializers/inflections.rb +16 -0
data/test/dummy/config/initializers/mime_types.rb +4 -0
data/test/dummy/config/initializers/new_framework_defaults.rb +23 -0
data/test/dummy/config/initializers/session_store.rb +3 -0
data/test/dummy/config/initializers/wrap_parameters.rb +14 -0
data/test/dummy/config/locales/en.yml +23 -0
data/test/dummy/config/puma.rb +47 -0
data/test/dummy/config/routes.rb +3 -0
data/test/dummy/config/secrets.yml +22 -0
data/test/dummy/config/spring.rb +6 -0
data/test/dummy/db/test.sqlite3 +0 -0
data/test/dummy/log/test.log +85939 -0
data/test/dummy/public/404.html +67 -0
data/test/dummy/public/422.html +67 -0
data/test/dummy/public/500.html +66 -0
data/test/dummy/public/favicon.ico +0 -0
data/test/hebrewword_test.rb +45 -0
data/test/permuter_test.rb +53 -0
data/test/phoneme_maps_test.rb +29 -0
data/test/phonemizer_test.rb +209 -0
data/test/test_helper.rb +29 -0
data/test/transliterator_test.rb +75 -0
metadata +155 -0

data/test/dummy/public/404.html ADDED Viewed

@@ -0,0 +1,67 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>The page you were looking for doesn't exist (404)</title>
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <style>
+  body {
+    background-color: #EFEFEF;
+    color: #2E2F30;
+    text-align: center;
+    font-family: arial, sans-serif;
+    margin: 0;
+  }
+  div.dialog {
+    width: 95%;
+    max-width: 33em;
+    margin: 4em auto 0;
+  }
+  div.dialog > div {
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #BBB;
+    border-top: #B00100 solid 4px;
+    border-top-left-radius: 9px;
+    border-top-right-radius: 9px;
+    background-color: white;
+    padding: 7px 12% 0;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  h1 {
+    font-size: 100%;
+    color: #730E15;
+    line-height: 1.5em;
+  }
+  div.dialog > p {
+    margin: 0 0 1em;
+    padding: 1em;
+    background-color: #F7F7F7;
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #999;
+    border-bottom-left-radius: 4px;
+    border-bottom-right-radius: 4px;
+    border-top-color: #DADADA;
+    color: #666;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  </style>
+</head>
+<body>
+  <!-- This file lives in public/404.html -->
+  <div class="dialog">
+    <div>
+      <h1>The page you were looking for doesn't exist.</h1>
+      <p>You may have mistyped the address or the page may have moved.</p>
+    </div>
+    <p>If you are the application owner check the logs for more information.</p>
+  </div>
+</body>
+</html>

data/test/dummy/public/422.html ADDED Viewed

@@ -0,0 +1,67 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>The change you wanted was rejected (422)</title>
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <style>
+  body {
+    background-color: #EFEFEF;
+    color: #2E2F30;
+    text-align: center;
+    font-family: arial, sans-serif;
+    margin: 0;
+  }
+  div.dialog {
+    width: 95%;
+    max-width: 33em;
+    margin: 4em auto 0;
+  }
+  div.dialog > div {
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #BBB;
+    border-top: #B00100 solid 4px;
+    border-top-left-radius: 9px;
+    border-top-right-radius: 9px;
+    background-color: white;
+    padding: 7px 12% 0;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  h1 {
+    font-size: 100%;
+    color: #730E15;
+    line-height: 1.5em;
+  }
+  div.dialog > p {
+    margin: 0 0 1em;
+    padding: 1em;
+    background-color: #F7F7F7;
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #999;
+    border-bottom-left-radius: 4px;
+    border-bottom-right-radius: 4px;
+    border-top-color: #DADADA;
+    color: #666;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  </style>
+</head>
+<body>
+  <!-- This file lives in public/422.html -->
+  <div class="dialog">
+    <div>
+      <h1>The change you wanted was rejected.</h1>
+      <p>Maybe you tried to change something you didn't have access to.</p>
+    </div>
+    <p>If you are the application owner check the logs for more information.</p>
+  </div>
+</body>
+</html>

data/test/dummy/public/500.html ADDED Viewed

@@ -0,0 +1,66 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>We're sorry, but something went wrong (500)</title>
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <style>
+  body {
+    background-color: #EFEFEF;
+    color: #2E2F30;
+    text-align: center;
+    font-family: arial, sans-serif;
+    margin: 0;
+  }
+  div.dialog {
+    width: 95%;
+    max-width: 33em;
+    margin: 4em auto 0;
+  }
+  div.dialog > div {
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #BBB;
+    border-top: #B00100 solid 4px;
+    border-top-left-radius: 9px;
+    border-top-right-radius: 9px;
+    background-color: white;
+    padding: 7px 12% 0;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  h1 {
+    font-size: 100%;
+    color: #730E15;
+    line-height: 1.5em;
+  }
+  div.dialog > p {
+    margin: 0 0 1em;
+    padding: 1em;
+    background-color: #F7F7F7;
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #999;
+    border-bottom-left-radius: 4px;
+    border-bottom-right-radius: 4px;
+    border-top-color: #DADADA;
+    color: #666;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  </style>
+</head>
+<body>
+  <!-- This file lives in public/500.html -->
+  <div class="dialog">
+    <div>
+      <h1>We're sorry, but something went wrong.</h1>
+    </div>
+    <p>If you are the application owner check the logs for more information.</p>
+  </div>
+</body>
+</html>

data/test/dummy/public/favicon.ico ADDED Viewed

File without changes

data/test/hebrewword_test.rb ADDED Viewed

@@ -0,0 +1,45 @@
+require 'test_helper'
+require 'hebrewword'
+require 'phonemizer'
+class HebrewWordTest < ActiveSupport::TestCase
+  test "correctly stores the hebrew word" do
+    @heb = HebrewWord.new "שׁבּת"
+    assert_equal @heb.raw, "שׁבּת"
+    assert_equal @heb.to_s, "שׁבּת"
+  end
+  test "should transliterate a word" do
+    @heb = HebrewWord.new "בְּרֵאשִׁית"
+    assert_equal ["bereishis"], @heb.transliterate(:single)
+    @heb = HebrewWord.new "תְּשׁוּבָה"
+    assert_equal ["teshuvoh"], @heb.transliterate(:single)
+    @heb = HebrewWord.new "אַברָהָם"
+    assert_equal ["avroom", "avroam", "avroem", "avrohom", "avroham", "avrohem", "avraom", "avraam", "avraem", "avrahom", "avraham", "avrahem", "avreom", "avream", "avreem", "avrehom", "avreham", "avrehem"],
+                @heb.transliterate(:short)
+  end
+  test "should reject unknown phoneme_map names" do
+    @heb = HebrewWord.new "אַברָהָם"
+    assert_raises(RuntimeError) { @heb.transliterate(:blah) }
+  end
+  test "`phonemes` is delegated to the Phonemizer class" do
+    @heb = HebrewWord.new "אַברָהָם"
+    assert_equal @heb.phonemes, Phonemizer.new(@heb.raw).phonemes
+  end
+  test "method `t` alias for `transliterate`" do
+    @heb = HebrewWord.new "אַברָהָם"
+    assert_equal @heb.transliterate(:single), @heb.t(:single)
+  end
+  test "`inspect` function outputs correct translit counts" do
+    @heb = HebrewWord.new "אַברָהָם"
+    assert_equal @heb.inspect, "אַברָהָם: Permutations: #{@heb.transliterate(:single).length} single | #{@heb.transliterate(:short).length} short | #{@heb.transliterate(:long).length} long"
+  end
+end

data/test/permuter_test.rb ADDED Viewed

@@ -0,0 +1,53 @@
+require 'test_helper'
+require 'permuter'
+class PermuterTest < ActiveSupport::TestCase
+  setup do
+    @p = Permuter.new
+  end
+  test "should return [] if no arrays added" do
+    assert_equal [], @p.permutations
+  end
+  test "should add arrays to internal list" do
+    assert @p.empty?
+    @p.add_array []
+    assert @p.any?
+  end
+  test "should remove arrays from internal list" do
+    @p.add_array []
+    assert @p.any?
+    @p.clear
+    assert @p.empty?
+  end
+  test "should produce single permutation if arrays have only one entry" do
+    @p.add_array [1]
+    @p.add_array [2]
+    @p.add_array [3]
+    assert_equal @p.permutations.length, 1
+  end
+  test "should produce all permutations for integer arrays" do
+    2.times { @p.add_array [0,1] }
+    assert_equal @p.permutations, ["00", "01", "10", "11"]
+    @p.clear
+    3.times { @p.add_array [1,2,3] }
+    assert_equal @p.permutations, ["111", "112", "113", "121", "122", "123", "131", "132", "133", "211", "212", "213", "221", "222", "223", "231", "232", "233", "311", "312", "313", "321", "322", "323", "331", "332", "333"]
+  end
+  test "should produce permutations for Latin letters" do
+    3.times { @p.add_array %w[a b c] }
+    assert_equal @p.permutations, ["aaa", "aab", "aac", "aba", "abb", "abc", "aca", "acb", "acc", "baa", "bab", "bac", "bba", "bbb", "bbc", "bca", "bcb", "bcc", "caa", "cab", "cac", "cba", "cbb", "cbc", "cca", "ccb", "ccc"]
+  end
+  test "should produce permutations for Hebrew letters" do
+    3.times { @p.add_array %w[ג ב א] }
+    assert_equal @p.permutations, ["גגג", "גגב", "גגא", "גבג", "גבב", "גבא", "גאג", "גאב", "גאא", "בגג", "בגב", "בגא", "בבג", "בבב", "בבא", "באג", "באב", "באא", "אגג", "אגב", "אגא", "אבג", "אבב", "אבא", "אאג", "אאב", "אאא"]
+  end
+end

data/test/phoneme_maps_test.rb ADDED Viewed

@@ -0,0 +1,29 @@
+require 'test_helper'
+require 'phoneme_maps'
+class PhonemeMapsTest < ActiveSupport::TestCase
+  def setup
+    @p = PhonemeMaps.new
+  end
+  test "should load files in directory without blowing up" do
+    # Fetch all the filenames
+    filenames = Dir.new(@p.directory).entries.select {|f| f.include? '.json'}
+    # Strip the '.json' extension and convert to symbols
+    symbols = filenames.map {|f|  f.sub(".json", '').to_sym }
+    assert_nothing_raised do
+      symbols.each {|s| @p.load s}
+    end
+  end
+  test "should reject unknown file list names" do
+    assert_raises (RuntimeError) {@p.load :pan_galactic_gargle_blaster}
+  end
+  test "should reject malformed JSON files" do
+    assert_raises (RuntimeError) {@p.validate_json "{'missing_brace': true"}
+  end
+end

data/test/phonemizer_test.rb ADDED Viewed

@@ -0,0 +1,209 @@
+require 'test_helper'
+require 'phonemizer'
+class PhonemizerTest < ActiveSupport::TestCase
+  # STORAGE
+  test"correctly stores the hebrew word" do
+    @phon = Phonemizer.new "שׁבּת"
+    assert_equal @phon.raw, "שׁבּת"
+  end
+  # UNCHANGED CHARACTERS
+  test "should pass normal characters unchanged" do
+    @phon = Phonemizer.new "א"
+    assert_equal ["א"], @phon.phonemes
+  end
+  test "should pass standard nekudos unchanged" do
+    @phon = Phonemizer.new "ְ ֶ ֻ ָ ַ ֵ ִ "
+    assert_equal ["ְ", "ֶ", "ֻ", "ָ", "ַ", "ֵ", "ִ"], @phon.phonemes
+  end
+  # Word splitting
+  test "should seperate letters" do
+    @phon = Phonemizer.new "אבג"
+    assert_equal ["א", "ב", "ג"], @phon.phonemes
+  end
+  test "should seperate letters and nekudos" do
+    @phon = Phonemizer.new "דָג"
+    assert_equal ["ד", "ָ", "ג"], @phon.phonemes
+  end
+  # Altered characters
+  test "should strip spaces" do
+    utf8_en_space = [160].pack "U"
+    utf8_he_space  = [32].pack  "U"
+    @phon = Phonemizer.new utf8_en_space
+    assert_equal [], @phon.phonemes
+    @phon = Phonemizer.new utf8_he_space
+    assert_equal [], @phon.phonemes
+  end
+  # Normalize exotic characters
+  # Final Letters
+  test "should normalize final letters" do
+    @phon = Phonemizer.new "םןץףך"
+    assert_equal ["מ", "נ", "צ", "פ", "כ"], @phon.phonemes
+  end
+  test "should raise an error when final-letter normalizer is given a standard letter" do
+    @phon = Phonemizer.new ''
+    "אבגדהוזחטיכלמנסעפקרשתצ".split('').each do |letter|
+      assert_raises (RuntimeError) {@phon.send :normalize_final_letter, letter}
+    end
+  end
+  # CHATAF nekudos
+  test "should normalize CHATAF nekudos" do
+    @phon = Phonemizer.new " ֲ ֳ ֱ"
+    assert_equal ["ַ", "ָ", "ֶ"], @phon.phonemes
+  end
+  test "should raise an error when the CHATAF-nekuda normalizer is given a standard nekuda" do
+    @phon = Phonemizer.new ''
+    ["ְ", "ֶ", "ֻ", "ָ", "ַ", "ֵ", "ִ"].each do |nekuda|
+      assert_raises (RuntimeError) {@phon.send :deCHATAFize, nekuda}
+    end
+  end
+  # 'Full' nekudos
+  test "should normalize a full chirik ( אִי –> אִ)" do
+    @phon = Phonemizer.new "אִי"
+    assert_equal ["א", "ִ"], @phon.phonemes
+  end
+  test "should normalize a full TZEIREI ( אֵי –> אֵ )" do
+    @phon = Phonemizer.new "אֵי"
+    assert_equal ["א", "ֵ"], @phon.phonemes
+  end
+  test "should normalize a full CHOLOM ( וֹ –> ֹ  )" do
+    @phon = Phonemizer.new "וֹ"
+    assert_equal ["ֹ"], @phon.phonemes
+  end
+# Bad seperations
+    test "should not separate a shin and its dot (שׁ) " do
+      @phon = Phonemizer.new "שׁ"
+      assert_equal ["שׁ"], @phon.phonemes
+    end
+    test "should not seperate a CHES and PATACH when they end the word (חַ)" do
+      # As in פותחַ
+      @phon = Phonemizer.new "חַ"
+      assert_equal ["חַ"], @phon.phonemes
+      @phon = Phonemizer.new "חַחַ"
+      assert_equal ["ח", "ַ", "חַ"], @phon.phonemes
+    end
+# DAGESH!
+  test "should strip DAGESH from unchanging letters" do
+    @phon = Phonemizer.new "אּבּגּדּהּוּזּחּטּיּכּלּמּנּסּעּפּצּקּרּשּשּׁתּ"
+    assert_equal ["בּ", "וּ", "כּ", "פּ", "תּ"], @phon.phonemes.select {|l| l.chars.include? "ּ"}
+  end
+  test "should not separate a DAGESH from its letter" do
+    @phon = Phonemizer.new "בּוּכּפּתּ"
+    assert_equal ["בּ", "וּ", "כּ", "פּ", "תּ"], @phon.phonemes
+  end
+  test "should not separate a DAGESH from its letter even separated by nekuda" do
+    @phon = Phonemizer.new "בַּ"
+    assert_equal ["בּ","ַ"], @phon.phonemes
+  end
+  test "should move DAGESH to correct letter" do
+    @phon = Phonemizer.new "פבּ"
+    assert_equal ["פ","בּ"], @phon.phonemes
+  end
+  test "should move DAGESH even when proceeded by nekuda" do
+      @phon = Phonemizer.new "פָבּ"
+      assert_equal ["פ", "ָ", "בּ"], @phon.phonemes
+  end
+  test "should move DAGESH even when followed by nekuda" do
+      @phon = Phonemizer.new "בָּא"
+      assert_equal ["בּ", "ָ", "א"], @phon.phonemes
+  end
+  test "should move DAGESH even when proceeded by SHIN ( שׁ )" do
+    @phon = Phonemizer.new "שׁבּ"
+    assert_equal ["שׁ", "בּ"], @phon.phonemes
+  end
+  test "should move DAGESH to VAV" do
+    @phon = Phonemizer.new "אוּ"
+    assert_equal ["א", "וּ"], @phon.phonemes
+  end
+  test "should raise error if DAGESH has no preceding letter (Orphaned DAGESH)" do
+    @phon = Phonemizer.new 'ּ'
+    assert_raises (RuntimeError) { @phon.phonemes }
+  end
+  # ==========================
+  # The internal index-finding helper method
+  test "should correctly identify the previous letter in raw hebrew string" do
+    # 'Returns the correct index'
+    word = "שַׁבָּת"
+    # word.chars.each_with_index {|c,i| puts "[#{i}]: \"#{c}\""}
+    #[0]: "ש"
+    #[1]: "ׁ"
+    #[2]: "ַ"
+    #[3]: "ב"
+    #[4]: "ּ"
+    #[5]: "ָ"
+    #[6]: "ת"
+    @phon = Phonemizer.new ""
+    assert_equal 6, @phon.send(:previous_letter_index, 6, word)
+    assert_equal 3, @phon.send(:previous_letter_index, 5, word)
+    assert_equal 3, @phon.send(:previous_letter_index, 4, word)
+    assert_equal 3, @phon.send(:previous_letter_index, 3, word)
+    assert_equal 0, @phon.send(:previous_letter_index, 2, word)
+    assert_equal 0, @phon.send(:previous_letter_index, 1, word)
+    assert_equal 0, @phon.send(:previous_letter_index, 0, word)
+  end
+  test "should correctly identify the previous letter" do
+    @phon = Phonemizer.new ""
+    # "Returns nil when no previous Hebrew chars"
+    assert_nil @phon.send(:previous_letter_index, 0, "abc")
+    # "Returns the index given if the index is a char"
+    assert_equal 0, @phon.send(:previous_letter_index, 0, "א")
+  end
+  # Full test cases
+  test "should break word into phonemes" do
+    @phon = Phonemizer.new "שַׁבָּת"
+    assert_equal ["שׁ", "ַ", "בּ", "ָ", "ת"], @phon.phonemes
+    @phon = Phonemizer.new "בְּרֵאשִׁית"
+    assert_equal ["בּ", "ְ", "ר", "ֵ", "א", "שׁ", "ִ", "ת"], @phon.phonemes
+  end
+end