thailang4r 0.0.1 → 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e1af7a4c89e108e784964062d2381ae2b2b81715eb5c0f2da4e6c5aa9604426b
+  data.tar.gz: 85aaded96eea07d8369007f76adac54abff69fc246ae798da78da1d6ffa51887
+SHA512:
+  metadata.gz: fc738de378f475f7eafa0439176293cd0b524cc3dd8cc246421bb7dc3b444b57f9e12ebdcfffebd56ce3cbdeb3226f72b4909674ae5438b942b040304c600799
+  data.tar.gz: 53aab1f982c0aeadc812e2e2618a1651f1e620dd745d54c2b42195bc3ffd8c60152f3f8fe6be08d73fb95f95bd7f3dff10edc0e91c50a5e2bc12bc3d3589473b

data/LICENSE

@@ -1,20 +1,201 @@
-Copyright (c) 2013 Vee Satayamas
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -4,14 +4,37 @@ Thai language utility for Ruby
 
 I have built this project in order to collect and share tools for Thai language, which are written in Ruby language. 
 
+Installation
+------------
+
+> gem install thailang4r
+
+Character level
+---------------
+
 * chlevel is similar th_chlevel in [libthai](http://linux.thai.net/projects/libthai).
 * string_chlevel gives array of level back for example string_chlevel("กี") will return [1, 2]
 
 Word breaker
 ------------
+
 ```ruby
 # encoding: UTF-8
 require 'thailang4r/word_breaker'
 word_breaker = ThaiLang::WordBreaker.new
 puts word_breaker.break_into_words("ฉันกินข้าว")
+# ["ฉัน", "กิน", "ข้าว"]
+```
+
+Romanization
+------------
+
+A port of royin.py transliteration from [PyThaiNLP](https://pythainlp.github.io/).
+
+```ruby
+# encoding: UTF-8
+require 'thailang4r/roman'
+royin = ThaiLang::Royin.new
+p .romanize("ฉันกินข้าว", "-")
+# => "chan-kin-khao"
 ```

data/lib/thailang4r/roman.rb

@@ -0,0 +1,169 @@
+# coding: utf-8
+
+require_relative 'word_breaker'
+
+module ThaiLang
+  # Ported from https://github.com/PyThaiNLP/pythainlp/blob/dev/pythainlp/transliterate/royin.py (be1265d)
+  class Royin
+    vowel_patterns = """เ*ียว,\\1iao
+แ*็ว,\\1aeo
+เ*ือย,\\1ueai
+แ*ว,\\1aeo
+เ*็ว,\\1eo
+เ*ว,\\1eo
+*ิว,\\1io
+*วย,\\1uai
+เ*ย,\\1oei
+*อย,\\1oi
+โ*ย,\\1oi
+*ุย,\\1ui
+*าย,\\1ai
+ไ*ย,\\1ai
+*ัย,\\1ai
+ไ**,\\1\\2ai
+ไ*,\\1ai
+ใ*,\\1ai
+*ว*,\\1ua\\2
+*ัวะ,\\1ua
+*ัว,\\1ua
+เ*ือะ,\\1uea
+เ*ือ,\\1uea
+เ*ียะ,\\1ia
+เ*ีย,\\1ia
+เ*อะ,\\1oe
+เ*อ,\\1oe
+เ*ิ,\\1oe
+*อ,\\1o
+เ*าะ,\\1o
+เ*็,\\1e
+โ*ะ,\\1o
+โ*,\\1o
+แ*ะ,\\1ae
+แ*,\\1ae
+เ*าะ,\\1e
+*าว,\\1ao
+เ*า,\\1ao
+เ*,\\1e
+*ู,\\1u
+*ุ,\\1u
+*ื,\\1ue
+*ึ,\\1ue
+*ี,\\1i
+*ิ,\\1i
+*ำ,\\1am
+*า,\\1a
+*ั,\\1a
+*ะ,\\1a
+#ฤ,\\1rue
+$ฤ,\\1ri"""
+
+    VOWELS = vowel_patterns.gsub("*", "([ก-ฮ])")
+               .gsub("#", "([คนพมห])")
+               .gsub("$", "([กตทปศส])")
+               .split("\n")
+               .map {_1.split(",")}
+               .map {[Regexp.new(_1), _2]}
+  
+    # พยัญชนะ ต้น สะกด
+    CONSONANTS = {
+      "ก" => ["k", "k"],
+      "ข" => ["kh", "k"],
+      "ฃ" => ["kh", "k"],
+      "ค" => ["kh", "k"],
+      "ฅ" => ["kh", "k"],
+      "ฆ" => ["kh", "k"],
+      "ง" => ["ng", "ng"],
+      "จ" => ["ch", "t"],
+      "ฉ" => ["ch", "t"],
+      "ช" => ["ch", "t"],
+      "ซ" => ["s", "t"],
+      "ฌ" => ["ch", "t"],
+      "ญ" => ["y", "n"],
+      "ฎ" => ["d", "t"],
+      "ฏ" => ["t", "t"],
+      "ฐ" => ["th", "t"],
+      # ฑ พยัญชนะต้น เป็น d ได้
+      "ฑ" => ["th", "t"],
+      "ฒ" => ["th", "t"],
+      "ณ" => ["n", "n"],
+      "ด" => ["d", "t"],
+      "ต" => ["t", "t"],
+      "ถ" => ["th", "t"],
+      "ท" => ["th", "t"],
+      "ธ" => ["th", "t"],
+      "น" => ["n", "n"],
+      "บ" => ["b", "p"],
+      "ป" => ["p", "p"],
+      "ผ" => ["ph", "p"],
+      "ฝ" => ["f", "p"],
+      "พ" => ["ph", "p"],
+      "ฟ" => ["f", "p"],
+      "ภ" => ["ph", "p"],
+      "ม" => ["m", "m"],
+      "ย" => ["y", ""],
+      "ร" => ["r", "n"],
+      "ฤ" => ["rue", ""],
+      "ล" => ["l", "n"],
+      "ว" => ["w", ""],
+      "ศ" => ["s", "t"],
+      "ษ" => ["s", "t"],
+      "ส" => ["s", "t"],
+      "ห" => ["h", ""],
+      "ฬ" => ["l", "n"],
+      "อ" => ["", ""],
+      "ฮ" => ["h", ""],
+    }
+
+    def normalize(word)
+      word.gsub(/จน์|มณ์|ณฑ์|ทร์|ตร์|[ก-ฮ]์|[ก-ฮ]ะ-ู์|[ฯๆ่-๏๚๛]/, "")
+    end
+
+    def replace_vowel(word)
+      VOWELS.each { word.gsub!(_1, _2) }
+      return word
+    end
+
+    def replace_consonants(word, consonants)
+      return word unless consonants
+      return word.gsub(consonants[0], CONSONANTS[consonants[0]][0]) if consonants.length == 1
+      consonants.reduce({rom: "", th: word}) do |w, consonant|
+        non_thai = w[:th].match(/^[^ก-์]+/)
+        if non_thai
+          w[:rom] += non_thai.to_s
+          w[:th] = w[:th][non_thai.to_s.length..-1]
+        end
+        if w[:skip]
+          {rom: w[:rom], th: w[:th]}
+        elsif w[:rom] == "" and w[:th] == "ห"
+          {rom: "", th: w[:th][1..-1]}
+        elsif w[:rom] == ""
+          {rom: CONSONANTS[consonant][0], th: w[:th][consonant.length..-1]}
+        elsif consonant == "ร" and w[:th] == "รร"
+          {rom: w[:rom] + "an", th: w[:th][2..-1], skip: true}
+        elsif consonant == "ร" and w[:th][0..1] == "รร"
+          {rom: w[:rom] + "a", th: w[:th][2..-1], skip: true}
+        else
+          {rom: w[:rom] + CONSONANTS[consonant][1], th: w[:th][consonant.length..-1]}
+        end
+      end[:rom]
+    end
+    
+    def romanize_word(word)
+      word = replace_vowel(normalize(word))
+      consonants = word.scan(/[ก-ฮ]/)
+      if word.length == 2 and consonants.length == 2
+        word = word.chars
+        word.insert(1, "o")
+        word = word.join("")      
+      end
+      word = replace_consonants(word, consonants)
+      return word
+    end
+
+    WORDCUT = WordBreaker.new
+  
+    def romanize(text, delim = "")
+      WORDCUT.break_into_words(text).map { romanize_word _1 }.join(delim)
+    end
+  end
+end

data/lib/thailang4r/roman.rb~

@@ -0,0 +1,3 @@
+module ThaiLang
+  
+end

data/lib/thailang4r/word_breaker.rb

@@ -1,29 +1,254 @@
-require 'rubygems'
-require 'thailang4r/dict.rb'
-require 'thailang4r/word_dag_builder.rb'
-require 'thailang4r/ranges_builder.rb'
+# coding: utf-8
 
 module ThaiLang
+  NODE_KEY_ROW_NO = 0
+  NODE_KEY_OFFSET = 1
+  NODE_KEY_CH = 2
+
+  NODE_PTR_ROW_NO = 0
+  NODE_PTR_IS_FINAL = 1
+  NODE_PTR_PAYLOAD = 2
+
+  NodeKey = Struct.new(:row_no, :offset, :ch)
+
+  class PrefixTree
+    def initialize(sorted_words_with_payload)
+      @prefix_tree = {}
+      sorted_words_with_payload.each.with_index do |(w, payload), i|
+        row_no = 0
+        ch_vec = w.codepoints
+        ch_len = w.length
+        ch_vec.each.with_index do |ch, j|
+          node_key = NodeKey.new(row_no, j, ch)
+          ex_node_ptr = @prefix_tree[node_key]
+          if ex_node_ptr
+            row_no = ex_node_ptr[NODE_PTR_ROW_NO]
+          else
+            is_final = (j + 1 == ch_len)
+            node_ptr = [i, is_final, if is_final; payload; end]
+            @prefix_tree[node_key] = node_ptr
+            row_no = i
+          end
+        end
+      end
+    end
+    
+    def lookup(row_id, offset, ch)
+      @prefix_tree[NodeKey.new(row_id, offset, ch)]
+    end
+  end
+
+  DEFAULT_THAI_DICT_PATH = File.expand_path('../../../data/tdict-std.txt', __FILE__)
+
   class WordBreaker
+    def initialize(dix_path = DEFAULT_THAI_DICT_PATH)
+      @dix = PrefixTree.new(File.open(dix_path).each_line.map { [_1.chomp, 1] })
+    end
+
+    def break_into_words(text)
+      tokenize(@dix, text)
+    end
+    
+    UNK   = 1
+    DICT  = 2
+    INIT  = 3
+    LATIN = 4
+    PUNC  = 5
+    
+    LINK_P_IDX = 0
+    LINK_W     = 1
+    LINK_UNK   = 2
+    LINK_KIND  = 3
+    
+    def better_link?(l, r)
+      l[LINK_UNK] < r[LINK_UNK] or l[LINK_W] < r[LINK_W]
+    end
+    
+    WAITING    = 1
+    ACTIVATED  = 2
+    COMPLETED  = 3
     
-    S = 0
-    E = 1
+    CAP_A = "A".ord
+    CAP_Z = "Z".ord
+    A = "a".ord
+    Z = "z".ord
     
-    def initialize(path = nil)
-      if path.nil?
-        path = File.expand_path('../../../data/tdict-std.txt', __FILE__)
-        puts path
+    def latin?(ch)
+      (ch >= CAP_A and ch <= CAP_Z) or (ch >= A and ch <= Z)
+    end
+    
+    TRANSDUCER_S     = 0
+    TRANSDUCER_E     = 1
+    TRANSDUCER_STATE = 3
+    TRANSDUCER_KIND  = 4
+    
+    def update_latin_transducer(transducer, ch, i, ch_vec)
+      if transducer[TRANSDUCER_STATE] == WAITING
+        if latin?(ch)
+          transducer[TRANSDUCER_S] = i
+          transducer[TRANSDUCER_STATE] = ACTIVATED
+          if i + 1 == ch_vec.length or not latin?(ch_vec[i + 1])
+            transducer[TRANSDUCER_E] = i + 1
+            transducer[TRANSDUCER_STATE] = COMPLETED
+          end
+        end
+      else
+        if latin?(ch)
+          transducer[TRANSDUCER_E] = i + 1
+          transducer[TRANSDUCER_STATE] = COMPLETED      
+        else
+          transducer[TRANSDUCER_STATE] = WAITING
+        end
       end
-      @dict = Dict.new path
-  		@dag_builder = WordDagBuilder.new @dict
-  		@ranges_builder = RangesBuilder.new
     end
+    
+    
+    SPACE = " ".ord
 
-    def break_into_words(string)
-  		len = string.length
-  		dag = @dag_builder.build(string, len)
-  		ranges = @ranges_builder.build_from_dag(dag, len)
-  		ranges.map{|range| string[range[S], range[E] - range[S]]}
-  	end  	  	
+    def punc?(ch)
+      ch == SPACE
+    end
+    
+    def update_punc_transducer(transducer, ch, i, ch_vec)
+      if transducer[TRANSDUCER_STATE] == WAITING
+        if punc?(ch)
+          transducer[TRANSDUCER_S] = i
+          transducer[TRANSDUCER_STATE] = ACTIVATED
+          if i + 1 == ch_vec.length or not punc?(ch_vec[i + 1])
+            transducer[TRANSDUCER_E] = i + 1
+            transducer[TRANSDUCER_STATE] = COMPLETED
+          end
+        end
+      else
+        if punc?(ch)
+          transducer[TRANSDUCER_E] = i + 1
+          transducer[TRANSDUCER_STATE] = COMPLETED      
+        else
+          transducer[TRANSDUCER_STATE] = WAITING
+        end
+      end
+    end
+
+    DIX_PTR_S        = 0
+    DIX_PTR_ROW_NO   = 1
+    DIX_PTR_IS_FINAL = 2
+    
+    def build_path(dix, s)
+      left_boundary = 0
+      ch_vec = s.codepoints
+      ch_len = ch_vec.length
+      path = [[0,0,0,INIT]]
+      dix_ptrs = []
+      latin_transducer = [0,0,WAITING,LATIN]
+      punc_transducer =[0,0,WAITING,PUNC]
+      ch_vec.each.with_index do |ch, i|
+        dix_ptrs << [i, 0, false]
+        unk_link = path[left_boundary]
+        link = [left_boundary, unk_link[LINK_W] + 1, unk_link[LINK_UNK] + 1, UNK]
+        j = 0
+        while j < dix_ptrs.length
+          dix_ptr = dix_ptrs[j]
+          offset = i - dix_ptr[DIX_PTR_S]
+          row_no = dix_ptr[DIX_PTR_ROW_NO]
+          child = dix.lookup(row_no, offset, ch)
+          #      puts "ch:#{ch} offset:#{offset} rowno:#{row_no} child:#{child}"
+          if child
+            dix_ptrs[j] = [dix_ptr[DIX_PTR_S], child[NODE_PTR_ROW_NO], child[NODE_PTR_IS_FINAL]]
+            j += 1
+          else
+            unless j + 1 == dix_ptrs.length
+              dix_ptrs[j] = dix_ptrs.pop
+            else
+              dix_ptrs.pop
+            end
+          end
+        end
+        
+        update_latin_transducer(latin_transducer, ch, i, ch_vec)
+        update_punc_transducer(punc_transducer, ch, i, ch_vec)
+        
+        dix_ptrs.each do |dix_ptr|
+          if dix_ptr[DIX_PTR_IS_FINAL]
+            new_s = dix_ptr[DIX_PTR_S]
+            #        puts "NEW_S:#{new_s} DIX-PTR:#{dix_ptr} i:#{i}"
+            prev_link = path[new_s]
+            w = prev_link[LINK_W]
+            unk = prev_link[LINK_UNK]
+            new_link = [new_s, w + 1, unk, DICT]
+            link = new_link if better_link?(new_link, link)
+          end
+        end
+        
+        if latin_transducer[TRANSDUCER_STATE] == COMPLETED
+          s = latin_transducer[TRANSDUCER_S]
+          prev_link = path[s]
+          w = prev_link[LINK_W]
+          unk = prev_link[LINK_UNK]
+          new_link = [s, w + 1, unk, LATIN]
+          link = new_link if better_link?(new_link, link)      
+        end
+        
+        if punc_transducer[TRANSDUCER_STATE] == COMPLETED
+          s = punc_transducer[TRANSDUCER_S]
+          prev_link = path[s]
+          w = prev_link[LINK_W]
+          unk = prev_link[LINK_UNK]
+          new_link = [s, w + 1, unk, PUNC]     
+          link = new_link if better_link?(new_link, link)
+        end
+        left_boundary = i if link[LINK_KIND] != UNK
+        path << link
+      end
+      path
+    end
+
+    RANGE_S = 0
+    RANGE_E = 1
+    
+    def path_to_ranges(path)
+      e = path.length - 1
+      ranges = []
+      while e > 0
+        link = path[e]
+        s = link[LINK_P_IDX]
+        ranges << [s,e]
+        e = s
+      end
+      ranges.reverse
+    end
+    
+    def ranges_to_toks(ranges, str)
+      ranges.map {|s,e| str[s...e]}
+    end
+
+    def tokenize(dix, str)
+      ranges_to_toks(path_to_ranges(build_path(dix, str)), str)
+    end
+
+    def tokenize_with_delim(dix, str, delim)
+      tokenize(dix, str).join(delim)
+    end   
   end
-end
+end
+
+
+  #dix = APrefixTree.new([["กา",1],["กาก",1]])
+  #p dix
+  #p tokenize_with_delim(dix, "บทความนี้ใช้ระบบคริสต์ศักราช เพราะอ้างอิงคริสต์ศักราชและคริสต์ศตวรรษ หรืออย่างใดอย่างหนึ่ง", "|")
+
+
+  #t2 = [0,0,WAITING,LATIN]
+  #update_punc_transducer(t2, 32, 0, [32])
+  #p t2
+  #t1 = PrefixTree.new([["A",1]])
+  #p t1.lookup(0, 0, "A".codepoints[0])
+
+  #t2 = [0,0,WAITING,LATIN]
+  #update_punc_transducer(t2, 32, 0, [32])
+  #p t2
+  #t1 = PrefixTree.new([["A",1]])
+  #p t1.lookup(0, 0, "A".codepoints[0])
+
+
+  #word_breaker = ThaiLang::WordBreaker.new
+  #puts word_breaker.break_into_words("ฉันกินข้าว")

metadata

@@ -1,71 +1,53 @@
 --- !ruby/object:Gem::Specification
 name: thailang4r
 version: !ruby/object:Gem::Version
-  version: 0.0.1
-  prerelease: 
+  version: 0.0.2
 platform: ruby
 authors:
 - Vee Satayamas
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-06-08 00:00:00.000000000 Z
-dependencies:
-- !ruby/object:Gem::Dependency
-  name: cucumber
-  requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 1.2.1
-description: Thai language utility for Ruby
+date: 2021-04-26 00:00:00.000000000 Z
+dependencies: []
+description: Thai language tools for Ruby, i.e. a word tokenizer, a character level
+  indentifier, and a romanization tool
 email:
-- v.satayamas@gmail.com
+- 5ssgdxltv@relay.firefox.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/thailang4r.rb
-- lib/thailang4r/word_dag_builder.rb
-- lib/thailang4r/dict.rb
-- lib/thailang4r/ranges_builder.rb
-- lib/thailang4r/word_breaker.rb
 - LICENSE
 - README.md
 - Rakefile
-- data/test_dict.txt
 - data/tdict-std.txt
+- data/test_dict.txt
+- lib/thailang4r.rb
+- lib/thailang4r/roman.rb
+- lib/thailang4r/roman.rb~
+- lib/thailang4r/word_breaker.rb
 homepage: https://github.com/veer66/thailang4r
-licenses: []
-post_install_message: 
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.3
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 1.8.25
-signing_key: 
-specification_version: 3
+rubygems_version: 3.2.3
+signing_key:
+specification_version: 4
 summary: Thai language utility for Ruby
 test_files: []

data/lib/thailang4r/dict.rb

@@ -1,89 +0,0 @@
-module ThaiLang
-  class Dict
-    def initialize(file_path)
-      load_dict(file_path)
-    end
-    
-    def load_dict(file_path)      
-      File.open(file_path) do |f|
-        @str_list = f.readlines.map{|line| line.chomp}
-      end
-    end
-    
-    def find_first_index_of_needle(prefix, offset = nil, s = nil, e = nil)
-      find_index_of_needle(:FIRST, prefix, offset, s, e)
-    end
-    
-    def find_last_index_of_needle(prefix, offset = nil, s = nil, e = nil)
-      find_index_of_needle(:LAST, prefix, offset, s, e)
-    end
-    
-    def find_index_of_needle(pos_type, prefix, offset = nil, s = nil, e = nil) 
-  		offset = offset.nil? ? 0 : offset
-  		s = s.nil? ? 0 : s
-      e = e.nil? ? @str_list.length : e
-      
-      l = s
-      r = e - 1;
-      ans = nil
-
-  		while l <= r do
-  			m = (l + r) / 2 
-  			ch = @str_list[m][offset]
-  			if ch.nil? or prefix > ch
-  			  l = m + 1
-  			elsif prefix < ch
-  			  r = m - 1
-  			else
-  			  ans = m
-  			  if pos_type == :FIRST
-  			    r = m - 1
-  			  else
-  			    l = m + 1
-  			  end
-  			end
-  		end
-  			
-  		ans
-  	end
-  	
-  	def size
-  	  @str_list.length
-	  end
-	  
-	  def [](i)
-	    @str_list[i]
-    end
-  end
-  
-  class DictIter
-    def initialize(dict)
-      @dict = dict
-      @e = @dict.size
-      @s = 0
-      @state = :ACTIVE
-      @offset = 0
-    end
-  
-    def walk(ch) 
-      if @state != :INVALID
-        first = @dict.find_first_index_of_needle ch, @offset, @s, @e
-        if first.nil?
-          @state = :INVALID
-        else
-          @s = first
-          last = @dict.find_last_index_of_needle ch, @offset, @s, @e
-          @e = last + 1
-          len = @dict[first].length
-          @offset += 1
-          if(@offset == len)
-            @state = :ACTIVE_BOUNDARY
-          else
-            @state = :ACTIVE
-          end
-        end
-      end
-      @state
-    end  	
-  end
-end

data/lib/thailang4r/ranges_builder.rb

@@ -1,90 +0,0 @@
-module ThaiLang  
-  class RangesBuilder 
-    S = 0
-    E = 1
-    LINK_TYPE = 2
-    
-    POINTER = 0
-    WEIGHT = 1
-    PATH_UNK = 2
-    PATH_LINK_TYPE = 3
-    
-    def _build_index(dag, pos)
-  		index = {}
-  		dag.each do |range|
-  		  if not index.has_key?(range[pos])
-  		    index[range[pos]] = []
-		    end
-		    index[range[pos]] << range
-		  end
-      index
-  	end
-  	
-  	def _build_e_index(dag)
-  	  _build_index(dag, E)
-	  end
-	  
-	  def _build_s_index(dag)
-	    _build_index(dag, S)
-    end
-    
-    def _compare_path_info(a, b)
-      a[PATH_UNK] < b[PATH_UNK] and a[WEIGHT] < b[WEIGHT]
-    end
-	
-	  def _build_path(len, s_index, e_index)
-  		path = Array.new(len + 1) {|i| nil}
-  		path[0] = [0, 0, 0, :UNK]
-  		left_boundary = 0
-  		for i in 1..len
-  		  if e_index.has_key?(i)
-  		    e_index[i].each do |range|
-  				  s = range[S]
-            if not path[s].nil?
-  					  info = [s, path[s][WEIGHT] + 1, path[s][PATH_UNK], range[LINK_TYPE]]
-  					  if path[i].nil? or _compare_path_info(info, path[i])
-  						  path[i] = info
-						  end
-  					end  				
-				  end
-				  if not path[i].nil?
-  					left_boundary = i
-  				end
-			  end
-  			if path[i].nil? and s_index.has_key?(i)
-  				info = [left_boundary, 
-  						 path[left_boundary][WEIGHT] + 1, 
-  						 path[left_boundary][PATH_UNK] + 1, 
-  						 :UNK]
-  				path[i] = info;												
-  			end
-  		end	
-			if path[len].nil?
-			  path[len] = [left_boundary, 
-  			             path[left_boundary][WEIGHT] + 1, 
-  			             path[left_boundary][PATH_UNK] + 1, :UNK]  			
-			end	  		
-      path
-    end 
-	
-  	def _path_to_ranges(path, len)
-  		ranges = []
-  		i = len
-  		while i > 0
-  			info = path[i]
-  			s = info[POINTER]
-  			ranges << [s, i, info[PATH_LINK_TYPE]]
-  			i = s
-  		end
-  		ranges.reverse
-  	end
-	
-    def build_from_dag(dag, len)
-      s_index = _build_s_index(dag)
-      e_index = _build_e_index(dag)
-  		path = _build_path(len, s_index, e_index)
-  		_path_to_ranges(path, len)
-  	end
-  end
-
-end

data/lib/thailang4r/word_dag_builder.rb

@@ -1,78 +0,0 @@
-module ThaiLang	
-  class WordDagBuilder
-    def initialize(dict)
-      @dict = dict
-    end
-  
-    def build(string, len)
-      dag = []
-      _build_by_dict(dag, string, len)
-      #_build_by_latin_rule(dag, string, len)
-      dag.sort do |a,b| 
-        r = 0
-        for i in 0..2
-          r = a[i] <=> b[i]
-          if r != 0
-            break
-          end
-        end
-        r
-      end
-    end
-  
-    def _build_by_latin_rule(dag, string, len)
-      next_latin = 0
-      for i in 0..(len-1)
-        space_e = nil
-        latin_e = nil
-        space_break = false
-        latin_break = false
-
-        for j in i..(len-1)
-          if space_break and latin_break
-            break
-          end
-          ch = string[j]
-          if not space_break
-            if ch == " "
-              space_e = j + 1
-            else
-              space_break = true
-            end
-          end
-        
-          if latin_break and j >= next_latin
-            if /A-Za-z/.match(ch)
-              latin_e = j + 1
-            else
-              latin_break = true
-            end
-          end
-        end
-           
-        if not space_e.nil?
-          dag << [i, space_e, :SPACE]        
-        end
-        if not latin_e.nil?
-          dag << [i, latin_e, :LATIN]
-          next_latin = latin_e;        
-        end
-      end    
-    end
-  
-    def _build_by_dict(dag, string, len) 
-    	for i in 0..(len-1)
-    	  iter = DictIter.new @dict
-    	  for j in i..(len-1)
-    	    ch = string[j]
-    	    status = iter.walk ch
-    	    if status == :INVALID
-    	      break
-  	      elsif status == :ACTIVE_BOUNDARY
-  	        dag << [i, j + 1, :DICT]
-  	      end
-  	    end
-    	end
-    end
-  end
-end