thailang4r 0.0.1

data/data/test_dict.txt

@@ -0,0 +1,8 @@
+ก
+กกต
+ข
+ขคทท
+ขจ
+จก
+มจ
+มม

data/lib/thailang4r.rb

@@ -0,0 +1,42 @@
+module ThaiLang
+  def ThaiLang.string_chlevel(s)
+    ch_level_list = []
+    s.each_char do |ch|
+      ch_level_list << chlevel(ch)
+    end
+    ch_level_list
+  end
+  
+  def ThaiLang.chlevel(ch)
+    _chlevel(ch.ord)
+  end
+  
+  def ThaiLang._chlevel(code)
+    level = nil    
+    if (code >= 0x0E01 and code <= 0x0E30) or
+       (code >= 0x0E32 and code <= 0x0E33) or
+       (code >= 0x0E3F and code <= 0x0E46) or
+       (code >= 0x0E4F and code <= 0x0E5B) then
+      level = 1
+    elsif (code >= 0x0E38 and code <= 0x0E3A) then
+      level = -1
+    elsif code == 0x0E31 or
+            (code >= 0x0E34 and code <= 0x0E37) or
+            (code >= 0x0E4C and code <= 0x0E4E) then
+      level = 2
+    elsif code >= 0x0E48 and code <= 0x0E4B then
+      level = 3
+    end
+    level
+  end
+  
+  def ThaiLang.exclude_thai_lower_upper(s)
+    included_list = []
+    s.each_char do |ch|
+      if chlevel(ch).nil? or chlevel(ch) == 1
+        included_list << ch
+      end
+    end
+    included_list.join('')
+  end
+end

data/lib/thailang4r/dict.rb

@@ -0,0 +1,89 @@
+module ThaiLang
+  class Dict
+    def initialize(file_path)
+      load_dict(file_path)
+    end
+    
+    def load_dict(file_path)      
+      File.open(file_path) do |f|
+        @str_list = f.readlines.map{|line| line.chomp}
+      end
+    end
+    
+    def find_first_index_of_needle(prefix, offset = nil, s = nil, e = nil)
+      find_index_of_needle(:FIRST, prefix, offset, s, e)
+    end
+    
+    def find_last_index_of_needle(prefix, offset = nil, s = nil, e = nil)
+      find_index_of_needle(:LAST, prefix, offset, s, e)
+    end
+    
+    def find_index_of_needle(pos_type, prefix, offset = nil, s = nil, e = nil) 
+  		offset = offset.nil? ? 0 : offset
+  		s = s.nil? ? 0 : s
+      e = e.nil? ? @str_list.length : e
+      
+      l = s
+      r = e - 1;
+      ans = nil
+
+  		while l <= r do
+  			m = (l + r) / 2 
+  			ch = @str_list[m][offset]
+  			if ch.nil? or prefix > ch
+  			  l = m + 1
+  			elsif prefix < ch
+  			  r = m - 1
+  			else
+  			  ans = m
+  			  if pos_type == :FIRST
+  			    r = m - 1
+  			  else
+  			    l = m + 1
+  			  end
+  			end
+  		end
+  			
+  		ans
+  	end
+  	
+  	def size
+  	  @str_list.length
+	  end
+	  
+	  def [](i)
+	    @str_list[i]
+    end
+  end
+  
+  class DictIter
+    def initialize(dict)
+      @dict = dict
+      @e = @dict.size
+      @s = 0
+      @state = :ACTIVE
+      @offset = 0
+    end
+  
+    def walk(ch) 
+      if @state != :INVALID
+        first = @dict.find_first_index_of_needle ch, @offset, @s, @e
+        if first.nil?
+          @state = :INVALID
+        else
+          @s = first
+          last = @dict.find_last_index_of_needle ch, @offset, @s, @e
+          @e = last + 1
+          len = @dict[first].length
+          @offset += 1
+          if(@offset == len)
+            @state = :ACTIVE_BOUNDARY
+          else
+            @state = :ACTIVE
+          end
+        end
+      end
+      @state
+    end  	
+  end
+end

data/lib/thailang4r/ranges_builder.rb

@@ -0,0 +1,90 @@
+module ThaiLang  
+  class RangesBuilder 
+    S = 0
+    E = 1
+    LINK_TYPE = 2
+    
+    POINTER = 0
+    WEIGHT = 1
+    PATH_UNK = 2
+    PATH_LINK_TYPE = 3
+    
+    def _build_index(dag, pos)
+  		index = {}
+  		dag.each do |range|
+  		  if not index.has_key?(range[pos])
+  		    index[range[pos]] = []
+		    end
+		    index[range[pos]] << range
+		  end
+      index
+  	end
+  	
+  	def _build_e_index(dag)
+  	  _build_index(dag, E)
+	  end
+	  
+	  def _build_s_index(dag)
+	    _build_index(dag, S)
+    end
+    
+    def _compare_path_info(a, b)
+      a[PATH_UNK] < b[PATH_UNK] and a[WEIGHT] < b[WEIGHT]
+    end
+	
+	  def _build_path(len, s_index, e_index)
+  		path = Array.new(len + 1) {|i| nil}
+  		path[0] = [0, 0, 0, :UNK]
+  		left_boundary = 0
+  		for i in 1..len
+  		  if e_index.has_key?(i)
+  		    e_index[i].each do |range|
+  				  s = range[S]
+            if not path[s].nil?
+  					  info = [s, path[s][WEIGHT] + 1, path[s][PATH_UNK], range[LINK_TYPE]]
+  					  if path[i].nil? or _compare_path_info(info, path[i])
+  						  path[i] = info
+						  end
+  					end  				
+				  end
+				  if not path[i].nil?
+  					left_boundary = i
+  				end
+			  end
+  			if path[i].nil? and s_index.has_key?(i)
+  				info = [left_boundary, 
+  						 path[left_boundary][WEIGHT] + 1, 
+  						 path[left_boundary][PATH_UNK] + 1, 
+  						 :UNK]
+  				path[i] = info;												
+  			end
+  		end	
+			if path[len].nil?
+			  path[len] = [left_boundary, 
+  			             path[left_boundary][WEIGHT] + 1, 
+  			             path[left_boundary][PATH_UNK] + 1, :UNK]  			
+			end	  		
+      path
+    end 
+	
+  	def _path_to_ranges(path, len)
+  		ranges = []
+  		i = len
+  		while i > 0
+  			info = path[i]
+  			s = info[POINTER]
+  			ranges << [s, i, info[PATH_LINK_TYPE]]
+  			i = s
+  		end
+  		ranges.reverse
+  	end
+	
+    def build_from_dag(dag, len)
+      s_index = _build_s_index(dag)
+      e_index = _build_e_index(dag)
+  		path = _build_path(len, s_index, e_index)
+  		_path_to_ranges(path, len)
+  	end
+  end
+
+end

data/lib/thailang4r/word_breaker.rb

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'thailang4r/dict.rb'
+require 'thailang4r/word_dag_builder.rb'
+require 'thailang4r/ranges_builder.rb'
+
+module ThaiLang
+  class WordBreaker
+    
+    S = 0
+    E = 1
+    
+    def initialize(path = nil)
+      if path.nil?
+        path = File.expand_path('../../../data/tdict-std.txt', __FILE__)
+        puts path
+      end
+      @dict = Dict.new path
+  		@dag_builder = WordDagBuilder.new @dict
+  		@ranges_builder = RangesBuilder.new
+    end
+
+    def break_into_words(string)
+  		len = string.length
+  		dag = @dag_builder.build(string, len)
+  		ranges = @ranges_builder.build_from_dag(dag, len)
+  		ranges.map{|range| string[range[S], range[E] - range[S]]}
+  	end  	  	
+  end
+end

data/lib/thailang4r/word_dag_builder.rb

@@ -0,0 +1,78 @@
+module ThaiLang	
+  class WordDagBuilder
+    def initialize(dict)
+      @dict = dict
+    end
+  
+    def build(string, len)
+      dag = []
+      _build_by_dict(dag, string, len)
+      #_build_by_latin_rule(dag, string, len)
+      dag.sort do |a,b| 
+        r = 0
+        for i in 0..2
+          r = a[i] <=> b[i]
+          if r != 0
+            break
+          end
+        end
+        r
+      end
+    end
+  
+    def _build_by_latin_rule(dag, string, len)
+      next_latin = 0
+      for i in 0..(len-1)
+        space_e = nil
+        latin_e = nil
+        space_break = false
+        latin_break = false
+
+        for j in i..(len-1)
+          if space_break and latin_break
+            break
+          end
+          ch = string[j]
+          if not space_break
+            if ch == " "
+              space_e = j + 1
+            else
+              space_break = true
+            end
+          end
+        
+          if latin_break and j >= next_latin
+            if /A-Za-z/.match(ch)
+              latin_e = j + 1
+            else
+              latin_break = true
+            end
+          end
+        end
+           
+        if not space_e.nil?
+          dag << [i, space_e, :SPACE]        
+        end
+        if not latin_e.nil?
+          dag << [i, latin_e, :LATIN]
+          next_latin = latin_e;        
+        end
+      end    
+    end
+  
+    def _build_by_dict(dag, string, len) 
+    	for i in 0..(len-1)
+    	  iter = DictIter.new @dict
+    	  for j in i..(len-1)
+    	    ch = string[j]
+    	    status = iter.walk ch
+    	    if status == :INVALID
+    	      break
+  	      elsif status == :ACTIVE_BOUNDARY
+  	        dag << [i, j + 1, :DICT]
+  	      end
+  	    end
+    	end
+    end
+  end
+end

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification
+name: thailang4r
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Vee Satayamas
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-06-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.2.1
+description: Thai language utility for Ruby
+email:
+- v.satayamas@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/thailang4r.rb
+- lib/thailang4r/word_dag_builder.rb
+- lib/thailang4r/dict.rb
+- lib/thailang4r/ranges_builder.rb
+- lib/thailang4r/word_breaker.rb
+- LICENSE
+- README.md
+- Rakefile
+- data/test_dict.txt
+- data/tdict-std.txt
+homepage: https://github.com/veer66/thailang4r
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: Thai language utility for Ruby
+test_files: []