wordcut 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: a8fea8a44ae91d8e478ab2333b1a6972e744fe1d
+  data.tar.gz: bd19ea76b4594f0d889014405cf7255f9b3b3d35
+SHA512:
+  metadata.gz: e712460bc2a4b2030281518eeb8e623eea60520ce4ad9ea80d9e222d0c0c1705bf1ac45224e673f5c07b3859cd3a84ef646a8e20a689e058e4bb6cb0d0dd9926
+  data.tar.gz: 6023f766a7c2e8808daa4c1fe646e9d81b10a0f82694ef3edaf2a14236ecbb85d2b41e0d4ab6324fe38d4733e8c38064733d5fb1b2fe6f478f679c55bc366797

data/README.md

@@ -0,0 +1,6 @@
+# wordcut.rb
+ASEAN word tokenizer written in Ruby.
+
+## Status: pre-alpha
+
+## The API is subject to change.

data/wordcut/dag.rb

@@ -0,0 +1,99 @@
+require_relative "edge_builder"
+require_relative "pointer"
+require_relative "space_slicer"
+
+module DictDagUpdater
+  def update_by_dict(i, pointers)
+    edge = self.build_edges(pointers).min
+    self[i] = edge
+    return i
+  end
+end
+
+module UnkDagUpdater
+  def update_by_unk(i, left)
+    src = self[left]
+    edge = Edge.new(:s => left,
+                    :unk => src.unk + 1,
+                    :chunk => src.chunk + 1,
+                    :etype => :UNK,
+                    :payload => nil)
+    self[i] = edge
+    return left
+  end
+end
+
+module SpaceDagUpdater
+  def update_by_space(i, slicer)
+    s = slicer.s
+    src = self[s]      
+    edge = Edge.new(:s => s,
+                    :unk => src.unk,
+                    :chunk => src.chunk + 1,
+                    :etype => :SPACE,
+                    :payload => nil)
+    self[i] = edge
+    return i
+  end
+end
+
+module BasicDagUpdater
+  include DictDagUpdater
+  include UnkDagUpdater
+  include SpaceDagUpdater
+  include PointersManipulator
+  
+  def update(i, left, pointers, space_slicer)
+    if not pointers&.empty?
+      update_by_dict(i, pointers)
+    elsif space_slicer&.final
+      update_by_space(i, space_slicer)
+    else
+      update_by_unk(i, left)
+    end
+  end
+end
+
+module DagBuilder
+  def build(dict, txt)
+    self[0] = init_edge
+    pointers = []
+    left = 0
+    space_slicer = SpaceSlicer.new(0)
+    for i in 1..txt.length
+      ch = txt[i - 1]
+      next_ch = i < txt.length ? txt[i] : nil
+      space_slicer.transit(ch, next_ch)
+      pointers << new_pointer(i, dict)
+      pointers = transit(pointers, ch)
+      left = update(i, left, pointers.select(&:final), space_slicer)
+    end
+  end
+end
+
+module DagToToken
+  def tokens(txt)
+    toks = []
+    i = txt.length
+    while i > 0
+      s = self[i].s
+      tok = txt.slice(s, i-s)
+      toks << tok
+      i = s
+    end
+    toks.reverse
+  end
+end
+
+class BasicDag < Array
+  include EdgeBuilder
+  include BasicDagUpdater
+  include DagBuilder
+  include DagToToken
+  
+  def self.build(dict, txt)
+    dag = BasicDag.new(txt.length + 1)
+    dag.build(dict, txt)
+    return dag
+  end
+end

data/wordcut/dict.rb

@@ -0,0 +1,51 @@
+require_relative "dict_seek"
+
+class WordItem
+  attr_reader :headword
+  def initialize(headword)
+    @headword = headword
+  end
+end
+
+module DictInfo
+  def l
+    0
+  end
+
+  def r
+    return nil if self.empty?
+    self.length - 1
+  end
+end
+
+module PathResolver
+  def resolve_path(lang, name)
+    File.expand_path(File.join(__FILE__, '..', '..', 'data', lang, name, ))
+  end
+end
+
+module BasicDictLoader
+  include PathResolver
+  def load_bundle(lang, name)
+    load(resolve_path(lang, name))
+  end
+
+  def load(path)
+    self.concat(open(path).each_line
+                 .map(&:strip)
+                 .reject(&:empty?)
+                 .map{|w| WordItem.new w})
+  end
+end
+
+class BasicDict < Array
+  include DictInfo
+  include DictSeeker
+  include BasicDictLoader
+
+  def self.from_bundle(lang, name)
+    dict = self.new
+    dict.load_bundle(lang, name)
+    return dict
+  end
+end

data/wordcut/dict_seek.rb

@@ -0,0 +1,28 @@
+module DictSeeker
+  def seek(ch, l, r, offset, policy)
+    idx = nil
+    while l <= r
+      m = (l + r) / 2
+      w = self[m].headword
+      wlen = w.length
+
+      if wlen <= offset
+        l = m + 1
+      else
+        ch_w = w[offset]
+        if ch_w < ch
+          l = m + 1
+        elsif ch_w > ch
+          r = m - 1
+        elsif policy == :LEFT
+          idx = m
+          r = m - 1
+        elsif policy == :RIGHT
+          idx = m
+          l = m + 1
+        end        
+      end
+    end
+    return idx
+  end
+end

data/wordcut/edge.rb

@@ -0,0 +1,21 @@
+class Edge
+  attr_reader :unk, :chunk, :s, :payload, :etype
+  
+  CMP_FUNCS = [lambda {|e| e.unk}, lambda {|e| e.chunk}]
+  
+  def initialize(args = {})
+    @unk = args[:unk] || 0
+    @chunk = args[:chunk] || 0
+    @s = args[:s] || 0
+    @payload = args[:payload]
+    @etype = args[:etype]
+  end
+
+  def <=>(o)
+    for fn in CMP_FUNCS
+      cmp = fn.call(self) <=> fn.call(o)
+      return cmp if cmp != 0
+    end
+    return 0
+  end
+end

data/wordcut/edge_builder.rb

@@ -0,0 +1,18 @@
+require_relative "edge.rb"
+
+module EdgeBuilder
+  def init_edge
+    Edge.new
+  end
+  
+  def build_edges(pointers)
+    pointers.map do |pointer|
+      src = self[pointer.s]
+      Edge.new(:s => pointer.s,
+               :unk => src.unk,
+               :chunk => src.chunk + 1,
+               :etype => :DICT,
+               :payload => nil)
+    end                    
+  end
+end

data/wordcut/pointer.rb

@@ -0,0 +1,29 @@
+class Pointer
+  attr_reader :s, :l, :r, :offset, :dict, :final
+  def initialize(s, l, r, offset, dict, final=false)
+    @s = s
+    @l = l
+    @r = r
+    @offset = offset
+    @dict = dict
+    @final = final
+  end
+
+  def update(ch)
+    l = @dict.seek(ch, @l, @r, @offset, :LEFT)
+    return nil unless l
+    r = @dict.seek(ch, l, @r, @offset, :RIGHT)
+    final = (@dict[l].headword.length == @offset + 1)
+    self.class.new(@s, l, r, @offset + 1, @dict, final)
+  end
+end
+
+module PointersManipulator
+  def new_pointer(i, dict)
+    Pointer.new(i-1, dict.l, dict.r, 0, dict)
+  end
+  
+  def transit(pointers, ch)
+    pointers.map{|p| p.update(ch)}.reject(&:nil?)
+  end
+end

data/wordcut/space_slicer.rb

@@ -0,0 +1,26 @@
+class SpaceSlicer
+  attr_reader :s, :offset, :final
+  
+  def initialize(s)
+    @s = s
+    @offset = 0
+    @final = false
+  end
+      
+  def transit(ch, next_ch)
+    current_is_space = (ch =~ /\s/)
+    next_is_space = (not nil? and next_ch =~ /\s/)
+    
+    if current_is_space and next_is_space
+      @offset += 1
+    elsif current_is_space and not next_is_space
+      @offset += 1
+      @final = true
+    elsif not current_is_space 
+      @final = false
+      @s += @offset
+      @s += 1
+      @offset = 0
+    end
+  end
+end

data/wordcut/tokenizer.rb

@@ -0,0 +1,15 @@
+require_relative "dag.rb"
+
+module Tokenizer
+  def tokenize(txt)
+    @dag_class.build(@dict, txt).tokens(txt)
+  end
+end
+
+class BasicTokenizer
+  include Tokenizer
+  def initialize(dict)
+    @dict = dict
+    @dag_class = BasicDag
+  end
+end

metadata

@@ -0,0 +1,53 @@
+--- !ruby/object:Gem::Specification
+name: wordcut
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Vee Satayamas
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-03 00:00:00.000000000 Z
+dependencies: []
+description: Word segmentation tools for ASEAN languages written in Ruby
+email:
+- v.satayamas@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- wordcut/dag.rb
+- wordcut/dict.rb
+- wordcut/dict_seek.rb
+- wordcut/edge.rb
+- wordcut/edge_builder.rb
+- wordcut/pointer.rb
+- wordcut/space_slicer.rb
+- wordcut/tokenizer.rb
+homepage: https://github.com/veer66/wordcut
+licenses:
+- LGPL-3.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- wordcut
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.3.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Word segmetation tools for ASEAN languages
+test_files: []