twitter_cldr 5.1.0 → 5.2.0

data/lib/twitter_cldr/segmentation/cursor.rb

@@ -6,10 +6,12 @@
 module TwitterCldr
   module Segmentation
     class Cursor
-      attr_reader :text, :position, :match_cache
+      attr_reader :text, :codepoints
+      attr_accessor :position
 
       def initialize(text)
         @text = text
+        @codepoints = text.codepoints
         reset
       end
 
@@ -19,15 +21,18 @@ module TwitterCldr
 
       def reset
         @position = 0
-        @match_cache = {}
       end
 
-      def eof?
+      def eos?
         position >= text.size
       end
 
-      def eos?
-        position >= text.size - 1
+      def codepoint(pos = position)
+        codepoints[pos]
+      end
+
+      def length
+        text.length
       end
     end
   end

data/lib/twitter_cldr/segmentation/metadata.rb

@@ -0,0 +1,20 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Segmentation
+    class Metadata
+      attr_reader :values
+
+      def initialize(values)
+        @values = values
+      end
+
+      def category_count
+        @category_count ||= values[:category_count]
+      end
+    end
+  end
+end

data/lib/twitter_cldr/segmentation/null_suppressions.rb

@@ -0,0 +1,18 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'singleton'
+
+module TwitterCldr
+  module Segmentation
+    class NullSuppressions
+      include Singleton
+
+      def should_break?(_cursor)
+        true
+      end
+    end
+  end
+end

data/lib/twitter_cldr/segmentation/rule_set.rb

@@ -8,109 +8,53 @@ module TwitterCldr
     class RuleSet
 
       class << self
-        def load(*args)
-          RuleSetBuilder.load(*args)
+        def create(locale, boundary_type, options = {})
+          new(locale, StateMachine.instance(boundary_type, locale), options)
         end
       end
 
-      attr_reader :locale, :rules, :boundary_type
+      attr_reader :locale, :state_machine
       attr_accessor :use_uli_exceptions
 
       alias_method :use_uli_exceptions?, :use_uli_exceptions
 
-      def initialize(locale, rules, boundary_type, options)
+      def initialize(locale, state_machine, options)
         @locale = locale
-        @rules = rules
-        @boundary_type = boundary_type
+        @state_machine = state_machine
         @use_uli_exceptions = options.fetch(
           :use_uli_exceptions, false
         )
       end
 
       def each_boundary(str)
-        if block_given?
-          cursor = Cursor.new(str)
-          last_boundary = 0
+        return to_enum(__method__, str) unless block_given?
 
-          # implicit start of text boundary
-          yield 0
+        cursor = Cursor.new(str)
 
-          until cursor.eof?
-            match = find_match(cursor)
-            rule = match.rule
+        # Let the state machine find the first boundary for the line
+        # boundary type. This helps pass nearly all the Unicode
+        # segmentation tests, so it must be the right thing to do.
+        # Normally the first boundary is the implicit start of text
+        # boundary, but potentially not for the line rules?
+        yield 0 unless state_machine.boundary_type == 'line'
 
-            if rule.break?
-              yield match.boundary_position
-              last_boundary = match.boundary_position
-            end
-
-            if match.boundary_position == cursor.position
-              cursor.advance
-            else
-              cursor.advance(
-                match.boundary_position - cursor.position
-              )
-            end
-          end
-
-          # implicit end of text boundary
-          yield str.size unless last_boundary == str.size
-        else
-          to_enum(__method__, str)
+        until cursor.eos?
+          state_machine.handle_next(cursor)
+          yield cursor.position if suppressions.should_break?(cursor)
         end
       end
 
-      private
-
-      def each_rule(&block)
-        if block_given?
-          if use_uli_exceptions? && supports_exceptions?
-            yield exception_rule
-          end
-
-          rules.each(&block)
-        else
-          to_enum(__method__)
-        end
-      end
-
-      def exception_rule
-        @exception_rule ||= RuleSetBuilder.exception_rule_for(
-          locale, boundary_type
-        )
-      end
-
-      def supports_exceptions?
-        boundary_type == 'sentence'
+      def boundary_type
+        state_machine.boundary_type
       end
 
-      def find_match(cursor)
-        match = find_cached_match(cursor)
+      private
 
-        match || if cursor.eos?
-          RuleSetBuilder.implicit_end_of_text_rule.match(cursor)
+      def suppressions
+        @suppressions ||= if use_uli_exceptions?
+          Suppressions.instance(boundary_type, locale)
         else
-          RuleSetBuilder.implicit_final_rule.match(cursor)
-        end
-      end
-
-      def find_cached_match(cursor)
-        cursor.match_cache.fetch(cursor.position) do
-          matches = match_all(cursor)
-
-          matches.each do |m|
-            cursor.match_cache[m.boundary_position - 1] ||= m
-          end
-
-          matches.first
-        end
-      end
-
-      def match_all(cursor)
-        each_rule.each_with_object([]) do |rule, ret|
-          if match = rule.match(cursor)
-            ret << match
-          end
+          NullSuppressions.instance
         end
       end
     end

data/lib/twitter_cldr/segmentation/state_machine.rb

@@ -0,0 +1,125 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'base64'
+require 'singleton'
+
+module TwitterCldr
+  module Segmentation
+    class StateMachine
+      include Singleton
+
+      START_STATE = 1
+      STOP_STATE = 0
+      NEXT_STATES = 4
+      ACCEPTING = 0
+
+      class << self
+        def instance(boundary_type, locale)
+          resource_path = find_resource(boundary_type, locale)
+
+          cache[resource_path] ||= begin
+            rsrc = TwitterCldr.get_resource(resource_path)
+
+            new(
+              boundary_type,
+              locale,
+              Metadata.new(rsrc[:metadata]),
+              StateTable.load16(rsrc[:forward_table]),
+              StateTable.load16(rsrc[:backward_table]),
+              StatusTable.load(rsrc[:status_table]),
+              CategoryTable.load16(rsrc[:category_table])
+            )
+          end
+        end
+
+        private
+
+        def find_resource(boundary_type, locale)
+          path = TwitterCldr.resource_file_path(
+            ['shared', 'segments', 'rules', locale, boundary_type]
+          )
+
+          return path if TwitterCldr.resource_exists?(path)
+
+          TwitterCldr.resource_file_path(
+            ['shared', 'segments', 'rules', 'root', boundary_type]
+          )
+        end
+
+        def cache
+          @cache ||= {}
+        end
+      end
+
+      attr_reader :boundary_type, :locale
+      attr_reader :metadata, :ftable, :rtable, :status_table, :category_table
+
+      def initialize(boundary_type, locale, metadata, ftable, rtable, status_table, category_table)
+        @boundary_type = boundary_type
+        @locale = locale
+        @metadata = metadata
+        @ftable = ftable
+        @rtable = rtable
+        @status_table = status_table
+        @category_table = category_table
+      end
+
+      def handle_next(cursor)
+        result = initial_position = cursor.position
+        state = START_STATE
+        row = row_index_for(state)
+        category = 3
+        mode = :run
+
+        if ftable.bof_required?
+          category = 2
+          mode = :start
+        end
+
+        until state == STOP_STATE
+          if cursor.eos?
+            break if mode == :stop
+            mode = :stop
+            category = 1
+          elsif mode == :run
+            category = category_table.get(cursor.codepoint)
+
+            if (category & 0x4000) != 0
+              category &= ~0x4000
+            end
+
+            cursor.advance
+          else
+            mode = :run
+          end
+
+          state = ftable[row + NEXT_STATES + category]
+          row = row_index_for(state)
+
+          if ftable[row + ACCEPTING] == -1
+            # match found
+            result = cursor.position
+          end
+        end
+
+        cursor.position = result
+
+        # don't let cursor get stuck
+        if cursor.position == initial_position
+          cursor.advance
+        end
+
+        result
+      end
+
+      private
+
+      def row_index_for(state)
+        state * (metadata.category_count + 4)
+      end
+    end
+  end
+end

data/lib/twitter_cldr/segmentation/state_table.rb

@@ -0,0 +1,46 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'base64'
+
+module TwitterCldr
+  module Segmentation
+    class StateTable
+      PACK_FMT_16 = 's!*'.freeze
+      BOF_REQUIRED_FLAG = 2
+
+      class << self
+        def load16(hash)
+          new(
+            Base64.decode64(hash[:table]).unpack(PACK_FMT_16),
+            hash[:flags]
+          )
+        end
+      end
+
+      attr_reader :values, :flags
+
+      def initialize(values, flags)
+        @values = values
+        @flags = flags
+      end
+
+      def [](idx)
+        values[idx]
+      end
+
+      def bof_required?
+        flags & BOF_REQUIRED_FLAG != 0
+      end
+
+      def dump16
+        {
+          table: Base64.encode64(values.pack(PACK_FMT_16)).strip,
+          flags: flags
+        }
+      end
+    end
+  end
+end

data/lib/twitter_cldr/segmentation/status_table.rb

@@ -0,0 +1,30 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'base64'
+
+module TwitterCldr
+  module Segmentation
+    class StatusTable
+      PACK_FMT = 'I!*'.freeze
+
+      class << self
+        def load(hash)
+          new(Base64.decode64(hash[:table]).unpack(PACK_FMT))
+        end
+      end
+
+      attr_reader :values
+
+      def initialize(values)
+        @values = values
+      end
+
+      def dump
+        { table: Base64.encode64(values.pack(PACK_FMT)).strip }
+      end
+    end
+  end
+end

data/lib/twitter_cldr/segmentation/suppressions.rb

@@ -0,0 +1,79 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'singleton'
+
+module TwitterCldr
+  module Segmentation
+    class Suppressions
+      include Singleton
+
+      class << self
+        def instance(boundary_type, locale)
+          resource_path = find_resource(boundary_type, locale)
+          return NullSuppressions.instance unless resource_path
+
+          cache[resource_path] ||= begin
+            rsrc = TwitterCldr.get_resource(resource_path)
+
+            new(
+              Marshal.load(rsrc[:forwards_trie]),
+              Marshal.load(rsrc[:backwards_trie])
+            )
+          end
+        end
+
+        private
+
+        def find_resource(boundary_type, locale)
+          path = TwitterCldr.resource_file_path(
+            ['shared', 'segments', 'suppressions', locale, boundary_type]
+          )
+
+          path if TwitterCldr.resource_exists?(path)
+        end
+
+        def cache
+          @cache ||= {}
+        end
+      end
+
+      attr_reader :forward_trie, :backward_trie
+
+      def initialize(forward_trie, backward_trie)
+        @forward_trie = forward_trie
+        @backward_trie = backward_trie
+      end
+
+      def should_break?(cursor)
+        idx = cursor.position
+
+        # consider case when a space follows the '.' (so we handle i.e. "Mr. Brown")
+        idx -= 2 if cursor.codepoint(idx - 1) == 32
+        node = backward_trie.root
+
+        found = loop do
+          break false if idx < 0 || idx >= cursor.length
+          node = node.child(cursor.codepoint(idx))
+          break false unless node
+          break true if node.value
+          idx -= 1
+        end
+
+        return true unless found
+
+        node = forward_trie.root
+
+        loop do
+          return true if idx >= cursor.length
+          node = node.child(cursor.codepoint(idx))
+          return true unless node
+          return false if node.value
+          idx += 1
+        end
+      end
+    end
+  end
+end