twitter_cldr 3.0.0.beta1 → 3.0.0

data/lib/twitter_cldr/formatters/numbers/percent_formatter.rb

@@ -6,23 +6,13 @@
 module TwitterCldr
   module Formatters
     class PercentFormatter < NumberFormatter
-      DEFAULT_PERCENT_SIGN = "%"
 
-      def format(number, options = {})
-        super(number, options).gsub('¤', @tokenizer.symbols[:percent_sign] || DEFAULT_PERCENT_SIGN)
-      end
+      DEFAULT_PERCENT_SIGN = "%"
 
-      def default_format_options_for(number)
-        { :precision => 0 }
+      def format(tokens, number, options = {})
+        super(tokens, number, options).gsub('¤', data_reader.symbols[:percent_sign] || DEFAULT_PERCENT_SIGN)
       end
 
-      def get_tokens(obj, options = {})
-        opts = options.dup.merge(
-          :sign => obj.abs == obj ? :positive : :negative,
-          :type => :percent
-        )
-        @tokenizer.tokens(opts)
-      end
     end
   end
 end

data/lib/twitter_cldr/formatters/numbers/rbnf/formatters.rb

@@ -0,0 +1,224 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Formatters
+    module Rbnf
+
+      class InvalidRbnfTokenError < StandardError; end
+
+      class RuleFormatter
+        class << self
+
+          attr_accessor :keep_soft_hyphens
+
+          def format(number, rule_set, rule_group, locale)
+            rule = rule_set.rule_for(number)
+            # puts "#{number}, #{rule.base_value}: '" + rule.rule_text + "'"
+            formatter = formatter_for(rule, rule_set, rule_group, locale)
+            result = formatter.format(number, rule)
+            keep_soft_hyphens ? result : remove_soft_hyphens(result)
+          end
+
+          def formatter_for(rule, rule_set, rule_group, locale)
+            const = case rule.base_value
+              when Rule::MASTER
+                MasterRuleFormatter
+              when Rule::IMPROPER_FRACTION
+                ImproperFractionRuleFormatter
+              when Rule::PROPER_FRACTION
+                ProperFractionRuleFormatter
+              when Rule::NEGATIVE
+                NegativeRuleFormatter
+              else
+                NormalRuleFormatter
+            end
+
+            const.new(rule_set, rule_group, locale)
+          end
+
+          private
+
+          def remove_soft_hyphens(result)
+            result.gsub([173].pack("U*"), "")
+          end
+
+        end
+
+        self.keep_soft_hyphens = true  # default value
+      end
+
+      class NormalRuleFormatter
+        attr_reader :rule_set, :rule_group, :omit, :is_fractional, :locale
+
+        def initialize(rule_set, rule_group, locale)
+          @rule_set = rule_set
+          @rule_group = rule_group
+          @is_fractional = false
+          @locale = locale
+        end
+
+        def format(number, rule)
+          rule.tokens.map do |token|
+            result = send(token.type, number, rule, token)
+            @omit ? "" : result
+          end.join
+        end
+
+        def right_arrow(number, rule, token)
+          # this seems to break things even though the docs require it:
+          # rule = rule_set.previous_rule_for(rule) if token.length == 3
+          remainder = number.abs % rule.divisor
+          generate_replacement(remainder, rule, token)
+        end
+
+        def left_arrow(number, rule, token)
+          quotient = number.abs / rule.divisor
+          generate_replacement(quotient, rule, token)
+        end
+
+        def equals(number, rule, token)
+          generate_replacement(number, rule, token)
+        end
+
+        def generate_replacement(number, rule, token)
+          if rule_set_name = token.rule_set_reference
+            RuleFormatter.format(
+              number,
+              rule_group.rule_set_for(rule_set_name),
+              rule_group,
+              locale
+            )
+          elsif decimal_format = token.decimal_format
+            @data_reader ||= TwitterCldr::DataReaders::NumberDataReader.new(locale)
+            @decimal_tokenizer ||= TwitterCldr::Tokenizers::NumberTokenizer.new(@data_reader)
+            decimal_tokens = @decimal_tokenizer.tokenize(decimal_format)
+            @decimal_formatter ||= TwitterCldr::Formatters::NumberFormatter.new(@data_reader)
+            @decimal_formatter.format(decimal_tokens, number, :type => :decimal)
+          else
+            RuleFormatter.format(number, rule_set, rule_group, locale)
+          end
+        end
+
+        def open_bracket(number, rule, token)
+          @omit = rule.even_multiple_of?(number)
+          ""
+        end
+
+        def close_bracket(number, rule, token)
+          @omit = false
+          ""
+        end
+
+        def plaintext(number, rule, token)
+          token.value
+        end
+
+        def semicolon(number, rule, token)
+          ""
+        end
+
+        protected
+
+        def invalid_token_error(token)
+          InvalidRbnfTokenError.new("'#{token.value}' not allowed in negative number rules.")
+        end
+
+        def fractional_part(number)
+          ".#{number.to_s.split(".")[1] || 0}".to_f
+        end
+
+        def integral_part(number)
+          number.to_s.split(".").first.to_i
+        end
+      end
+
+      class NegativeRuleFormatter < NormalRuleFormatter
+        def right_arrow(number, rule, token)
+          generate_replacement(number.abs, rule, token)
+        end
+
+        def left_arrow(number, rule, token)
+          raise invalid_token_error(token)
+        end
+
+        def open_bracket(number, rule, token)
+          raise invalid_token_error(token)
+        end
+
+        def close_bracket(number, rule, token)
+          raise invalid_token_error(token)
+        end
+      end
+
+      class MasterRuleFormatter < NormalRuleFormatter
+        def right_arrow(number, rule, token)
+          # Format by digits. This is not explained in the main doc. See:
+          # http://grepcode.com/file/repo1.maven.org/maven2/com.ibm.icu/icu4j/51.2/com/ibm/icu/text/NFSubstitution.java#FractionalPartSubstitution.%3Cinit%3E%28int%2Ccom.ibm.icu.text.NFRuleSet%2Ccom.ibm.icu.text.RuleBasedNumberFormat%2Cjava.lang.String%29
+
+          # doesn't seem to matter if the descriptor is two or three arrows, although three seems to indicate
+          # we should or should not be inserting spaces somewhere (not sure where)
+          is_fractional = true
+          number.to_s.split(".")[1].each_char.map do |digit|
+            RuleFormatter.format(digit.to_i, rule_set, rule_group, locale)
+          end.join(" ")
+        end
+
+        def left_arrow(number, rule, token)
+          if is_fractional
+            # is this necessary?
+            RuleFormatter.format(
+              (number * fractional_rule(number).base_value).to_i,
+              rule_set, rule_group, locale
+            )
+          else
+            generate_replacement(integral_part(number), rule, token)
+          end
+        end
+
+        def open_bracket(number, rule, token)
+          @omit = if is_fractional
+            # is this necessary?
+            (number * fractional_rule(number).base_value) == 1
+          else
+            # Omit the optional text if the number is an integer (same as specifying both an x.x rule and an x.0 rule)
+            @omit = number.is_a?(Integer)
+          end
+          ""
+        end
+
+        def close_bracket(number, rule, token)
+          @omit = false
+          ""
+        end
+
+        protected
+
+        def fractional_rule(number)
+          @fractional_rule ||= rule_set.rule_for(number, true)
+        end
+      end
+
+      class ProperFractionRuleFormatter < MasterRuleFormatter
+        def open_bracket(number, rule, token)
+          raise invalid_token_error(token)
+        end
+
+        def close_bracket(number, rule, token)
+          raise invalid_token_error(token)
+        end
+      end
+
+      class ImproperFractionRuleFormatter < MasterRuleFormatter
+        def open_bracket(number, rule, token)
+          # Omit the optional text if the number is between 0 and 1 (same as specifying both an x.x rule and a 0.x rule)
+          @omit = number > 0 && number < 1
+          ""
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/formatters/numbers/rbnf/post_processors/chinese.rb

@@ -0,0 +1,122 @@
+# This code was ported from the java version available here:
+# http://grepcode.com/file/repo1.maven.org/maven2/com.ibm.icu/icu4j/51.2/com/ibm/icu/text/RBNFChinesePostProcessor.java
+
+# This code is incomplete, untested, and unused. It should remain here until
+# I can figure out why it's necessary in ICU and wether to make use of it here or not.
+
+RULE_SET_NAMES = ["traditional", "simplified", "accounting", "time"]
+DIAN = 40670  # decimal point
+MARKERS = [
+  [33836, 20740, 20806, 12295], # marker chars, last char is the 'zero'
+  [19975, 20159, 20806, 12295],
+  [33836, 20740, 20806, 38646]
+  # need markers for time?
+]
+
+def process(str, rule_set) {
+  # markers depend on what rule set we are using
+  buf = str.unpack("U*")
+
+  name = rule_set.name
+  format = RULE_SET_NAMES.find_index { |rule_set_name| rule_set.name == rule_set_name }
+  long_form = format == 1 || format == 3
+
+  if long_form
+    i = buf.index("*".ord)
+    while i != -1
+      buf.delete(i...i + 1)
+      i = buf.index("*".ord)
+    end
+  else
+
+    # remove unwanted lings
+    # a '0' (ling) with * might be removed
+    # mark off 10,000 'chunks', markers are Z, Y, W (zhao, yii, and wan)
+    # already, we avoid two lings in the same chunk -- ling without * wins
+    # now, just need  to avoid optional lings in adjacent chunks
+    # process right to left
+
+    # decision matrix:
+    # state, situation
+    #     state         none       opt.          req.
+    #     -----         ----       ----          ----
+    # none to right     none       opt.          req.  
+    # opt. to right     none   clear, none  clear right, req.
+    # req. to right     none   clear, none       req.
+
+    # mark chunks with '|' for convenience
+    m = MARKERS[format]
+    0.upto(m.length - 2) do |i|
+      n = buf.index(m[i])
+      if n != -1
+        buf.insert(n + m[i].length, '|'.ord)
+      end
+    end
+
+    x = buf.index(DIAN)
+    x = buf.length if x == -1
+
+    s = 0   # 0 = none to right, 1 = opt. to right, 2 = req. to right
+    n = -1  # previous optional ling
+    ling = MARKERS[format][3]
+
+    while x >= 0
+      m = buf.rindex("|", x)
+      nn = buf.rindex(ling, x)
+      ns = 0
+
+      if nn > m
+        ns = (nn > 0 && buf[nn - 1] != '*'.ord) ? 2 : 1
+      end
+
+      x = m - 1
+
+      # actually much simpler, but leave this verbose for now so it's easier to follow
+      case s * 3 + ns
+        when 0: # none, none
+          s = ns # redundant
+          n = -1
+        when 1 # none, opt.
+          s = ns
+          n = nn # remember optional ling to right
+        when 2 # none, req.
+          s = ns
+          n = -1
+        when 3 # opt., none
+          s = ns
+          n = -1
+        when 4 # opt., opt.
+          # n + ling.length
+          buf.delete((nn - 1)...(nn + 1)) # delete current optional ling
+          s = 0
+          n = -1
+        when 5 # opt., req.
+          # n + ling.length
+          buf.delete((n - 1)...(n + 1)) # delete previous optional ling
+          s = ns
+          n = -1
+        when 6 # req., none
+          s = ns
+          n = -1
+        when 7 # req., opt.
+          # nn + ling.length
+          buf.delete((nn - 1)...(nn + 1)) # delete current optional ling
+          s = 0
+          n = -1
+        case 8 # req., req.
+          s = ns
+          n = -1
+        else
+          raise "Illegal state"
+      end
+    end
+
+    buf.length.downto(0) do |i|
+      if buf[i] == "*".ord || buf[i] == "|".ord
+        buf.delete(i...i + 1)
+      end
+    end
+  end
+
+  buf.pack("U*")
+end

data/lib/twitter_cldr/formatters/numbers/rbnf/rule.rb

@@ -0,0 +1,93 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Formatters
+    module Rbnf
+      class Rule
+
+        SUBSTITUTION_TYPES = [:equals, :left_arrow, :right_arrow]
+        MASTER = "x.0"
+        IMPROPER_FRACTION = "x.x"
+        PROPER_FRACTION = "0.x"
+        NEGATIVE = "-x"
+
+        attr_reader :base_value, :rule_text, :radix
+
+        def initialize(base_value, rule_text, radix)
+          @base_value = base_value
+          @rule_text = rule_text
+          @radix = radix
+        end
+
+        def divisor
+          @divisor ||= begin
+            val = base_value.to_i
+            exp = val > 0 ? (Math.log(val) / Math.log(radix || 10)).ceil : 1
+            div = exp >= 0 ? (radix || 10) ** exp : 1
+
+            # if result is too big, subtract one from exponent and try again
+            if div > val
+              (radix || 10) ** (exp - 1)
+            else
+              div
+            end
+          end
+        end
+
+        def substitution_count
+          @substitution_count ||= tokens.inject(0) do |ret, token|
+            token.is_a?(Substitution) ? ret + 1 : ret
+          end
+        end
+
+        def even_multiple_of?(num)
+          num % divisor == 0
+        end
+
+        def normal?
+          !(master? || improper_fraction? || proper_fraction? || negative?)
+        end
+
+        def master?
+          base_value == MASTER
+        end
+
+        def improper_fraction?
+          base_value == IMPROPER_FRACTION
+        end
+
+        def proper_fraction?
+          base_value == PROPER_FRACTION
+        end
+
+        def negative?
+          base_vaue == NEGATIVE
+        end
+
+        def tokens
+          @tokens ||= inline_substitutions(
+            tokenizer.tokenize(rule_text)
+          )
+        end
+
+        private
+
+        def inline_substitutions(tokens)
+          parser.parse(tokens)
+        end
+
+        def parser
+          @@parser ||= RuleParser.new
+        end
+
+        def tokenizer
+          @@tokenizer ||= TwitterCldr::Tokenizers::RbnfTokenizer.new
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/formatters/numbers/rbnf/rule_group.rb

@@ -0,0 +1,20 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Formatters
+    module Rbnf
+
+      RuleGroup = Struct.new(:rule_sets, :name) do
+        def rule_set_for(rule_set_name)
+          rule_sets.find do |rule_set|
+            rule_set.name == rule_set_name
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/formatters/numbers/rbnf/rule_parser.rb

@@ -0,0 +1,86 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Formatters
+    module Rbnf
+
+      class RuleParser < TwitterCldr::Parsers::Parser
+
+        private
+
+        def do_parse(options)
+          switch([])
+        end
+
+        def switch(list)
+          send(current_token.type, list)
+        end
+
+        def equals(list)
+          contents = descriptor(current_token)
+          list << Substitution.new(:equals, contents, 2)
+          next_token(:equals)
+          switch(list)
+        end
+
+        def left_arrow(list)
+          contents = descriptor(current_token)
+          list << Substitution.new(:left_arrow, contents, 2)
+          next_token(:left_arrow)
+          switch(list)
+        end
+
+        def right_arrow(list)
+          contents = descriptor(current_token)
+          sub = Substitution.new(:right_arrow, contents, 2)
+          next_token(:right_arrow)
+
+          # handle >>> case
+          if current_token.type == :right_arrow
+            sub.length += 1
+            next_token(:right_arrow)
+          end
+
+          list << sub
+          switch(list)
+        end
+
+        def plaintext(list)
+          add_and_advance(list)
+        end
+
+        def open_bracket(list)
+          add_and_advance(list)
+        end
+
+        def close_bracket(list)
+          add_and_advance(list)
+        end
+
+        def semicolon(list)
+          list
+        end
+
+        def add_and_advance(list)
+          list << current_token
+          next_token(current_token.type)
+          switch(list)
+        end
+
+        def descriptor(token)
+          next_token(token.type)
+          contents = []
+          until current_token.type == token.type
+            contents << current_token
+            next_token(current_token.type)
+          end
+          contents
+        end
+
+      end
+    end
+  end
+end