twitter_cldr_js 2.3.2 → 2.4.0

data/spec/js/parsers/segmentation_parser.spec.js

@@ -0,0 +1,67 @@
+// Copyright 2012 Twitter, Inc
+// http://www.apache.org/licenses/LICENSE-2.0
+
+var TwitterCldr = require('../../../lib/assets/javascripts/twitter_cldr/en.js');
+
+describe("SegmentationParser", function() {
+  var tokenizer = new TwitterCldr.SegmentationTokenizer();
+  var parser = new TwitterCldr.SegmentationParser();
+  var tokenize = function (text) {
+    return tokenizer.tokenize(text);
+  }
+  var parse = function (text, options) {
+    return parser.parse(text, options);
+  }
+  var symbol_table = new TwitterCldr.SymbolTable({"$FOO" : tokenize("[abc]")});
+  describe("#parse", function() {
+    it("should parse a rule with a break", function() {
+      var rule = parse(tokenize("[a-z] ÷ [0-9]"));
+      expect(rule.left.to_regexp_str()).toEqual("^(?:[\\u0061-\\u007a])");
+      expect(rule.right.to_regexp_str()).toEqual("^(?:[\\u0030-\\u0039])");
+      expect(rule.boundary_symbol).toEqual("break");
+    });
+    it("should parse a rule with a non-break", function() {
+      var rule = parse(tokenize("[a-z] × [0-9]"));
+      expect(rule.regex.to_regexp_str()).toEqual("^(?:[\\u0061-\\u007a])(?:[\\u0030-\\u0039])");
+      expect(rule.boundary_symbol).toEqual("no_break");
+    });
+    it("should parse a rule containing a variable", function() {
+      var rule = parse(tokenize("$FOO × bar"), {'symbol_table' : symbol_table});
+      expect(rule.regex.to_regexp_str()).toEqual("^(?:[\\u0061-\\u0063])(?:\\u0062)(?:\\u0061)(?:\\u0072)");
+      expect(rule.boundary_symbol).toEqual("no_break");
+    });
+  });
+  describe("SegmentationParser.BreakRule", function() {
+    var rule = parse(tokenize("[a-z] ÷ [0-9]"));
+    it("rule should be the right type", function() {
+      expect(rule instanceof TwitterCldr.SegmentationParser.BreakRule)
+    });
+    it("should match and return the right offset and text", function() {
+      var match = rule.match("c7");
+      expect(match.boundary_offset).toEqual(1);
+      expect(match.text).toEqual("c7");
+    });
+    it("should not match if the input string doesn't contain a matching right- and/or left-hand side", function() {
+      expect(rule.match("C7")).toBe(null);
+      expect(rule.match("cc")).toBe(null);
+      expect(rule.match("CC")).toBe(null);
+    });
+  });
+  describe("SegmentationParser.NoBreakRule", function() {
+    var rule = parse(tokenize("[a-z] × [0-9]"));
+    it("rule should be the right type", function() {
+      expect(rule instanceof TwitterCldr.SegmentationParser.NoBreakRule)
+    });
+    it("should match and returh the right offset and text", function() {
+      var match = rule.match("c7");
+      //non-break rules send you to the end of the match (maybe that's wrong?)
+      expect(match.boundary_offset).toEqual(2)
+      expect(match.text).toEqual("c7")
+    });
+    it("should not match if the input string doesn't contain matching text", function() {
+      expect(rule.match("C7")).toBe(null);
+      expect(rule.match("cc")).toBe(null);
+      expect(rule.match("CC")).toBe(null);
+    });
+  });
+});

data/spec/js/parsers/symbol_table.spec.js

@@ -0,0 +1,20 @@
+// Copyright 2012 Twitter, Inc
+// http://www.apache.org/licenses/LICENSE-2.0
+
+var TwitterCldr = require('../../../lib/assets/javascripts/twitter_cldr/en.js');
+
+describe("SymbolTable", function() {
+  beforeEach(function() {
+    table = new TwitterCldr.SymbolTable ({"a":"b", "c":"d"});
+  });
+  describe("#fetch", function() {
+    it("should be able to retrieve values for symbols", function() {
+      expect(table.fetch("a")).toEqual("b");
+      expect(table.fetch("z")).not.toBeDefined();
+    });
+    it("should be able to add then fetch new values for symbols", function() {
+      table.add("e", "f");
+      expect(table.fetch("e")).toEqual("f");
+    });
+  });
+});

data/spec/js/parsers/unicode_regex/character_class.spec.js

@@ -0,0 +1,121 @@
+// Copyright 2012 Twitter, Inc
+// http://www.apache.org/licenses/LICENSE-2.0
+
+var TwitterCldr = require('../../../../lib/assets/javascripts/twitter_cldr/en.js');
+
+describe("Character Class", function() {
+  var tokenizer = new TwitterCldr.UnicodeRegexTokenizer();
+  var parser = new TwitterCldr.UnicodeRegexParser();
+  var tokenize = function (text) {
+    return tokenizer.tokenize(text);
+  }
+  var parse = function (text, options) {
+    return parser.parse(text, options);
+  }
+  var char_class_from = function (elements) {
+    return elements[0];
+  }
+  describe("#to_set", function() {
+    it("unions together char classes with no explicit operator", function() {
+      var char_class = char_class_from(parse(tokenize("[[a][b]]")));
+      expect(char_class.to_set()).toEqualRangeSet(new TwitterCldr.RangeSet([new TwitterCldr.Range(97, 98)]));
+    });
+    it("unions together other entities within char classes when operator is not explicit", function() {
+      var char_class = char_class_from(parse(tokenize("[a-z0-9\\u0123]")));
+      expect(char_class.to_set()).toEqualRangeSet(new TwitterCldr.RangeSet([new TwitterCldr.Range(48, 57), new TwitterCldr.Range(97, 122), new TwitterCldr.Range(291, 291)]));
+    });
+    it("intersects correctly", function() {
+      var char_class = char_class_from(parse(tokenize("[[a-m]&[g-z]]")));
+      expect(char_class.to_set()).toEqualRangeSet(new TwitterCldr.RangeSet([new TwitterCldr.Range(103, 109)]));
+    });
+    it("finds symmetric differences correctly", function() {
+      var char_class = char_class_from(parse(tokenize("[[a-m]-[g-z]]")));
+      expect(char_class.to_set()).toEqualRangeSet(new TwitterCldr.RangeSet([new TwitterCldr.Range(97, 102), new TwitterCldr.Range(110, 122)]));      
+    });
+    it("computes sets for nested expressions", function() {
+      // (97..109) U (104..106)
+      // = (104..106)
+      // ((104..106) U (107..122)) subtr ((104..106) C (107..122))
+      // = (104..122) subtr ()
+      // = (104..122)
+      var char_class = char_class_from(parse(tokenize("[[[a-m]&[h-j]]-[k-z]]")));
+      expect(char_class.to_set()).toEqualRangeSet(new TwitterCldr.RangeSet([new TwitterCldr.Range(104, 122)]));
+    });
+    it("pulls in ranges for unicode character sets", function() {
+      var char_class = char_class_from(parse(tokenize("[\\p{Zs}]")));
+      expect(char_class.to_set()).toEqualRangeSet(new TwitterCldr.RangeSet([
+        new TwitterCldr.Range(32, 32),
+        new TwitterCldr.Range(160, 160),
+        new TwitterCldr.Range(5760, 5760),
+        new TwitterCldr.Range(6158, 6158),
+        new TwitterCldr.Range(8192, 8202),
+        new TwitterCldr.Range(8239, 8239),
+        new TwitterCldr.Range(8287, 8287),
+        new TwitterCldr.Range(12288, 12288)
+      ]));
+    });
+    it("computes unions between unicode character sets", function() {
+      var char_class = char_class_from(parse(tokenize("[[\\p{Zs}][\\p{Cc}]]")));
+      expect(char_class.to_set()).toEqualRangeSet(new TwitterCldr.RangeSet([
+        new TwitterCldr.Range(0, 1),
+        new TwitterCldr.Range(8, 32),
+        new TwitterCldr.Range(127, 160),
+        new TwitterCldr.Range(5760, 5760),
+        new TwitterCldr.Range(6158, 6158),
+        new TwitterCldr.Range(8192, 8202),
+        new TwitterCldr.Range(8239, 8239),
+        new TwitterCldr.Range(8287, 8287),
+        new TwitterCldr.Range(12288, 12288)
+      ]));
+    });
+    it("computes intersections between unicode character sets", function() {
+      var char_class = char_class_from(parse(tokenize("[[\\p{Zs}]&[\\u2000-\\u202B]]")));
+      expect(char_class.to_set()).toEqualRangeSet(new TwitterCldr.RangeSet([new TwitterCldr.Range(8192, 8202)]));
+    });
+    it("supports negating character sets", function() {
+      var char_class = char_class_from(parse(tokenize("[^\\u2000-\\u202B]")));
+      expect(char_class.to_set()).toEqualRangeSet(new TwitterCldr.RangeSet([
+        new TwitterCldr.Range(0, 1),
+        new TwitterCldr.Range(8, 8191),
+        new TwitterCldr.Range(8236, 55295),
+        new TwitterCldr.Range(57344, 65535), //1114111),
+      ]));
+    });
+    it("supports literal and excaped characters", function() {
+      var char_class = char_class_from(parse(tokenize("[abc\\edf\\g]")));
+      expect(char_class.to_set()).toEqualRangeSet(new TwitterCldr.RangeSet([new TwitterCldr.Range(97, 103)]));      
+    });
+    it("supports special switch characters", function() {
+      var char_class = char_class_from(parse(tokenize("[\\w]"))); // a-z, A-Z, 0-9, _
+      expect(char_class.to_set()).toEqualRangeSet(new TwitterCldr.RangeSet([
+        new TwitterCldr.Range(48, 57),
+        new TwitterCldr.Range(65, 90),
+        new TwitterCldr.Range(95, 95),
+        new TwitterCldr.Range(97, 122),
+      ]));
+    });
+    it("supports negated switch characters", function() {
+      char_class = char_class_from(parse(tokenize("[\\D]")))  // i.e. NOT \d
+      expect(char_class.to_set()).toEqualRangeSet(new TwitterCldr.RangeSet([
+        new TwitterCldr.Range(0, 1),
+        new TwitterCldr.Range(8, 47),
+        new TwitterCldr.Range(58, 55295),
+        new TwitterCldr.Range(57344, 65535), //1114111),
+      ]));
+    });
+  });
+  describe("#to_regexp_str", function() {
+    it("wraps ranges in square brackets", function() {
+      var char_class = char_class_from(parse(tokenize("[a-z]")));
+      expect(char_class.to_regexp_str()).toEqual("(?:[\\u0061-\\u007a])");
+    });
+    it("hex-encodes and wraps sequential characters to isolate bytes", function() {
+      var char_class = char_class_from(parse(tokenize("[{foo}]")));
+      expect(char_class.to_regexp_str()).toEqual("(?:(?:\\u0066)(?:\\u006f)(?:\\u006f))");
+    });
+    it("combines multiple components with 'or' pipe characters", function() {
+      var char_class = char_class_from(parse(tokenize("[{foo}abc]")));
+      expect(char_class.to_regexp_str()).toEqual("(?:(?:\\u0066)(?:\\u006f)(?:\\u006f)|[\\u0061-\\u0063])");
+    });
+  });
+});

data/spec/js/parsers/unicode_regex/character_range.spec.js

@@ -0,0 +1,17 @@
+// Copyright 2012 Twitter, Inc
+// http://www.apache.org/licenses/LICENSE-2.0
+
+var TwitterCldr = require('../../../../lib/assets/javascripts/twitter_cldr/en.js');
+
+describe("Character Range", function() {
+	describe("#to_set", function() {
+		it("should return a range between the initial and the final values", function() {
+			range = new TwitterCldr.CharacterRange (
+				new TwitterCldr.UnicodeString([97]),
+				new TwitterCldr.UnicodeString([98])
+			);
+
+			expect (range.to_set().to_array(true)).toEqualRangeArray([new TwitterCldr.Range(97,98)]);
+		});
+	});
+});

data/spec/js/parsers/unicode_regex/character_set.spec.js

@@ -0,0 +1,17 @@
+// Copyright 2012 Twitter, Inc
+// http://www.apache.org/licenses/LICENSE-2.0
+
+var TwitterCldr = require('../../../../lib/assets/javascripts/twitter_cldr/en.js');
+
+describe("Character Set", function() {
+	describe("#to_set", function() {
+		it("should return a set containing codepoints for the given general property", function() {
+			char_set = new TwitterCldr.CharacterSet ("Zs");
+			expect(char_set.to_set().to_array(true)).toEqualRangeArray([32, 160, 5760, 6158, new TwitterCldr.Range(8192,8202), 8239, 8287, 12288]);
+		});
+		it("should return a set containing codepoints for the given named property", function() {
+			char_set = new TwitterCldr.CharacterSet ("Sentence_Break=Sp");
+			expect(char_set.to_set().to_array(true)).toEqualRangeArray([9, new TwitterCldr.Range(11,12), 32, 160, 5760, new TwitterCldr.Range(8192,8202), 8239, 8287, 12288]);
+		});
+	});
+});

data/spec/js/parsers/unicode_regex/literal.spec.js

@@ -0,0 +1,30 @@
+// Copyright 2012 Twitter, Inc
+// http://www.apache.org/licenses/LICENSE-2.0
+
+var TwitterCldr = require('../../../../lib/assets/javascripts/twitter_cldr/en.js');
+
+describe("Literal", function() {
+  describe("#to_set", function() {
+    it("should set an array of tokens", function() {
+      literal = new TwitterCldr.Literal ("a");
+      expect(literal.to_set().to_array(true)).toEqual([97]);
+    });
+    it("should return escaped characters with no special meaning as codepoints", function() {
+      literal = new TwitterCldr.Literal ("\\a");
+      expect(literal.to_set().to_array(true)).toEqual([97]);
+    });
+    it("should convert special regex switches to their range equivalents", function() {
+      literal = new TwitterCldr.Literal ("\\d"); // digit
+      expect(literal.to_set()).toEqualRangeSet(new TwitterCldr.RangeSet([new TwitterCldr.Range(48,57)]));
+    });
+    it("should convert negated special regex switches to their range equivalents", function() {
+      literal = new TwitterCldr.Literal ("\\D"); // NOT digit
+      expect(literal.to_set()).toEqualRangeSet(new TwitterCldr.RangeSet([
+        new TwitterCldr.Range(0, 1),
+        new TwitterCldr.Range(8, 47),
+        new TwitterCldr.Range(58, 55295),
+        new TwitterCldr.Range(57344, 65535),//1114111),
+      ]));
+    });
+  });
+});

data/spec/js/parsers/unicode_regex/unicode_string.spec.js

@@ -0,0 +1,17 @@
+// Copyright 2012 Twitter, Inc
+// http://www.apache.org/licenses/LICENSE-2.0
+
+var TwitterCldr = require('../../../../lib/assets/javascripts/twitter_cldr/en.js');
+
+describe("Unicode String", function() {
+	describe("#to_set", function() {
+		it("should return a zero-length range when representing a single codepoint", function() {
+			str = new TwitterCldr.UnicodeString([97]);
+			expect (str.to_set().to_array()).toEqualRangeArray([new TwitterCldr.Range(97,97)]);
+		});
+		it("should return a range containing the codepoint array as both the first and last elements", function() {
+			str = new TwitterCldr.UnicodeString([97,98,99]);
+			expect (str.to_set().to_array()).toEqualRangeArray([new TwitterCldr.Range([97,98,99], [97,98,99])]);
+		});
+	});
+});

data/spec/js/parsers/unicode_regex_parser.spec.js

@@ -0,0 +1,76 @@
+// Copyright 2012 Twitter, Inc
+// http://www.apache.org/licenses/LICENSE-2.0
+
+var TwitterCldr = require('../../../lib/assets/javascripts/twitter_cldr/en.js');
+
+describe("UnicodeRegexParser", function() {
+  var tokenizer = new TwitterCldr.UnicodeRegexTokenizer();
+  var parser = new TwitterCldr.UnicodeRegexParser();
+  var tokenize = function (text) {
+    return tokenizer.tokenize(text);
+  }
+  var parse = function (text, options) {
+    return parser.parse(text, options);
+  }
+
+  describe("#parse", function() {
+    it("identifies ranges", function() {
+      var elements = parse(tokenize("[a-z]"));
+      expect(elements[0] instanceof TwitterCldr.CharacterClass).toBe(true);
+      var root = elements[0].root;
+      expect(root instanceof TwitterCldr.CharacterRange).toBe(true);
+      expect(root.initial.codepoints).toEqual(TwitterCldr.Utilities.unpack_string("a"));
+      expect(root.final.codepoints).toEqual(TwitterCldr.Utilities.unpack_string("z"));
+    });
+    it("replaces variables", function() {
+      var symbol_table = new TwitterCldr.SymbolTable({"$VAR" : tokenize("\\p{L}")});
+      var elements = parse(tokenize("($VAR)?"), {'symbol_table' : symbol_table});
+      expect(elements[1] instanceof TwitterCldr.CharacterSet).toBe(true);
+      expect(elements[1].property_value).toEqual("L");
+    });
+    it("handles character and negated character sets", function() {
+      var elements = parse(tokenize("\\p{L}[:^P:]\\P{L}[:P:]"));
+
+      var element = elements[0];
+      expect(element instanceof TwitterCldr.CharacterSet).toBe(true);
+      expect(element.property_value).toEqual("L");
+
+      element = elements[1];
+      expect(element instanceof TwitterCldr.CharacterClass).toBe(true);
+      expect(element.root.child.property_value).toEqual("P");
+      expect(element.root.operator).toEqual("negate");
+
+      element = elements[2];
+      expect(element instanceof TwitterCldr.CharacterClass).toBe(true);
+      expect(element.root.child.property_value).toEqual("L");      
+
+      element = elements[3];
+      expect(element instanceof TwitterCldr.CharacterSet).toBe(true);
+      expect(element.property_value).toEqual("P");
+    });
+    it("handles unicode characters", function() {
+      var elements = parse(tokenize("\\u0123"));
+      expect(elements[0] instanceof TwitterCldr.UnicodeString).toBe(true);
+      expect(elements[0].codepoints).toEqual([291]);
+    });
+    it("handles multichar and escaped unicode strings", function() {
+      var elements = parse(tokenize("\\g{abc}"));
+      expect(elements[0] instanceof TwitterCldr.Literal).toBe(true);
+      expect(elements[0].text).toEqual("\\g");
+      expect(elements[1] instanceof TwitterCldr.UnicodeString).toBe(true);
+      expect(elements[1].codepoints).toEqual([97, 98, 99]);
+    });
+    it("handles special chars", function() {
+      var elements = parse(tokenize("^(?:)$"));
+      for (var i = 0; i < elements.length; i ++) {
+        expect(elements[i] instanceof TwitterCldr.Literal).toBe(true);
+      }
+      expect(elements[0].text).toEqual("^");
+      expect(elements[1].text).toEqual("(");
+      expect(elements[2].text).toEqual("?");
+      expect(elements[3].text).toEqual(":");
+      expect(elements[4].text).toEqual(")");
+      expect(elements[5].text).toEqual("$");
+    });
+  });
+});

data/spec/js/plurals/plural_rules.spec.js

@@ -8,6 +8,17 @@ describe("PluralRules", function() {
     it("returns an array of all English plural rules", function() {
       expect(TwitterCldr.PluralRules.all()).toEqual(["one", "other"]);
     });
+
+    it("returns an array of the plural rules for the given type", function() {
+      var actual_rules = TwitterCldr.PluralRules.all('ordinal');
+      var expected_rules = ["one", "two", "few", "other"];
+
+      expect(actual_rules.length).toEqual(expected_rules.length);
+
+      for (actual_rule_idx in actual_rules) {
+        expect(expected_rules).toContain(actual_rules[actual_rule_idx]);
+      }
+    });
   });
 
   describe("#rule_for", function() {
@@ -24,5 +35,15 @@ describe("PluralRules", function() {
     it("returns 'other' for the number 0", function() {
       expect(TwitterCldr.PluralRules.rule_for(0)).toEqual("other");
     });
+
+    it("returns correct ordinal plurals", function() {
+      expect(TwitterCldr.PluralRules.rule_for(1, 'ordinal')).toEqual("one");
+      expect(TwitterCldr.PluralRules.rule_for(2, 'ordinal')).toEqual("two");
+      expect(TwitterCldr.PluralRules.rule_for(3, 'ordinal')).toEqual("few");
+      expect(TwitterCldr.PluralRules.rule_for(4, 'ordinal')).toEqual("other");
+      expect(TwitterCldr.PluralRules.rule_for(11, 'ordinal')).toEqual("other");
+      expect(TwitterCldr.PluralRules.rule_for(12, 'ordinal')).toEqual("other");
+      expect(TwitterCldr.PluralRules.rule_for(22, 'ordinal')).toEqual("two");
+    });
   });
 });

data/spec/js/shared/break_iterator.spec.js

@@ -0,0 +1,68 @@
+// Copyright 2012 Twitter, Inc
+// http://www.apache.org/licenses/LICENSE-2.0
+
+var TwitterCldr = require('../../../lib/assets/javascripts/twitter_cldr/en.js');
+
+describe("BreakIterator", function() {
+  var iterator = new TwitterCldr.BreakIterator ("en", {"use_uli_exceptions" : true});
+  describe("#each_sentence", function() {
+    it("should return an array", function() {
+      expect(iterator.each_sentence("foo bar") instanceof Array).toBe(true);
+    });
+
+    it("splits a simple string into sentences", function() {
+      var str = "The. Quick. Brown. Fox.";
+      expect(iterator.each_sentence(str)).toEqual([
+        "The.", " Quick.", " Brown.", " Fox."
+      ]);
+    });
+
+    it("does not split on commas, for example", function() {
+      var str = "The. Quick, brown. Fox.";
+      expect(iterator.each_sentence(str)).toEqual([
+        "The.", " Quick, brown.", " Fox."
+      ]);
+    });
+
+    it("does not split periods in the midst of other letters, eg. in a URL", function() {
+      var str = "Visit us. Go to http://translate.twitter.com.";
+      expect(iterator.each_sentence(str)).toEqual([
+        "Visit us.",
+        " Go to http://translate.twitter.com."
+      ]);
+    });
+
+    it("splits on sentences that end with other kinds of punctuation", function() {
+      var str = "Help us translate! Speak another language? You really, really rock.";
+      expect(iterator.each_sentence(str)).toEqual([
+        "Help us translate!",
+        " Speak another language?",
+        " You really, really rock."
+      ]);
+    });
+
+    describe("with ULI exceptions", function() {
+      it("does not split on certain abbreviations like Mr. and Mrs.", function() {
+        var str = "I really like Mrs. Patterson. She's nice.";
+        expect(iterator.each_sentence(str)).toEqual([
+          "I really like Mrs. Patterson.",
+          " She's nice."
+        ]);
+      });
+    });
+
+    describe("without ULI exceptions", function() {
+      var iterator = new TwitterCldr.BreakIterator ("en", {"use_uli_exceptions" : false});
+      it("splits on certain abbreviations like Mr. and Mrs. (use ULI rules to avoid this behavior)", function() {
+        var str = "I really like Mrs. Patterson. She's nice.";
+        expect(iterator.each_sentence(str)).toEqual([
+          "I really like Mrs.",
+          " Patterson.",
+          " She's nice."
+        ]);
+      });
+    });
+  });
+});
+
+