twitter_cldr 3.0.1 → 3.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d66b9de7cc80964cf816c2981bf6aac51ec26134
-  data.tar.gz: e5039d2b57b98ea896830872a6d6e2ea72229d9f
+  metadata.gz: a4b16ad78f734a596fa4488a61e58b0e971cf2ce
+  data.tar.gz: 1179820a11c171adab4e841cf14efe0e6534a6a7
 SHA512:
-  metadata.gz: ba0506579dd20f7d92d480c4d46834bb25eae55a622e3bb28ffdb59cc3039a9c74f7571072a4b38686620e56b7a7c8fccade116a9ce4cfd5e16c269ad6819c66
-  data.tar.gz: 06a830ff2fa877e6ee2e98d0d3acbfced0e7529181418c247e56b07980750acd364cbdc93fbf247218f022ee3557fdae0b5f92f228e2cca1abae52fffb363ee2
+  metadata.gz: e4268cc2df5dda027c6b3d2fad2398a0ac5f0597fc620931f0e86cc4dabc775a3d6d0b731290a042ec1114f088244bbd8cc457f2a36066ba8c7c5d8b2810f0b3
+  data.tar.gz: 635c217f690dabc37991f4eff3d215e1312c66a543b2aa7e665e0e3469f0cd10bd37fee2452901ba8c5665f331b0cac2cd059a93f451f03f9118dfc1b97e1217

data/Gemfile

@@ -14,6 +14,8 @@ group :development, :test do
   if RUBY_VERSION <= "1.8.7"
     gem 'oniguruma'
   end
+
+  gem 'regexp_parser', '~> 0.1.5'
 end
 
 group :development do
@@ -25,8 +27,8 @@ group :development do
 end
 
 group :test do
-  gem 'rspec', '~> 2.11.0'
-  gem 'rr',    '~> 1.0.4'
+  gem 'rspec', '~> 2.14.0'
+  gem 'rr',    '~> 1.1.2'
 
   if RUBY_VERSION >= "1.9"
     gem 'rubyzip'

data/History.txt

@@ -1,3 +1,7 @@
+== 3.0.2
+
+* Adding ability to generate sample postal codes from their regexes.
+
 == 3.0.1
 
 * Fixing abbreviated timespan formats for en-GB (backport from 2.4.3).

data/README.md

@@ -439,17 +439,21 @@ The CLDR contains postal code validation regexes for a number of countries.
 
 ```ruby
 # United States
-TwitterCldr::Shared::PostalCodes.valid?(:us, "94103")     # true
-TwitterCldr::Shared::PostalCodes.valid?(:us, "9410")      # false
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:us) 
+postal_code.valid?("94103")     # true
+postal_code.valid?("9410")      # false
 
 # England (Great Britain)
-TwitterCldr::Shared::PostalCodes.valid?(:gb, "BS98 1TL")  # true
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:gb) 
+postal_code.valid?("BS98 1TL")  # true
 
 # Sweden
-TwitterCldr::Shared::PostalCodes.valid?(:se, "280 12")    # true
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:se) 
+postal_code.valid?("280 12")    # true
 
 # Canada
-TwitterCldr::Shared::PostalCodes.valid?(:ca, "V3H 1Z7")   # true
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:ca) 
+postal_code.valid?("V3H 1Z7")   # true
 ```
 
 Get a list of supported territories by using the `#territories` method:
@@ -461,7 +465,14 @@ TwitterCldr::Shared::PostalCodes.territories  # [:ad, :am, :ar, :as, :at, ... ]
 Just want the regex?  No problem:
 
 ```ruby
-TwitterCldr::Shared::PostalCodes.regex_for_territory(:us)  # /\d{5}([ \-]\d{4})?/
+postal_code = TwitterCldr::Shared::PostalCodes.for_territory(:us) 
+postal_code.regexp  # /\d{5}([ \-]\d{4})?/
+```
+
+Get a sample of valid postal codes with the `#sample` method:
+
+```ruby
+postal_code.sample(5)  # ["83526", "31748-8754", "55851", "25788-4914", "55335"]
 ```
 
 ### Phone Codes

data/lib/twitter_cldr/resources/postal_codes_importer.rb

@@ -35,7 +35,18 @@ module TwitterCldr
 
         postal_codes = Hash[postal_codes.sort_by(&:first)]
 
-        File.open(File.join(@output_path, 'postal_codes.yml'), 'w') { |output| output.write(YAML.dump(postal_codes)) }
+        postal_codes = postal_codes.each_with_object({}) do |(territory, regex), memo|
+          memo[territory] = {
+            :regex => regex,
+            :ast => TwitterCldr::Utils::RegexpAst.dump(
+              RegexpAstGenerator.generate(regex.source)
+            )
+          }
+        end
+
+        File.open(File.join(@output_path, 'postal_codes.yml'), 'w') do |output|
+          output.write(YAML.dump(postal_codes))
+        end
       end
 
     end

data/lib/twitter_cldr/resources/regexp_ast_generator.rb

@@ -0,0 +1,41 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'regexp_parser'
+
+module TwitterCldr
+  module Resources
+
+    class RegexpAstGenerator
+      class << self
+
+        def generate(regexp_str)
+          tree = Regexp::Parser.parse(regexp_str)
+          walk(tree)
+        end
+
+        private
+
+        def walk(node)
+          expressions = if node.respond_to?(:expressions)
+            node.expressions.map { |expr| walk(expr) }
+          else
+            []
+          end
+
+          class_for(node).from_parser_node(node, expressions)
+        end
+
+        def class_for(klass)
+          TwitterCldr::Utils::RegexpAst.const_get(
+            klass.class.to_s.split("::").last.to_sym
+          )
+        end
+
+      end
+    end
+
+  end
+end

data/lib/twitter_cldr/resources.rb

@@ -26,5 +26,6 @@ module TwitterCldr
     autoload :RbnfTestImporter,                   'twitter_cldr/resources/rbnf_test_importer'
     autoload :ReadmeRenderer,                     'twitter_cldr/resources/readme_renderer'
     autoload :CasefolderClassGenerator,           'twitter_cldr/resources/casefolder_class_generator'
+    autoload :RegexpAstGenerator,                 'twitter_cldr/resources/regexp_ast_generator'
   end
 end

data/lib/twitter_cldr/shared/postal_code_generator.rb

@@ -0,0 +1,50 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'set'
+
+module TwitterCldr
+  module Shared
+    class PostalCodeGenerator
+
+      SAMPLE_MULTIPLIER = 4
+
+      def initialize(regexp_ast)
+        @regexp_generator = TwitterCldr::Utils::RegexpSampler.new(regexp_ast)
+      end
+
+      def generate
+        clean_result(@regexp_generator.generate)
+      end
+
+      def sample(sample_size = 1)
+        sample_set = Set.new
+        counter = 1
+
+        until sample_set.size == sample_size
+          sample = generate
+          sample_set << sample unless sample.empty?
+          counter += 1
+
+          # Stop if the number of attempted generations is
+          # n times more than requested. Some territories only
+          # have one postal code, so if the user asks for 10
+          # they'll get an infinite loop.
+          break if counter > sample_size * SAMPLE_MULTIPLIER
+        end
+
+        sample_set.to_a
+      end
+
+      private
+
+      # remove spaces that trail a dash
+      def clean_result(str)
+        str.gsub(/- /, '-')
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/shared/postal_codes.rb

@@ -5,7 +5,10 @@
 
 module TwitterCldr
   module Shared
-    module PostalCodes
+
+    class InvalidTerritoryError < StandardError; end
+
+    class PostalCodes
 
       class << self
 
@@ -13,23 +16,59 @@ module TwitterCldr
           resource.keys
         end
 
-        def regex_for_territory(territory)
-          resource[territory.to_s.downcase.to_sym]
-        end
-
-        def valid?(territory, postal_code)
-          regexp = regex_for_territory(territory)
-          !!(regexp && regexp =~ postal_code)
+        def for_territory(territory)
+          key = territory.to_s.downcase.to_sym
+          if res = resource[key]
+            territory_cache[key] ||= begin
+              new(
+                territory,
+                res[:regex],
+                TwitterCldr::Utils::RegexpAst.load(res[:ast])
+              )
+            end
+          else
+            raise InvalidTerritoryError, "invalid territory"
+          end
         end
 
         private
 
+        def territory_cache
+          @territory_cache ||= {}
+        end
+
         def resource
           @resource ||= TwitterCldr.get_resource(:shared, :postal_codes)
         end
 
       end
 
+      attr_reader :territory, :regexp, :ast
+
+      def initialize(territory, regexp, ast)
+        @territory = territory
+        @regexp = regexp
+        @ast = ast
+      end
+
+      def valid?(postal_code)
+        !!(regexp && regexp =~ postal_code)
+      end
+
+      def sample(sample_size = 1)
+        generator.sample(sample_size)
+      end
+
+      private
+
+      def generator
+        generator_cache[territory] ||= PostalCodeGenerator.new(ast)
+      end
+
+      def generator_cache
+        @@generator_cache ||= {}
+      end
+
     end
   end
-end
+end

data/lib/twitter_cldr/shared.rb

@@ -5,19 +5,20 @@
 
 module TwitterCldr
   module Shared
-    autoload :Calendar,        'twitter_cldr/shared/calendar'
-    autoload :CodePoint,       'twitter_cldr/shared/code_point'
-    autoload :Currencies,      'twitter_cldr/shared/currencies'
-    autoload :LanguageCodes,   'twitter_cldr/shared/language_codes'
-    autoload :Languages,       'twitter_cldr/shared/languages'
-    autoload :Numbers,         'twitter_cldr/shared/numbers'
-    autoload :PhoneCodes,      'twitter_cldr/shared/phone_codes'
-    autoload :PostalCodes,     'twitter_cldr/shared/postal_codes'
-    autoload :Bidi,            'twitter_cldr/shared/bidi'
-    autoload :Territories,     'twitter_cldr/shared/territories'
-    autoload :NumberingSystem, 'twitter_cldr/shared/numbering_system'
-    autoload :Casefolder,      'twitter_cldr/shared/casefolder'
-    autoload :UnicodeRegex,    'twitter_cldr/shared/unicode_regex'
-    autoload :BreakIterator,   'twitter_cldr/shared/break_iterator'
+    autoload :Calendar,            'twitter_cldr/shared/calendar'
+    autoload :CodePoint,           'twitter_cldr/shared/code_point'
+    autoload :Currencies,          'twitter_cldr/shared/currencies'
+    autoload :LanguageCodes,       'twitter_cldr/shared/language_codes'
+    autoload :Languages,           'twitter_cldr/shared/languages'
+    autoload :Numbers,             'twitter_cldr/shared/numbers'
+    autoload :PhoneCodes,          'twitter_cldr/shared/phone_codes'
+    autoload :PostalCodes,         'twitter_cldr/shared/postal_codes'
+    autoload :PostalCodeGenerator, 'twitter_cldr/shared/postal_code_generator'
+    autoload :Bidi,                'twitter_cldr/shared/bidi'
+    autoload :Territories,         'twitter_cldr/shared/territories'
+    autoload :NumberingSystem,     'twitter_cldr/shared/numbering_system'
+    autoload :Casefolder,          'twitter_cldr/shared/casefolder'
+    autoload :UnicodeRegex,        'twitter_cldr/shared/unicode_regex'
+    autoload :BreakIterator,       'twitter_cldr/shared/break_iterator'
   end
 end

data/lib/twitter_cldr/utils/regexp_ast.rb

@@ -0,0 +1,115 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+require 'base64'
+
+module TwitterCldr
+  module Utils
+    module RegexpAst
+
+      def self.load(ast_str)
+        Marshal.load(Base64.decode64(ast_str))
+      end
+
+      def self.dump(ast)
+        Base64.encode64(Marshal.dump(ast))
+      end
+
+      class Node
+        attr_reader :expressions, :quantifier
+
+        def initialize(expressions, quantifier)
+          @expressions = expressions
+          @quantifier = quantifier
+        end
+
+        def quantified?
+          !!quantifier
+        end
+
+        def self.from_parser_node(node, expressions)
+          new(expressions, Quantifier.from_parser_node(node))
+        end
+      end
+
+      class CharacterSet < Node
+        attr_reader :members, :negated
+        alias :negated? :negated
+
+        def initialize(expressions, quantifier, members, negated)
+          @members = members; @negated = negated
+          super(expressions, quantifier)
+        end
+
+        def self.from_parser_node(node, expressions)
+          new(
+            expressions, Quantifier.from_parser_node(node),
+            fix_members(node.members), node.negative?
+          )
+        end
+
+        private
+
+        # CLDR occasionally uses \d and other escapes in character classes
+        # to signify 0-9 and friends. This is legal regex syntax, but the
+        # regexp_parser gem doesn't handle it correctly, so we have to
+        # repair things here.
+        def self.fix_members(members)
+          members.join.scan(/(\\[wd]|\w-\w|\w|-)/).to_a.flatten.inject([]) do |ret, member|
+            case member
+              when '\d' then ret << '0-9'
+              when '\w' then ret += ['A-Z', 'a-z', '0-9', '_']
+              else ret << member
+            end
+
+            ret
+          end
+        end
+      end
+
+      class Literal < Node
+        attr_reader :text
+
+        def initialize(expressions, quantifier, text)
+          @text = text
+          super(expressions, quantifier)
+        end
+
+        def self.from_parser_node(node, expressions)
+          new(
+            expressions, Quantifier.from_parser_node(node), node.text
+          )
+        end
+      end
+
+      class Quantifier
+        attr_reader :max, :min
+
+        def initialize(max, min)
+          @max = max; @min = min
+        end
+
+        def self.from_parser_node(node)
+          if node.quantifier
+            new(
+              node.quantifier.max,
+              node.quantifier.min
+            )
+          end
+        end
+      end
+
+      class EscapeSequence < Literal; end
+      class Word < Node; end
+      class Digit < Node; end
+      class Sequence < Node; end
+      class Alternation < Node; end
+      class Capture < Node; end
+      class Passive < Node; end
+      class Root < Node; end
+
+    end
+  end
+end

data/lib/twitter_cldr/utils/regexp_sampler.rb

@@ -0,0 +1,149 @@
+# encoding: UTF-8
+
+# Copyright 2012 Twitter, Inc
+# http://www.apache.org/licenses/LICENSE-2.0
+
+module TwitterCldr
+  module Utils
+
+    # Generates a valid string that would match the given regexp ast.
+    class RegexpSampler
+
+      attr_reader :regexp_ast
+
+      DIGITS       = ('0'..'9').to_a
+      WORD_LETTERS = ('a'..'z').to_a + ('A'..'Z').to_a + ['_']
+
+      def initialize(regexp_ast)
+        @regexp_ast = regexp_ast
+      end
+
+      def generate
+        walk_children(regexp_ast)
+      end
+
+      private
+
+      def walk(node)
+        method = :"walk_#{class_name_for(node)}"
+        puts method unless respond_to?(method, true)
+        respond_to?(method, true) ? send(method, node) : ""
+      end
+
+      def walk_children(node)
+        node.expressions.map { |expr| walk(expr) }.join
+      end
+
+      def walk_digit(node)
+        if node.quantified?
+          quantifier_sample(DIGITS, node.quantifier)
+        else
+          [single_sample(DIGITS)]
+        end.join + walk_children(node)
+      end
+
+      def walk_word(node)
+        if node.quantified?
+          quantifier_sample(WORD_LETTERS, node.quantifier)
+        else
+          [single_sample(WORD_LETTERS)]
+        end.join + walk_children(node)
+      end
+
+      def walk_literal(node)
+        node.text * if node.quantified?
+          rand_in_quantifier(node.quantifier)
+        else
+          1
+        end + walk_children(node)
+      end
+
+      def walk_character_set(node)
+        charset = expand_charset(node.members)
+
+        if node.quantified?
+          quantifier_sample(charset, node.quantifier)
+        else
+          [single_sample(charset)]
+        end.join + walk_children(node)
+      end
+
+      def walk_capture(node)
+        if node.quantified?
+          rand_in_quantifier(node.quantifier).times.map do
+            walk_children(node)
+          end.join
+        else
+          walk_children(node)
+        end
+      end
+
+      # "passive" means non-capturing group.
+      # Since we don't need to distinguish between
+      # captures/non-captures, we can just delegate
+      # to the walk_capture method.
+      def walk_passive(node)
+        walk_capture(node)
+      end
+
+      def walk_alternation(node)
+        if node.quantified?
+          rand_in_quantifier(node.quantifier).times.map do
+            walk(single_sample(node.expressions))
+          end.join
+        else
+          walk(single_sample(node.expressions))
+        end
+      end
+
+      def walk_sequence(node)
+        if node.quantified?
+          rand_in_quantifier(node.quantifier).times.map do
+            node.expressions.map { |expr| walk(expr) }.join
+          end.join
+        else
+          node.expressions.map { |expr| walk(expr) }.join
+        end
+      end
+
+      def expand_charset(members)
+        members.inject([]) do |ret, member|
+          ret + expand_charset_member(member)
+        end
+      end
+
+      def expand_charset_member(member)
+        left, right = member.scan(/([^\\])-?/).flatten
+        right ? (left..right).to_a : [left]
+      end
+
+      def quantifier_sample(arr, quantifier)
+        sample_size = if quantifier.min == quantifier.max
+          quantifier.min
+        else
+          rand_in_quantifier(quantifier)
+        end
+
+        sample_size.times.map { single_sample(arr) }
+      end
+
+      def single_sample(arr)
+        arr[rand(arr.size)]
+      end
+
+      def rand_in_quantifier(quantifier)
+        rand_in_range(quantifier.min, quantifier.max)
+      end
+
+      def rand_in_range(min, max)
+        min + rand((max - min) + 1)
+      end
+
+      def class_name_for(node)
+        name = node.class.to_s.split("::").last
+        name.gsub(/\A|([A-Z])/) { $1 ? "_#{$1.downcase}" : "" }.downcase
+      end
+
+    end
+  end
+end

data/lib/twitter_cldr/utils.rb

@@ -6,9 +6,11 @@
 module TwitterCldr
   module Utils
 
-    autoload :CodePoints,  'twitter_cldr/utils/code_points'
-    autoload :YAML,        'twitter_cldr/utils/yaml'
-    autoload :RangeSet,    'twitter_cldr/utils/range_set'
+    autoload :CodePoints,    'twitter_cldr/utils/code_points'
+    autoload :YAML,          'twitter_cldr/utils/yaml'
+    autoload :RangeSet,      'twitter_cldr/utils/range_set'
+    autoload :RegexpAst,     'twitter_cldr/utils/regexp_ast'
+    autoload :RegexpSampler, 'twitter_cldr/utils/regexp_sampler'
 
     class << self
 

data/lib/twitter_cldr/version.rb

@@ -4,5 +4,5 @@
 # http://www.apache.org/licenses/LICENSE-2.0
 
 module TwitterCldr
-  VERSION = "3.0.1"
+  VERSION = "3.0.2"
 end