regexp_property_values 0.3.5 → 1.0.0

data/lib/regexp_property_values/updater.rb

@@ -0,0 +1,133 @@
+module RegexpPropertyValues
+  module Updater
+    module_function
+
+    require 'fileutils'
+    require 'set'
+
+    BASE_URL = 'http://www.unicode.org/Public/'
+
+    UCD_FILES = %w[
+      Blocks.txt
+      DerivedAge.txt
+      DerivedCoreProperties.txt
+      PropertyAliases.txt
+      PropertyValueAliases.txt
+      PropList.txt
+      Scripts.txt
+    ]
+
+    EMOJI_FILES = %w[
+      emoji-data.txt
+    ]
+
+    TMP_DIR = File.join(__dir__, 'tmp_ucd')
+
+    def call
+      prepare_tmp_dir
+      download_ucd_files
+      write_values
+      write_aliases
+      remove_tmp_dir
+      print_stats
+    end
+
+    def prepare_tmp_dir
+      FileUtils.rm_rf(TMP_DIR) if File.exist?(TMP_DIR)
+      FileUtils.mkdir(TMP_DIR)
+    end
+
+    def download_ucd_files
+      unicode_version = RbConfig::CONFIG.fetch('UNICODE_VERSION')
+      emoji_version   = RbConfig::CONFIG.fetch('UNICODE_EMOJI_VERSION')
+      puts 'This will load ucd and emoji data for the CURRENT RUBY '\
+           "(#{unicode_version} / #{emoji_version}). Run this on the "\
+           'latest Ruby version you want to support. Continue? [y/n]'
+      return puts 'download skipped.' unless $stdin.gets =~ /^y/i
+
+      Dir.chdir(TMP_DIR) do
+        UCD_FILES.each   { |f| `wget #{BASE_URL}/#{unicode_version}/ucd/#{f}` }
+        EMOJI_FILES.each { |f| `wget #{BASE_URL}/emoji/#{emoji_version}/#{f}` }
+      end
+    end
+
+    def write_values
+      @values = Set.new
+
+      # posix properties
+      @values += %w[
+        Alpha Blank Cntrl Digit Graph Lower Print
+        Punct Space Upper XDigit Word Alnum ASCII
+        XPosixPunct
+      ]
+
+      # special properties
+      @values += %w[Any Assigned In_No_Block Unknown]
+
+      # legacy properties
+      @values += %w[Newline]
+
+      regexp = /^[0-9a-fA-F]+(?:\.\.[0-9a-fA-F]+)? *; (?<prop_name>\w+) +# /
+      %w[
+        DerivedCoreProperties.txt
+        PropList.txt
+        Scripts.txt
+        emoji-data.txt
+      ].each { |file| scan(file, regexp) { |caps| @values << caps[:prop_name] } }
+
+      scan('PropertyValueAliases.txt', /^gc ; \w+ *; (?<prop_name>\w+)/) do |caps|
+        @values << caps[:prop_name]
+      end
+
+      scan('Blocks.txt', /^[\dA-F.]+ *; (?<block_name>[-\w ]+)/) do |caps|
+        @values << 'In_' + caps[:block_name].gsub(/\W/, '_')
+      end
+
+      scan('DerivedAge.txt', /^[\dA-F.]+ *; (?<age_num>[\d.]+)/) do |caps|
+        @values << 'Age=' + caps[:age_num]
+      end
+
+      File.write(RegexpPropertyValues::VALUES_PATH, @values.sort.join("\n"))
+    end
+
+    def write_aliases
+      @aliases = Set.new
+
+      scan('PropertyAliases.txt', /^(?<alias>\w+) *; (?<name>\w+)/) do |caps|
+        if in_values?(caps[:name]) && !in_values?(caps[:alias])
+          @aliases << [caps[:alias], caps[:name]]
+        end
+      end
+
+      scan('PropertyValueAliases.txt',
+        /^[gs]c ; (?<alias1>\w+) *; (?<name>\w+)(?: *; (?<alias2>\w+))?/) do |caps|
+        if in_values?(caps[:name]) && !in_values?(caps[:alias1])
+          @aliases << [caps[:alias1], caps[:name]]
+        end
+        if in_values?(caps[:name]) && caps[:alias2] && !in_values?(caps[:alias2])
+          @aliases << [caps[:alias2], caps[:name]]
+        end
+      end
+
+      File.write(RegexpPropertyValues::ALIASES_PATH,
+                 @aliases.sort.map { |pair| pair.join(';') }.join("\n"))
+    end
+
+    def in_values?(string)
+      @values.any? { |value| value.casecmp?(string) }
+    end
+
+    def scan(file, pattern)
+      path = File.join(TMP_DIR, file)
+      File.read(path).scan(pattern) { yield(Regexp.last_match) }
+    end
+
+    def remove_tmp_dir
+      FileUtils.rm_rf(TMP_DIR)
+    end
+
+    def print_stats
+      print "\nFetched #{@values.size} values and #{@aliases.size} aliases.\n\n"
+    end
+  end
+end

data/lib/regexp_property_values/value.rb

@@ -0,0 +1,14 @@
+module RegexpPropertyValues
+  class Value
+    require_relative 'value/shared_methods'
+    include SharedMethods
+
+    if const_defined?(:OnigRegexpPropertyHelper)
+      require_relative 'value/ext_adapter'
+      include ExtAdapter
+    else
+      require_relative 'value/ruby_fallback'
+      include RubyFallback
+    end
+  end
+end

data/lib/regexp_property_values/value/ext_adapter.rb

@@ -0,0 +1,19 @@
+module RegexpPropertyValues
+  class Value
+    module ExtAdapter
+      def matched_characters
+        matched_codepoints.map { |cp| cp.chr('utf-8') }
+      end
+
+      def matched_codepoints
+        matched_ranges.flat_map(&:to_a)
+      end
+
+      def matched_ranges
+        OnigRegexpPropertyHelper.matched_ranges(name)
+      rescue ArgumentError
+        raise_unsupported_or_unknown_error
+      end
+    end
+  end
+end

data/lib/regexp_property_values/value/ruby_fallback.rb

@@ -0,0 +1,21 @@
+module RegexpPropertyValues
+  class Value
+    module RubyFallback
+      def matched_characters
+        matched_codepoints.map { |cp| cp.chr('utf-8') }
+      end
+
+      def matched_codepoints
+        # turns out scanning one big string is the least slow way to do this
+        @@test_str ||= (0..0xD7FF).map { |cp| cp.chr('utf-8') }.join <<
+                       (0xE000..0x10FFFF).map { |cp| cp.chr('utf-8') }.join
+        @@test_str.scan(regexp).flat_map(&:codepoints)
+      end
+
+      def matched_ranges
+        require 'range_compressor'
+        RangeCompressor.compress(matched_codepoints)
+      end
+    end
+  end
+end

data/lib/regexp_property_values/value/shared_methods.rb

@@ -0,0 +1,63 @@
+module RegexpPropertyValues
+  class Value
+    module SharedMethods
+      attr_reader :name
+
+      def initialize(name)
+        @name = name
+      end
+
+      def supported_by_current_ruby?
+        !!regexp rescue false
+      end
+
+      def ==(other)
+        identifier == other.identifier
+      end
+      alias eql? ==
+
+      def hash
+        @hash ||= identifier.hash
+      end
+
+      def identifier
+        @identifier ||= name.to_s.downcase.gsub(/[^0-9a-z=.]/, '')
+      end
+      alias to_s identifier
+
+      def full_name
+        (original = find_original) ? original.name : raise_unknown_error
+      end
+
+      def character_set
+        require 'character_set'
+        CharacterSet.from_ranges(*matched_ranges)
+      end
+
+      private
+
+      def regexp
+        @regexp ||= /\p{#{identifier}}/u
+      rescue RegexpError, SyntaxError
+        raise_unsupported_or_unknown_error
+      end
+
+      def find_original
+        RegexpPropertyValues.all.find { |orig| orig.eql?(self) } ||
+          RegexpPropertyValues.alias_hash[self]
+      end
+
+      def raise_unsupported_or_unknown_error
+        find_original ? raise_unsupported_error : raise_unknown_error
+      end
+
+      def raise_unsupported_error
+        raise Error, "Property name `#{name}` is known, but not in this Ruby"
+      end
+
+      def raise_unknown_error
+        raise Error, "Property name `#{name}` is not known in any Ruby"
+      end
+    end
+  end
+end

data/lib/regexp_property_values/version.rb

@@ -1,3 +1,3 @@
 module RegexpPropertyValues
-  VERSION = '0.3.5'
+  VERSION = '1.0.0'
 end

data/lib/values

@@ -0,0 +1,590 @@
+ASCII
+ASCII_Hex_Digit
+Adlam
+Age=1.1
+Age=10.0
+Age=11.0
+Age=12.0
+Age=12.1
+Age=2.0
+Age=2.1
+Age=3.0
+Age=3.1
+Age=3.2
+Age=4.0
+Age=4.1
+Age=5.0
+Age=5.1
+Age=5.2
+Age=6.0
+Age=6.1
+Age=6.2
+Age=6.3
+Age=7.0
+Age=8.0
+Age=9.0
+Ahom
+Alnum
+Alpha
+Alphabetic
+Anatolian_Hieroglyphs
+Any
+Arabic
+Armenian
+Assigned
+Avestan
+Balinese
+Bamum
+Bassa_Vah
+Batak
+Bengali
+Bhaiksuki
+Bidi_Control
+Blank
+Bopomofo
+Brahmi
+Braille
+Buginese
+Buhid
+Canadian_Aboriginal
+Carian
+Case_Ignorable
+Cased
+Cased_Letter
+Caucasian_Albanian
+Chakma
+Cham
+Changes_When_Casefolded
+Changes_When_Casemapped
+Changes_When_Lowercased
+Changes_When_Titlecased
+Changes_When_Uppercased
+Cherokee
+Close_Punctuation
+Cntrl
+Common
+Connector_Punctuation
+Control
+Coptic
+Cuneiform
+Currency_Symbol
+Cypriot
+Cyrillic
+Dash
+Dash_Punctuation
+Decimal_Number
+Default_Ignorable_Code_Point
+Deprecated
+Deseret
+Devanagari
+Diacritic
+Digit
+Dogra
+Duployan
+Egyptian_Hieroglyphs
+Elbasan
+Elymaic
+Emoji
+Emoji_Component
+Emoji_Modifier
+Emoji_Modifier_Base
+Emoji_Presentation
+Enclosing_Mark
+Ethiopic
+Extender
+Final_Punctuation
+Format
+Georgian
+Glagolitic
+Gothic
+Grantha
+Graph
+Grapheme_Base
+Grapheme_Extend
+Grapheme_Link
+Greek
+Gujarati
+Gunjala_Gondi
+Gurmukhi
+Han
+Hangul
+Hanifi_Rohingya
+Hanunoo
+Hatran
+Hebrew
+Hex_Digit
+Hiragana
+Hyphen
+IDS_Binary_Operator
+IDS_Trinary_Operator
+ID_Continue
+ID_Start
+Ideographic
+Imperial_Aramaic
+In_Adlam
+In_Aegean_Numbers
+In_Ahom
+In_Alchemical_Symbols
+In_Alphabetic_Presentation_Forms
+In_Anatolian_Hieroglyphs
+In_Ancient_Greek_Musical_Notation
+In_Ancient_Greek_Numbers
+In_Ancient_Symbols
+In_Arabic
+In_Arabic_Extended_A
+In_Arabic_Mathematical_Alphabetic_Symbols
+In_Arabic_Presentation_Forms_A
+In_Arabic_Presentation_Forms_B
+In_Arabic_Supplement
+In_Armenian
+In_Arrows
+In_Avestan
+In_Balinese
+In_Bamum
+In_Bamum_Supplement
+In_Basic_Latin
+In_Bassa_Vah
+In_Batak
+In_Bengali
+In_Bhaiksuki
+In_Block_Elements
+In_Bopomofo
+In_Bopomofo_Extended
+In_Box_Drawing
+In_Brahmi
+In_Braille_Patterns
+In_Buginese
+In_Buhid
+In_Byzantine_Musical_Symbols
+In_CJK_Compatibility
+In_CJK_Compatibility_Forms
+In_CJK_Compatibility_Ideographs
+In_CJK_Compatibility_Ideographs_Supplement
+In_CJK_Radicals_Supplement
+In_CJK_Strokes
+In_CJK_Symbols_and_Punctuation
+In_CJK_Unified_Ideographs
+In_CJK_Unified_Ideographs_Extension_A
+In_CJK_Unified_Ideographs_Extension_B
+In_CJK_Unified_Ideographs_Extension_C
+In_CJK_Unified_Ideographs_Extension_D
+In_CJK_Unified_Ideographs_Extension_E
+In_CJK_Unified_Ideographs_Extension_F
+In_Carian
+In_Caucasian_Albanian
+In_Chakma
+In_Cham
+In_Cherokee
+In_Cherokee_Supplement
+In_Chess_Symbols
+In_Combining_Diacritical_Marks
+In_Combining_Diacritical_Marks_Extended
+In_Combining_Diacritical_Marks_Supplement
+In_Combining_Diacritical_Marks_for_Symbols
+In_Combining_Half_Marks
+In_Common_Indic_Number_Forms
+In_Control_Pictures
+In_Coptic
+In_Coptic_Epact_Numbers
+In_Counting_Rod_Numerals
+In_Cuneiform
+In_Cuneiform_Numbers_and_Punctuation
+In_Currency_Symbols
+In_Cypriot_Syllabary
+In_Cyrillic
+In_Cyrillic_Extended_A
+In_Cyrillic_Extended_B
+In_Cyrillic_Extended_C
+In_Cyrillic_Supplement
+In_Deseret
+In_Devanagari
+In_Devanagari_Extended
+In_Dingbats
+In_Dogra
+In_Domino_Tiles
+In_Duployan
+In_Early_Dynastic_Cuneiform
+In_Egyptian_Hieroglyph_Format_Controls
+In_Egyptian_Hieroglyphs
+In_Elbasan
+In_Elymaic
+In_Emoticons
+In_Enclosed_Alphanumeric_Supplement
+In_Enclosed_Alphanumerics
+In_Enclosed_CJK_Letters_and_Months
+In_Enclosed_Ideographic_Supplement
+In_Ethiopic
+In_Ethiopic_Extended
+In_Ethiopic_Extended_A
+In_Ethiopic_Supplement
+In_General_Punctuation
+In_Geometric_Shapes
+In_Geometric_Shapes_Extended
+In_Georgian
+In_Georgian_Extended
+In_Georgian_Supplement
+In_Glagolitic
+In_Glagolitic_Supplement
+In_Gothic
+In_Grantha
+In_Greek_Extended
+In_Greek_and_Coptic
+In_Gujarati
+In_Gunjala_Gondi
+In_Gurmukhi
+In_Halfwidth_and_Fullwidth_Forms
+In_Hangul_Compatibility_Jamo
+In_Hangul_Jamo
+In_Hangul_Jamo_Extended_A
+In_Hangul_Jamo_Extended_B
+In_Hangul_Syllables
+In_Hanifi_Rohingya
+In_Hanunoo
+In_Hatran
+In_Hebrew
+In_High_Private_Use_Surrogates
+In_High_Surrogates
+In_Hiragana
+In_IPA_Extensions
+In_Ideographic_Description_Characters
+In_Ideographic_Symbols_and_Punctuation
+In_Imperial_Aramaic
+In_Indic_Siyaq_Numbers
+In_Inscriptional_Pahlavi
+In_Inscriptional_Parthian
+In_Javanese
+In_Kaithi
+In_Kana_Extended_A
+In_Kana_Supplement
+In_Kanbun
+In_Kangxi_Radicals
+In_Kannada
+In_Katakana
+In_Katakana_Phonetic_Extensions
+In_Kayah_Li
+In_Kharoshthi
+In_Khmer
+In_Khmer_Symbols
+In_Khojki
+In_Khudawadi
+In_Lao
+In_Latin_1_Supplement
+In_Latin_Extended_A
+In_Latin_Extended_Additional
+In_Latin_Extended_B
+In_Latin_Extended_C
+In_Latin_Extended_D
+In_Latin_Extended_E
+In_Lepcha
+In_Letterlike_Symbols
+In_Limbu
+In_Linear_A
+In_Linear_B_Ideograms
+In_Linear_B_Syllabary
+In_Lisu
+In_Low_Surrogates
+In_Lycian
+In_Lydian
+In_Mahajani
+In_Mahjong_Tiles
+In_Makasar
+In_Malayalam
+In_Mandaic
+In_Manichaean
+In_Marchen
+In_Masaram_Gondi
+In_Mathematical_Alphanumeric_Symbols
+In_Mathematical_Operators
+In_Mayan_Numerals
+In_Medefaidrin
+In_Meetei_Mayek
+In_Meetei_Mayek_Extensions
+In_Mende_Kikakui
+In_Meroitic_Cursive
+In_Meroitic_Hieroglyphs
+In_Miao
+In_Miscellaneous_Mathematical_Symbols_A
+In_Miscellaneous_Mathematical_Symbols_B
+In_Miscellaneous_Symbols
+In_Miscellaneous_Symbols_and_Arrows
+In_Miscellaneous_Symbols_and_Pictographs
+In_Miscellaneous_Technical
+In_Modi
+In_Modifier_Tone_Letters
+In_Mongolian
+In_Mongolian_Supplement
+In_Mro
+In_Multani
+In_Musical_Symbols
+In_Myanmar
+In_Myanmar_Extended_A
+In_Myanmar_Extended_B
+In_NKo
+In_Nabataean
+In_Nandinagari
+In_New_Tai_Lue
+In_Newa
+In_No_Block
+In_Number_Forms
+In_Nushu
+In_Nyiakeng_Puachue_Hmong
+In_Ogham
+In_Ol_Chiki
+In_Old_Hungarian
+In_Old_Italic
+In_Old_North_Arabian
+In_Old_Permic
+In_Old_Persian
+In_Old_Sogdian
+In_Old_South_Arabian
+In_Old_Turkic
+In_Optical_Character_Recognition
+In_Oriya
+In_Ornamental_Dingbats
+In_Osage
+In_Osmanya
+In_Ottoman_Siyaq_Numbers
+In_Pahawh_Hmong
+In_Palmyrene
+In_Pau_Cin_Hau
+In_Phags_pa
+In_Phaistos_Disc
+In_Phoenician
+In_Phonetic_Extensions
+In_Phonetic_Extensions_Supplement
+In_Playing_Cards
+In_Private_Use_Area
+In_Psalter_Pahlavi
+In_Rejang
+In_Rumi_Numeral_Symbols
+In_Runic
+In_Samaritan
+In_Saurashtra
+In_Sharada
+In_Shavian
+In_Shorthand_Format_Controls
+In_Siddham
+In_Sinhala
+In_Sinhala_Archaic_Numbers
+In_Small_Form_Variants
+In_Small_Kana_Extension
+In_Sogdian
+In_Sora_Sompeng
+In_Soyombo
+In_Spacing_Modifier_Letters
+In_Specials
+In_Sundanese
+In_Sundanese_Supplement
+In_Superscripts_and_Subscripts
+In_Supplemental_Arrows_A
+In_Supplemental_Arrows_B
+In_Supplemental_Arrows_C
+In_Supplemental_Mathematical_Operators
+In_Supplemental_Punctuation
+In_Supplemental_Symbols_and_Pictographs
+In_Supplementary_Private_Use_Area_A
+In_Supplementary_Private_Use_Area_B
+In_Sutton_SignWriting
+In_Syloti_Nagri
+In_Symbols_and_Pictographs_Extended_A
+In_Syriac
+In_Syriac_Supplement
+In_Tagalog
+In_Tagbanwa
+In_Tags
+In_Tai_Le
+In_Tai_Tham
+In_Tai_Viet
+In_Tai_Xuan_Jing_Symbols
+In_Takri
+In_Tamil
+In_Tamil_Supplement
+In_Tangut
+In_Tangut_Components
+In_Telugu
+In_Thaana
+In_Thai
+In_Tibetan
+In_Tifinagh
+In_Tirhuta
+In_Transport_and_Map_Symbols
+In_Ugaritic
+In_Unified_Canadian_Aboriginal_Syllabics
+In_Unified_Canadian_Aboriginal_Syllabics_Extended
+In_Vai
+In_Variation_Selectors
+In_Variation_Selectors_Supplement
+In_Vedic_Extensions
+In_Vertical_Forms
+In_Wancho
+In_Warang_Citi
+In_Yi_Radicals
+In_Yi_Syllables
+In_Yijing_Hexagram_Symbols
+In_Zanabazar_Square
+Inherited
+Initial_Punctuation
+Inscriptional_Pahlavi
+Inscriptional_Parthian
+Javanese
+Join_Control
+Kaithi
+Kannada
+Katakana
+Kayah_Li
+Kharoshthi
+Khmer
+Khojki
+Khudawadi
+Lao
+Latin
+Lepcha
+Letter
+Letter_Number
+Limbu
+Line_Separator
+Linear_A
+Linear_B
+Lisu
+Logical_Order_Exception
+Lower
+Lowercase
+Lowercase_Letter
+Lycian
+Lydian
+Mahajani
+Makasar
+Malayalam
+Mandaic
+Manichaean
+Marchen
+Mark
+Masaram_Gondi
+Math
+Math_Symbol
+Medefaidrin
+Meetei_Mayek
+Mende_Kikakui
+Meroitic_Cursive
+Meroitic_Hieroglyphs
+Miao
+Modi
+Modifier_Letter
+Modifier_Symbol
+Mongolian
+Mro
+Multani
+Myanmar
+Nabataean
+Nandinagari
+New_Tai_Lue
+Newa
+Newline
+Nko
+Noncharacter_Code_Point
+Nonspacing_Mark
+Number
+Nushu
+Nyiakeng_Puachue_Hmong
+Ogham
+Ol_Chiki
+Old_Hungarian
+Old_Italic
+Old_North_Arabian
+Old_Permic
+Old_Persian
+Old_Sogdian
+Old_South_Arabian
+Old_Turkic
+Open_Punctuation
+Oriya
+Osage
+Osmanya
+Other
+Other_Alphabetic
+Other_Default_Ignorable_Code_Point
+Other_Grapheme_Extend
+Other_ID_Continue
+Other_ID_Start
+Other_Letter
+Other_Lowercase
+Other_Math
+Other_Number
+Other_Punctuation
+Other_Symbol
+Other_Uppercase
+Pahawh_Hmong
+Palmyrene
+Paragraph_Separator
+Pattern_Syntax
+Pattern_White_Space
+Pau_Cin_Hau
+Phags_Pa
+Phoenician
+Prepended_Concatenation_Mark
+Print
+Private_Use
+Psalter_Pahlavi
+Punct
+Punctuation
+Quotation_Mark
+Radical
+Regional_Indicator
+Rejang
+Runic
+Samaritan
+Saurashtra
+Sentence_Terminal
+Separator
+Sharada
+Shavian
+Siddham
+SignWriting
+Sinhala
+Soft_Dotted
+Sogdian
+Sora_Sompeng
+Soyombo
+Space
+Space_Separator
+Spacing_Mark
+Sundanese
+Surrogate
+Syloti_Nagri
+Symbol
+Syriac
+Tagalog
+Tagbanwa
+Tai_Le
+Tai_Tham
+Tai_Viet
+Takri
+Tamil
+Tangut
+Telugu
+Terminal_Punctuation
+Thaana
+Thai
+Tibetan
+Tifinagh
+Tirhuta
+Titlecase_Letter
+Ugaritic
+Unassigned
+Unified_Ideograph
+Unknown
+Upper
+Uppercase
+Uppercase_Letter
+Vai
+Variation_Selector
+Wancho
+Warang_Citi
+White_Space
+Word
+XDigit
+XID_Continue
+XID_Start
+XPosixPunct
+Yi
+Zanabazar_Square