uniprop 0.1.0

data/lib/resources/settings.rb

@@ -0,0 +1,120 @@
+DOWNLOADER_SETTINGS = {
+  default: {
+    cache_path: Pathname.new(__dir__) / "UCD",
+    excluded_extensions: %w{zip gz Z pdf ps gif jpg C html},
+    excluded_directories: %w{MAPPINGS PROGRAMS UCA cldr idna math reconstructed security vertical zipped charts ucdxml },
+    excluded_files: [
+      "Index",
+      "CJKXREF",
+      "StandardizedVariants",
+      "TangutSources",
+      "NushuSources",
+      "USourceData",
+      "NamedSequencesProv",
+      "ReadMe",
+
+      # files don't have property
+      "NormalizationCorrections",
+      "NamedSequences",
+      "CJKRadicals",
+      "NamesList",
+      "emoji-variation-sequences",
+      "EmojiSources",
+    ],
+    included_files: [
+      "Unihan",
+    ],
+    unicode_beta: false,
+  },
+
+  # can override settings for each version if need
+  # "15.0.0": {
+  #     unicode_beta: true,
+  # },
+}
+
+FILES_INFORMATION = {
+  default: {
+    property_aliases_file_name: "PropertyAliases",
+    property_value_aliases_file_name: "PropertyValueAliases",
+
+    default_file_format: {
+      strip: "\s",
+      split: ";",
+    },
+
+    file_formats: [
+      {
+          file_name: "NushuSources",
+          strip: "",
+          split: "\s",
+      },
+    ],
+
+    unihan_file_format: {
+      strip: "",
+      split: "\s",
+    }
+  },
+
+  "15.0.0": {
+    file_formats: [
+      {
+          file_name: "NushuSources",
+          strip: "",
+          split: "\s",
+      },
+    ],
+
+  }
+}
+
+PROPERTIES_INFORMATION = {
+  default: {
+    miscellaneous_formats: [
+      {
+        property_name: "Bidi_Mirroring_Glyph",
+        format_type: "String"
+      },
+      {
+        property_name: "Bidi_Paired_Bracket",
+        format_type: "String"
+      },
+      {
+        property_name: "Equivalent_Unified_Ideograph",
+        format_type: "String"
+      },
+      {
+        property_name: "Jamo_Short_Name",
+        format_type: "Jamo_Short_Name"
+      },
+      {
+        property_name: "Name",
+        format_type: "Unique",
+        unique_threshold: 0.9
+      },
+      {
+        property_name: "Name_Alias",
+        format_type: "Unique",
+        unique_threshold: 0.9
+      },
+      {
+        property_name: "Script_Extensions",
+        format_type: "Script_Extensions"
+      },
+      {
+        property_name: "Unicode_1_Name",
+        format_type: "text"
+      },
+      {
+        property_name: "ISO_Comment",
+        format_type: "text"
+      }
+    ]
+  },
+
+  # "15.0.0": {
+  #   miscellaneous_formats: [
+  #   ]
+  # }
+}

data/lib/uniprop/consts.rb

@@ -0,0 +1,31 @@
+module UniProp
+  RE_CODEPOINT = /[0-9A-F]{4,6}\.\.[0-9A-F]{4,6}|[0-9A-F]{4,6}/
+  MIN_CODEPOINT = 0x0000
+  MAX_CODEPOINT = 0x10ffff
+  CODEPOINT_RANGE = MIN_CODEPOINT..MAX_CODEPOINT
+
+  # プロパティの記述箇所を管理するためのStruct
+  # @param [PropFile] propfile
+  # @param [Integer] block
+  # @param [Range<Integer>] range
+  # @param [Integer] column
+  Position = Struct.new(:propfile, :range, :block, :columns)
+
+  # missingコメント1行を解析した結果を格納するためのStruct
+  # @param [Range<Integer>] codepoint_range
+  # @param [Property] property
+  # @param [String] missing_value
+  MissingDef = Struct.new(:codepoint_range, :property, :missing_value)
+
+  # メタデータに記述された1つのブロックを管理するためのStruct
+  # @note 主にメタデータの再作成時に使用。メタデータの内容をそのままStringで管理
+  # @param [Array<String>/Array<Array<String>>] content
+  # @param [String] range
+  RawBlock = Struct.new(:content, :range)
+
+  # メタデータに記述された1つのブロックを管理するためのStruct
+  # @note 主にメタデータの利用時に使用。メタデータを解析した結果、適したオブジェクトで管理
+  # @param [Array<Property?>/Array<Array<Property?>>] content
+  # @param [Range<Integer>]
+  Block = Struct.new(:content, :range)
+end

data/lib/uniprop/downloader.rb

@@ -0,0 +1,262 @@
+require 'fileutils'
+require 'open-uri'
+require 'pathname'
+begin
+  require 'net/https'
+rescue LoadError
+  https = 'http'
+else
+  https = 'https'
+
+  # open-uri of ruby 2.2.0 accepts an array of PEMs as ssl_ca_cert, but old
+  # versions do not.  so, patching OpenSSL::X509::Store#add_file instead.
+  class OpenSSL::X509::Store
+    alias orig_add_file add_file
+    def add_file(pems)
+      Array(pems).each do |pem|
+        if File.directory?(pem)
+          add_path pem
+        else
+          orig_add_file pem
+        end
+      end
+    end
+  end
+  # since open-uri internally checks ssl_ca_cert using File.directory?,
+  # allow to accept an array.
+  class <<File
+    alias orig_directory? directory?
+    def File.directory? files
+      files.is_a?(Array) ? false : orig_directory?(files)
+    end
+  end
+end
+
+class Downloader
+  def self.https=(https)
+    @@https = https
+  end
+
+  def self.https?
+    @@https == 'https'
+  end
+
+  def self.https
+    @@https
+  end
+
+  def self.mode_for(data)
+    /\A#!/ =~ data ? 0755 : 0644
+  end
+
+  def self.http_options(file, since)
+    options = {}
+    if since
+      case since
+      when true
+        since = (File.mtime(file).httpdate rescue nil)
+      when Time
+        since = since.httpdate
+      end
+      if since
+        options['If-Modified-Since'] = since
+      end
+    end
+    options['Accept-Encoding'] = 'identity' # to disable Net::HTTP::GenericRequest#decode_content
+    options
+  end
+
+  def self.httpdate(date)
+    Time.httpdate(date)
+  rescue ArgumentError => e
+    # Some hosts (e.g., zlib.net) return similar to RFC 850 but 4
+    # digit year, sometimes.
+    /\A\s*
+     (?:Mon|Tues|Wednes|Thurs|Fri|Satur|Sun)day,\x20
+     (\d\d)-(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)-(\d{4})\x20
+     (\d\d):(\d\d):(\d\d)\x20
+     GMT
+     \s*\z/ix =~ date or raise
+    warn e.message
+    Time.utc($3, $2, $1, $4, $5, $6)
+  end
+
+  def self.download(url, name, dir = nil, since = true, options = {})
+    options = options.dup
+    url = URI(url)
+    dryrun = options.delete(:dryrun)
+    options.delete(:unicode_beta) # just to be on the safe side for gems and gcc
+
+    if name
+      file = Pathname.new(under(dir, name))
+    else
+      name = File.basename(url.path)
+    end
+    cache_save = options.delete(:cache_save) {
+      ENV["CACHE_SAVE"] != "no"
+    }
+    cache = cache_file(url, name, options.delete(:cache_dir))
+    file ||= cache
+    if since.nil? and file.exist?
+      if $VERBOSE
+        $stdout.puts "#{file} already exists"
+        $stdout.flush
+      end
+      if cache_save
+        save_cache(cache, file, name)
+      end
+      return file.to_path
+    end
+    if dryrun
+      puts "Download #{url} into #{file}"
+      return
+    end
+    if link_cache(cache, file, name, $VERBOSE)
+      return file.to_path
+    end
+    if !https? and URI::HTTPS === url
+      warn "*** using http instead of https ***"
+      url.scheme = 'http'
+      url = URI(url.to_s)
+    end
+    if $VERBOSE
+      $stdout.print "downloading #{name} ... "
+      $stdout.flush
+    end
+    mtime = nil
+    options = options.merge(http_options(file, since.nil? ? true : since))
+    begin
+      data = with_retry(10) do
+        data = url.read(options)
+        if mtime = data.meta["last-modified"]
+          mtime = Time.httpdate(mtime)
+        end
+        data
+      end
+    rescue OpenURI::HTTPError => http_error
+      if http_error.message =~ /^304 / # 304 Not Modified
+        if $VERBOSE
+          $stdout.puts "not modified"
+          $stdout.flush
+        end
+        return file.to_path
+      end
+      raise
+    rescue Timeout::Error
+      if since.nil? and file.exist?
+        puts "Request for #{url} timed out, using old version."
+        return file.to_path
+      end
+      raise
+    rescue SocketError
+      if since.nil? and file.exist?
+        puts "No network connection, unable to download #{url}, using old version."
+        return file.to_path
+      end
+      raise
+    end
+    dest = (cache_save && cache && !cache.exist? ? cache : file)
+    dest.parent.mkpath
+    dest.open("wb", 0600) do |f|
+      data.scrub!('?') # invalid byte sequence in UTF-8 対策
+      f.write(data)
+      f.chmod(mode_for(data))
+    end
+    if mtime
+      dest.utime(mtime, mtime)
+    end
+    if $VERBOSE
+      $stdout.puts "done"
+      $stdout.flush
+    end
+    if dest.eql?(cache)
+      link_cache(cache, file, name)
+    elsif cache_save
+      save_cache(cache, file, name)
+    end
+    return file.to_path
+  rescue => e
+    raise "failed to download #{name}\n#{e.class}: #{e.message}: #{url}"
+  end
+
+  def self.under(dir, name)
+    dir ? File.join(dir, File.basename(name)) : name
+  end
+
+  def self.cache_file(url, name, cache_dir = nil)
+    case cache_dir
+    when false
+      return nil
+    when nil
+      cache_dir = ENV['CACHE_DIR']
+      if !cache_dir or cache_dir.empty?
+        cache_dir = ".downloaded-cache"
+      end
+    end
+    Pathname.new(cache_dir) + (name || File.basename(URI(url).path))
+  end
+
+  def self.link_cache(cache, file, name, verbose = false)
+    return false unless cache and cache.exist?
+    return true if cache.eql?(file)
+    if /cygwin/ !~ RUBY_PLATFORM or /winsymlink:nativestrict/ =~ ENV['CYGWIN']
+      begin
+        file.make_symlink(cache.relative_path_from(file.parent))
+      rescue SystemCallError
+      else
+        if verbose
+          $stdout.puts "made symlink #{name} to #{cache}"
+          $stdout.flush
+        end
+        return true
+      end
+    end
+    begin
+      file.make_link(cache)
+    rescue SystemCallError
+    else
+      if verbose
+        $stdout.puts "made link #{name} to #{cache}"
+        $stdout.flush
+      end
+      return true
+    end
+  end
+
+  def self.save_cache(cache, file, name)
+    return unless cache or cache.eql?(file)
+    begin
+      st = cache.stat
+    rescue
+      begin
+        file.rename(cache)
+      rescue
+        return
+      end
+    else
+      return unless st.mtime > file.lstat.mtime
+      file.unlink
+    end
+    link_cache(cache, file, name)
+  end
+
+  def self.with_retry(max_times, &block)
+    times = 0
+    begin
+      block.call
+    rescue Errno::ETIMEDOUT, SocketError, OpenURI::HTTPError, Net::ReadTimeout, Net::OpenTimeout, ArgumentError => e
+      raise if e.is_a?(OpenURI::HTTPError) && e.message !~ /^50[023] / # retry only 500, 502, 503 for http error
+      times += 1
+      if times <= max_times
+        $stderr.puts "retrying #{e.class} (#{e.message}) after #{times ** 2} seconds..."
+        sleep(times ** 2)
+        retry
+      else
+        raise
+      end
+    end
+  end
+  private_class_method :with_retry
+end
+
+Downloader.https = https.freeze

data/lib/uniprop/dsl.rb

@@ -0,0 +1,53 @@
+module UniProp
+  def prop_data
+    @@prop_data ||= PropData.new(
+      Pathname.new(__dir__) / "../resources/settings.rb",
+      Pathname.new(__dir__) / "../resources/metadata.json"
+    )
+  end
+
+  # @param [String] version_name
+  # @return [VersionManager]
+  def version(version_name)
+    prop_data.version_manager(version_name)
+  end
+
+  # @return [UnicodeManager]
+  def unicode_manager
+    prop_data.unicode_manager
+  end
+
+  # 最新バージョンの名前を取得
+  # @param [Boolean] update_metadata trueの場合、バージョン名を取得し、メタデータを更新する
+  # @return [String]
+  def latest_version(update_metadata: false)
+    version_names = prop_data.metadata.version_names(update_metadata: update_metadata, confirm: update_metadata)
+    
+    version_names.sort_by { Version.name_to_weight(_1) }.last
+  end
+end
+
+class Module
+  alias_method :const_missing_orig, :const_missing
+  def const_missing(const, *args, &block)
+    # VersionManager
+    if const =~ /^V([\d_]+)$/
+      # A_B_C -> A.B.C
+      version_nums = $1.split(/_/)
+      version_nums[1] ||= "0"
+      version_nums[2] ||= "0"
+      version_name = version_nums.join(".")
+      UniProp::prop_data.version_manager(version_name)
+    elsif const =~ /^V(\d+)_(\d+)_Update(\d+)$/
+      # A_B_C -> A.B-UpdateC
+      version_name = "#{$1}.#{$2}-Update#{$3}"
+      UniProp::prop_data.version_manager(version_name)
+    
+    # UnicodeManager
+    elsif const =~ /UNICODE/
+      UniProp::prop_data.unicode_manager
+    else
+      const_missing_orig(const, *args, &block)
+    end
+  end
+end

data/lib/uniprop/efficient_elements.rb

@@ -0,0 +1,40 @@
+module UniProp
+  # 完全なメタデータが存在する事を前提とし、必要なファイルをキャッシュにダウンロードしてVersionと同じ動きをするクラス
+  class EfficientVersion < Version    
+    # @return [Set<PropFile>]
+    def files
+      return @files if @files
+      @files = version_metadata.propfile_names.map { create_propfile(_1) }
+
+      # PropertyAliases, PropertyValueAliasesはメタデータに記述されない
+      @files << property_aliases_file
+      @files << property_value_aliases_file
+      @files
+    end
+
+    # @return [Array<String>?]
+    def unihan_file_names
+      version_metadata.unihan_file_names
+    end
+
+    # @return [Array<Property>]
+    def unihan_properties
+      version_metadata.unihan_properties
+    end
+    
+    # @return [PropFile]
+    def find_file(propfile)
+      super(propfile, confirm: false)
+    end
+
+    # @param [String] filename basename_prefixに該当するファイル名
+    # @return [PropFile]
+    def create_propfile(filename)
+      if UniPropUtils::FileManager.unihan_file?(filename, unihan_file_names)
+        return PropFile::UnihanFile.new(filename, self)
+      else
+        return PropFile.new(filename, self)
+      end
+    end
+  end
+end

data/lib/uniprop/errors.rb

@@ -0,0 +1,31 @@
+module UniProp
+  class VersionNotMatchedError < StandardError; end
+  class ParseError < StandardError; end
+  class PropertyNotFoundError < StandardError
+    def initialize(searched_property)
+      super("property not found. (searched property: #{searched_property})")
+    end
+  end
+  class PropertyValueTypeNotExistsError < StandardError
+    def initialize(type)
+      super("#{type} does not exist as property value type.")
+    end
+  end
+  class FileExistsError < StandardError
+    def initialize(file_path)
+      super("#{file_path} is already exists. Please delete the file and run again.")
+    end
+  end
+  class FileNotFoundError < StandardError; end
+  class MetaDataNotFoundError < StandardError; end
+  class MetaDataParseError < StandardError; end
+  class PropDataDifferentError < StandardError; end
+  class MetaDataExistsError < StandardError
+    def initialize(version_name)
+      super("Metadata for #{version_name} is already exists. Please delete the data and run again.")
+    end
+  end
+  class VersionMetaDataNotExistsError < StandardError; end
+  class PropertyValueNotFoundError < StandardError; end
+  class VersionDifferentError < StandardError; end
+end

data/lib/uniprop/inspects.rb

@@ -0,0 +1,122 @@
+# :nocov:
+module UniProp
+  class PropFile
+    # @private
+    def inspect
+      "#<#{self.class.name} #{basename_prefix}>"
+    end
+  end
+
+  class Version
+    # @private
+    def inspect
+      "#<#{self.class.name} #{major}.#{minor}.#{tiny}>"
+    end
+  end
+  
+  class EfficientVersion < Version
+    # @private
+    def inspect
+      "#<#{self.class} #{version_metadata.version.version_name}>"
+    end
+  end
+
+  class PropFileValueGroup
+    # @private
+    def inspect
+      "#<#{self.class.name} #{propfile.basename_prefix}>"
+    end
+  end
+
+  class ActualPropertyValueGroup
+      # @private
+      def inspect
+        property_names = properties.map { _1.longest_alias }
+                                    .join(', ')
+        "#<#{self.class.name} #{propfile.basename_prefix} (#{property_names})>"
+      end
+  end
+
+  class RevisingHintGenerator
+    # @private
+    def inspect
+      "#<#{self.class} #{recreator.old_version.version_name},#{recreator.new_version.version_name}>"
+    end
+  end
+
+  class MetaData
+    # @private
+    def inspect
+      "#<#{self.class.name}>"
+    end
+  end
+
+  class VersionMetaData
+    # @private
+    def inspect
+      "#<#{self.class.name} #{version.major}.#{version.minor}.#{version.tiny}>"
+    end
+  end
+
+  class PropFileMetaData
+    # @private
+    def inspect
+      "#<#{self.class.name} #{propfile.basename_prefix}>"
+    end
+  end
+
+  class VersionMetaDataValidator
+    # @private
+    def inspect
+      "#<#{self.class.name} #{version_metadata.version.major}.#{version_metadata.version.minor}.#{version_metadata.version.tiny}>"
+    end
+  end
+
+  class PropData
+      # @private
+      def inspect
+        "#<#{self.class.name}>"
+      end
+  end
+
+  module Alias
+    # @private
+    def inspect
+      "#<#{self.class.name} #{longest_alias}>"
+    end
+  end
+
+  class BasePropertyValueGroup
+    # @private
+    def inspect
+      "#<#{self.class.name}>"
+    end
+  end
+
+  class PropertyMetaData
+    # @private
+    def inspect
+      "#<#{self.class.name}>"
+    end
+  end
+
+  class VersionPropertyMetaData
+    # @private
+    def inspect
+      "#<#{self.class.name}>"
+    end
+  end
+
+  class UnicodeManager
+    def inspect
+      "#<#{self.class.name}>"
+    end
+  end
+
+  class VersionManager
+    def inspect
+      "#<#{self.class.name} #{version.version_name}>"
+    end
+  end
+end
+# :nocov: