u 0.5.0 → 1.0.0

data/build/ext/u/data/constants.rb

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+
+require 'u/build'
+
+class Constants
+  def initialize(data, version, io = $stdout)
+    # TODO: Defines should be made with a list, but we can’t match formatting
+    # quite yet, so wait until we have stable tests.
+    U::Build::Header.new(io) do
+      io.puts <<EOH
+#define UNICODE_DATA_VERSION "#{version}"
+
+#define UNICODE_LAST_CHAR #{'%#04x' % data.last}
+
+#define UNICODE_MAX_TABLE_INDEX ((UNICODE_LAST_CHAR + 1) / 256)
+
+#define UNICODE_LAST_CHAR_PART1 #{data.last_char_part1_x}
+
+#define UNICODE_FIRST_CHAR_PART2 0xe0000
+
+#define UNICODE_LAST_PAGE_PART1 (UNICODE_LAST_CHAR_PART1 / 256)
+
+#define UNICODE_LAST_PAGE_PART2 ((UNICODE_LAST_CHAR + 1 - UNICODE_FIRST_CHAR_PART2) / 256)
+
+#define UNICODE_SPECIAL_CASE_TABLE_START 0x1000000
+EOH
+    end
+  end
+end
+
+Constants.new(Marshal.load(File.open(ARGV[0], 'rb', &:read)), ARGV[1])

data/build/ext/u/data/decompose.rb

@@ -0,0 +1,85 @@
+# -*- coding: utf-8 -*-
+
+require 'u/build'
+
+class Decompose
+  NotPresentOffset = 1 << 16 - 1
+
+  def initialize(data, io = $stdout)
+    # TODO: Defines should be made with a list, but we can’t match formatting
+    # quite yet, so wait until we have stable tests.
+    U::Build::Header.new(io) do
+      io.puts <<EOD
+#define UNICODE_NOT_PRESENT_OFFSET UINT16_MAX
+EOD
+      io.puts DecompositionTable.new(data)
+    end
+  end
+
+private
+
+  class DecompositionTable < U::Build::Header::Table
+    def initialize(data)
+      super "static const struct {
+\tuint32_t ch;
+\tuint16_t canon_offset;
+\tuint16_t compat_offset;
+} decomp_table[]"
+      @data = data
+      @decomp_offsets = {}
+      @decomp_string = ''
+      @decomp_string_offset = 0
+      0.upto(data.last) do |i|
+        next if data[i].decomposition.empty?
+        canon_decomp = data[i].decomposition.canonical? ? make_decomp(i, false) : nil
+        compat_decomp = make_decomp(i, true)
+        compat_decomp = nil if canon_decomp and compat_decomp == canon_decomp
+        canon_offset = handle_decomp(canon_decomp)
+        compat_offset = handle_decomp(compat_decomp)
+        raise RuntimeError,
+          'decomposition string offset beyond not-present offset, upgrade value: offset: %d, max: %d' %
+            [@decomp_string_offset, NotPresentOffset] if
+              @decomp_string_offset > NotPresentOffset
+        self << U::Build::Header::Table::Row.new('%#06x' % i, canon_offset.to_s, compat_offset.to_s)
+      end
+    end
+
+    def to_s
+      "%s\nstatic const char decomp_expansion_string[] = %s;" % [super, @decomp_string]
+    end
+
+  private
+
+    def make_decomp(point, compatible)
+      expand_decomp(point, compatible).flatten.pack('U*')
+    end
+
+    def expand_decomp(point, compatible)
+      @data[point].decomposition.map{ |part|
+        if not @data[part].decomposition.empty? and
+            (compatible or @data[part].decomposition.canonical?)
+          expand_decomp(part, compatible)
+        else
+          part
+        end
+      }
+    end
+
+    def handle_decomp(decomp)
+      return 'UNICODE_NOT_PRESENT_OFFSET' unless decomp
+      return @decomp_offsets[decomp] if @decomp_offsets.include? decomp
+      @decomp_offsets[decomp] = @decomp_string_offset
+      @decomp_string << "\n  \"%s\\0\" /* offset %d */" %
+        [escape(decomp), @decomp_string_offset]
+      result = @decomp_string_offset
+      @decomp_string_offset += decomp.bytesize + 1
+      result
+    end
+
+    def escape(decomp)
+      decomp.unpack('H*')[0].gsub(/(.{2})/, '\\x\1')
+    end
+  end
+end
+
+Decompose.new(Marshal.load(File.open(ARGV[0], 'rb', &:read)))

data/build/ext/u/data/general-category.rb

@@ -0,0 +1,61 @@
+# -*- coding: utf-8 -*-
+
+require 'u/build'
+
+class Category
+  def initialize(data, io = $stdout)
+    U::Build::Header.new(io) do
+      io.puts U::Build::Header::Tables::Split.new(data, 'uint8_t', 'general_category'){ |i|
+        'U_GENERAL_CATEGORY_%s' % Mappings[data[i].type]
+      }
+    end
+  end
+
+private
+
+  # Map general category code onto symbolic name.
+  module Mappings
+    Mappings = {
+      'Cc' => 'OTHER_CONTROL',
+      'Cf' => 'OTHER_FORMAT',
+      'Cn' => 'OTHER_NOT_ASSIGNED',
+      'Co' => 'OTHER_PRIVATE_USE',
+      'Cs' => 'OTHER_SURROGATE',
+      'Ll' => 'LETTER_LOWERCASE',
+      'Lm' => 'LETTER_MODIFIER',
+      'Lo' => 'LETTER_OTHER',
+      'Lt' => 'LETTER_TITLECASE',
+      'Lu' => 'LETTER_UPPERCASE',
+      'Mc' => 'MARK_SPACING_COMBINING',
+      'Me' => 'MARK_ENCLOSING',
+      'Mn' => 'MARK_NON_SPACING',
+      'Nd' => 'NUMBER_DECIMAL',
+      'Nl' => 'NUMBER_LETTER',
+      'No' => 'NUMBER_OTHER',
+      'Pc' => 'PUNCTUATION_CONNECTOR',
+      'Pd' => 'PUNCTUATION_DASH',
+      'Pe' => 'PUNCTUATION_CLOSE',
+      'Pf' => 'PUNCTUATION_FINAL_QUOTE',
+      'Pi' => 'PUNCTUATION_INITIAL_QUOTE',
+      'Po' => 'PUNCTUATION_OTHER',
+      'Ps' => 'PUNCTUATION_OPEN',
+      'Sc' => 'SYMBOL_CURRENCY',
+      'Sk' => 'SYMBOL_MODIFIER',
+      'Sm' => 'SYMBOL_MATH',
+      'So' => 'SYMBOL_OTHER',
+      'Zl' => 'SEPARATOR_LINE',
+      'Zp' => 'SEPARATOR_PARAGRAPH',
+      'Zs' => 'SEPARATOR_SPACE',
+    }.freeze
+
+    class << self
+      def [](value)
+        Mappings[value] or
+          raise 'unknown General_Category property value: %s; the Unicode standard has changed' %
+            value
+      end
+    end
+  end
+end
+
+Category.new(Marshal.load(File.open(ARGV[0], 'rb', &:read)))

data/build/ext/u/data/grapheme-word-break.rb

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+
+require 'u/build'
+
+class Break
+  def initialize(data, breaks, name, io = $stdout)
+    U::Build::Header.new(io) do
+      io.puts U::Build::Header::Tables::Split.new(data, 'uint8_t', '%s_break_property' % name){ |i|
+        'U_%s_BREAK_%s' % [name.upcase, breaks[i].upcase]
+      }
+    end
+  end
+end
+
+Break.new Marshal.load(File.open(ARGV[0], 'rb', &:read)), U::Build::Data::Break.new(ARGV[1]), ARGV[2]

data/build/ext/u/data/marshalled.rb

@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+
+require 'u/build'
+
+Marshal.dump(U::Build::Data::Unicode.new(ARGV[0]), $stdout)

data/build/ext/u/data/script.rb

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+
+require 'u/build'
+
+class Script
+  EasyRange = 0x0000...0x2000
+
+  class << self
+    def name(script)
+      'U_SCRIPT_%s' % script.upcase
+    end
+  end
+
+  def initialize(scripts, io = $stdout)
+    U::Build::Header.new(io) do
+      io.puts <<EOH
+#define EASY_SCRIPTS_RANGE #{EasyRange.end}
+EOH
+      io.puts EasyTable.new(scripts)
+      io.puts Table.new(scripts)
+    end
+  end
+
+private
+
+  class EasyTable < U::Build::Header::Table
+    def initialize(scripts)
+      super 'static const unsigned char script_easy_table[%d]' % EasyRange.end
+      row = Row.new
+      EasyRange.each do |i|
+        row << Script.name(scripts[i])
+      end
+      self << row
+    end
+
+  private
+
+    class Row < U::Build::Header::Table::Row
+    private
+
+      def multi_limit
+        72
+      end
+
+      def multi_format
+        single_format
+      end
+
+      def multi_joiner
+        ",\n\t"
+      end
+
+      def single_format
+        "\t%s"
+      end
+    end
+  end
+
+  class Table < U::Build::Header::Table
+    def initialize(scripts)
+      super "static const struct {
+\tuint32_t start;
+\tuint16_t chars;
+\tuint16_t script;
+} script_table[]"
+      sorted = scripts.select{ |point, _script| point >= EasyRange.end }.sort
+      first = sorted[0][0]
+      last = first - 1
+      current = sorted[0][1]
+      sorted.each do |point, script|
+        if point == last + 1 and script == current
+          last += 1
+        else
+          row first, last, current
+          first = point
+          last = first
+          current = script
+        end
+      end
+      row first, last, current
+    end
+
+  private
+
+    def row(first, last, script)
+      self << U::Build::Header::Table::Row.new('%#06x, %5d, %s' % [first, last - first + 1, Script.name(script)])
+    end
+  end
+end
+
+Script.new(U::Build::Data::Scripts.new(ARGV[0]))

data/build/ext/u/data/soft-dotted.rb

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+require 'u/build'
+
+class SoftDotted
+  def initialize(prop_list, io = $stdout)
+    U::Build::Header.new(io) do
+      io.puts U::Build::Header::Tables::Intervals.
+        new(prop_list.
+              select{ |point, properties| properties.include? 'Soft_Dotted' }.
+              map{ |point, properties| point },
+            'soft_dotted')
+    end
+  end
+end
+
+SoftDotted.new(U::Build::Data::PropList.new(ARGV[0]))

data/build/ext/u/data/title-table.rb

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+
+require 'u/build'
+
+class TitleTable
+  def initialize(data, io = $stdout)
+    U::Build::Header.new(io) do
+      io.puts TitleTable.new(data)
+    end
+  end
+
+private
+
+  class TitleTable < U::Build::Header::Table
+    def initialize(data)
+      super "static const struct {
+\tuint32_t ch;
+\tuint32_t upper;
+\tuint32_t lower;
+} title_table[]"
+      # TODO: Add #code to Entry and use #select on data here.
+      data.each_with_index do |entry, code|
+        next unless entry.title_to_lower
+        self << U::Build::Header::Table::Row.new(*[code, entry.title_to_upper, entry.title_to_lower].map{ |i| '0x%04x' % i })
+      end
+    end
+  end
+end
+
+TitleTable.new(Marshal.load(File.open(ARGV[0], 'rb', &:read)))

data/build/ext/u/data/wide.rb

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+
+require 'u/build'
+
+class Wide
+  def initialize(derived_east_asian_width, types, io = $stdout)
+    U::Build::Header.new(io) do
+      io.puts U::Build::Header::Tables::Intervals.
+        new(derived_east_asian_width.
+              select{ |point, width| types.include? width }.
+              map{ |point, width| point },
+            'wide')
+    end
+  end
+end
+
+Wide.new(U::Build::Data::DerivedEastAsianWidth.new(ARGV[0]), ARGV[1..-1])

data/build/lib/u/build.rb

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+
+module U
+  module Build
+    require 'u/build/data'
+    require 'u/build/header'
+  end
+end

data/build/lib/u/build/data.rb

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+module U::Build::Data
+  require 'u/build/data/file'
+  require 'u/build/data/bidimirroring'
+  require 'u/build/data/break'
+  require 'u/build/data/casefolding'
+  require 'u/build/data/compositionexclusions'
+  require 'u/build/data/derivedeastasianwidth'
+  require 'u/build/data/derivednormalizationprops'
+  require 'u/build/data/linebreak'
+  require 'u/build/data/proplist'
+  require 'u/build/data/scripts'
+  require 'u/build/data/specialcasing'
+  require 'u/build/data/unicode'
+end

data/build/lib/u/build/data/bidimirroring.rb

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+
+class U::Build::Data::BidiMirroring
+  include Enumerable
+
+  def initialize(path)
+    @entries = []
+    U::Build::Data::File.each(path, 2) do |point, mirrored|
+      @entries << Entry.new(point, mirrored.hex)
+    end
+  end
+
+  def each
+    @entries.each do |entry|
+      yield entry
+    end
+  end
+
+  class Entry
+    def initialize(char, mirrored)
+      @char, @mirrored = char, mirrored
+    end
+
+    attr_reader :char, :mirrored
+  end
+end

data/build/lib/u/build/data/break.rb

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+class U::Build::Data::Break
+  def initialize(path)
+    @entries = Hash.new{ 'Other' }
+    U::Build::Data::File.each(path, 2) do |point, property|
+      @entries[point] = property
+    end
+  end
+
+  def [](point)
+    @entries[point]
+  end
+end

data/build/lib/u/build/data/casefolding.rb

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+
+class U::Build::Data::CaseFolding
+  include Enumerable
+
+  def initialize(data, special_casing, path)
+    @entries = []
+    File.new(data, special_casing, path).each do |entry|
+      @entries << entry
+    end
+  end
+
+  def each
+    @entries.each do |entry|
+      yield entry
+    end
+  end
+
+  class Entry
+    def initialize(char, points)
+      @char, @points = char, points
+    end
+
+    def to_s
+      @points.to_s
+    end
+
+    def to_escaped_s
+      @points.to_escaped_s
+    end
+
+    attr_reader :char
+  end
+
+private
+
+  class File
+    def initialize(data, special_casing, path)
+      @data, @special_casing, @path = data, special_casing, path
+    end
+
+    def each
+      U::Build::Data::File.each(@path, 4) do |point, status, mapping|
+        next if simple_or_turkic? status
+        points = U::Build::Data::Unicode::Points.new(mapping)
+        next if simple? points, point
+        yield Entry.new(point, points)
+      end
+    end
+
+  private
+
+    def simple_or_turkic?(status)
+      %w[S T].include? status
+    end
+
+    def simple?(points, code)
+      points.length == 1 and
+        not @special_casing.include? code and
+        @data[code].type and
+        lower(code) == points.first
+    end
+
+    def lower(code)
+      case @data[code].type
+      when 'Ll'
+        code
+      when 'Lt'
+        @data[code].title_to_lower
+      when 'Lu'
+        @data[code].value
+      else
+        code
+      end
+    end
+  end
+end