unicode-multibyte 0.1.0

data/.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>UnicodeChars</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.rubypeople.rdt.core.rubybuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.rubypeople.rdt.core.rubynature</nature>
+	</natures>
+</projectDescription>

data/Rakefile

@@ -0,0 +1,18 @@
+require "bundler"
+require "rake/testtask"
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = Dir.glob("test/**/*test.rb")
+  t.verbose = true
+end
+
+desc 'Builds the gem'
+task :build do
+  sh "gem build unicode-multibyte.gemspec"
+end
+
+desc 'Builds and installs the gem'
+task :install => :build do
+  sh "gem install unicode-multibyte-0.1.0"
+end

data/lib/unicodechars/multibyte/chars.rb

@@ -0,0 +1,155 @@
+require File.join(File.dirname(__FILE__), "handlers", "utf8_handler")
+require File.join(File.dirname(__FILE__), "handlers", "passthru_handler")
+
+# Encapsulates all the functionality related to the Chars proxy.
+module UnicodeMultibyte::Multibyte
+  # Chars enables you to work transparently with multibyte encodings in the Ruby String class without having extensive
+  # knowledge about the encoding. A Chars object accepts a string upon initialization and proxies String methods in an
+  # encoding safe manner. All the normal String methods are also implemented on the proxy.
+  #
+  # String methods are proxied through the Chars object, and can be accessed through the +chars+ method. Methods
+  # which would normally return a String object now return a Chars object so methods can be chained.
+  #
+  #   "The Perfect String  ".chars.downcase.strip.normalize #=> "the perfect string"
+  #
+  # Chars objects are perfectly interchangeable with String objects as long as no explicit class checks are made.
+  # If certain methods do explicitly check the class, call +to_s+ before you pass chars objects to them.
+  #
+  #   bad.explicit_checking_method "T".chars.downcase.to_s
+  #
+  # The actual operations on the string are delegated to handlers. Theoretically handlers can be implemented for
+  # any encoding, but the default handler handles UTF-8. This handler is set during initialization, if you want to
+  # use you own handler, you can set it on the Chars class. Look at the UTF8Handler source for an example how to
+  # implement your own handler. If you your own handler to work on anything but UTF-8 you probably also
+  # want to override Chars#handler.
+  #
+  #   UnicodeMultibyte::Multibyte::Chars.handler = MyHandler
+  #
+  # Note that a few methods are defined on Chars instead of the handler because they are defined on Object or Kernel
+  # and method_missing can't catch them.
+  class Chars
+    
+    attr_reader :string # The contained string
+    alias_method :to_s, :string
+    
+    include Comparable
+    
+    # The magic method to make String and Chars comparable
+    def to_str
+      # Using any other ways of overriding the String itself will lead you all the way from infinite loops to
+      # core dumps. Don't go there.
+      @string
+    end
+
+    # Makes unicode string look like a string in the console
+    def inspect
+      @string.inspect
+    end
+    
+    def is_a?(type)
+      if type == String
+        true
+      else
+        super
+      end
+    end
+    
+    # Fix [] for single numbers
+    def [](num)
+      if num.is_a?(Fixnum)
+        self[num..num]
+      else
+        super
+      end
+    end
+
+    # Create a new Chars instance.
+    def initialize(str)
+      @string = (str.string rescue str)
+    end
+    
+    def each &block
+      split(//).each(&block)
+    end
+    
+    # Returns -1, 0 or +1 depending on whether the Chars object is to be sorted before, equal or after the
+    # object on the right side of the operation. It accepts any object that implements +to_s+. See String.<=>
+    # for more details.
+    def <=>(other); @string <=> other.to_s; end
+    
+    # Works just like String#split, with the exception that the items in the resulting list are Chars
+    # instances instead of String. This makes chaining methods easier.
+    def split(*args)
+      @string.split(*args).map { |i| i.chars }
+    end
+    
+    # Gsub works exactly the same as gsub on a normal string.
+    def gsub(*a, &b); @string.gsub(*a, &b).chars; end
+    
+    # Like String.=~ only it returns the character offset (in codepoints) instead of the byte offset.
+    def =~(other)
+      handler.translate_offset(@string, @string =~ other)
+    end
+    
+    # Try to forward all undefined methods to the handler, when a method is not defined on the handler, send it to
+    # the contained string. Method_missing is also responsible for making the bang! methods destructive.
+    def method_missing(m, *a, &b)
+      begin
+        # Simulate methods with a ! at the end because we can't touch the enclosed string from the handlers.
+        if m.to_s =~ /^(.*)\!$/
+          result = handler.send($1, @string, *a, &b)
+          if result == @string
+            result = nil
+          else
+            @string.replace result
+          end
+        else
+          result = handler.send(m, @string, *a, &b)
+        end
+      rescue NoMethodError
+        result = @string.send(m, *a, &b)
+      rescue Handlers::EncodingError
+        @string.replace handler.tidy_bytes(@string)
+        retry
+      end
+      
+      if result.kind_of?(String)
+        result.chars
+      else
+        result
+      end
+    end
+    
+    # Set the handler class for the Char objects.
+    def self.handler=(klass)
+      @@handler = klass
+    end
+
+    # Returns the proper handler for the contained string depending on $KCODE and the encoding of the string. This
+    # method is used internally to always redirect messages to the proper classes depending on the context.
+    def handler
+      if utf8_pragma?
+        @@handler
+      else
+        UnicodeMultibyte::Multibyte::Handlers::PassthruHandler
+      end
+    end
+
+    private
+      
+      # +utf8_pragma+ checks if it can send this string to the handlers. It makes sure @string isn't nil and $KCODE is
+      # set to 'UTF8'.
+      def utf8_pragma?
+        !@string.nil? && ($KCODE == 'UTF8')
+      end
+  end
+end
+
+# When we can load the utf8proc library, override normalization with the faster methods
+begin
+  require 'utf8proc_native'
+  require File.join(File.dirname(__FILE__), "handlers", "utf8_handler_proc")
+  UnicodeMultibyte::Multibyte::Chars.handler = UnicodeMultibyte::Multibyte::Handlers::UTF8HandlerProc
+rescue LoadError
+  UnicodeMultibyte::Multibyte::Chars.handler = UnicodeMultibyte::Multibyte::Handlers::UTF8Handler
+end

data/lib/unicodechars/multibyte/generators/generate_tables.rb

@@ -0,0 +1,146 @@
+#!/usr/bin/env ruby
+
+require 'open-uri'
+require 'tmpdir'
+
+module UnicodeMultibyte::Multibyte::Handlers #:nodoc:
+  class UnicodeDatabase #:nodoc:
+    def self.load
+      [Hash.new(Codepoint.new),[],{},{}]
+    end
+  end
+  
+  class UnicodeTableGenerator #:nodoc:
+    BASE_URI = "http://www.unicode.org/Public/#{UnicodeMultibyte::Multibyte::UNICODE_VERSION}/ucd/"
+    SOURCES = {
+      :codepoints => BASE_URI + 'UnicodeData.txt',
+      :composition_exclusion => BASE_URI + 'CompositionExclusions.txt',
+      :grapheme_break_property => BASE_URI + 'auxiliary/GraphemeBreakProperty.txt',
+      :cp1252 => 'http://unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP1252.TXT'
+    }
+    
+    def initialize
+      @ucd = UnicodeDatabase.new
+      
+      default = Codepoint.new
+      default.combining_class = 0
+      default.uppercase_mapping = 0
+      default.lowercase_mapping = 0
+      @ucd.codepoints = Hash.new(default)
+      
+      @ucd.composition_exclusion = []
+      @ucd.composition_map = {}
+      @ucd.boundary = {}
+      @ucd.cp1252 = {}
+    end
+    
+    def parse_codepoints(line)
+      codepoint = Codepoint.new
+      raise "Could not parse input." unless line =~ /^
+        ([0-9A-F]+);        # code
+        ([^;]+);            # name
+        ([A-Z]+);           # general category
+        ([0-9]+);           # canonical combining class
+        ([A-Z]+);           # bidi class
+        (<([A-Z]*)>)?       # decomposition type
+        ((\ ?[0-9A-F]+)*);  # decompomposition mapping
+        ([0-9]*);           # decimal digit
+        ([0-9]*);           # digit
+        ([^;]*);            # numeric
+        ([YN]*);            # bidi mirrored
+        ([^;]*);            # unicode 1.0 name
+        ([^;]*);            # iso comment
+        ([0-9A-F]*);        # simple uppercase mapping
+        ([0-9A-F]*);        # simple lowercase mapping
+        ([0-9A-F]*)$/ix     # simple titlecase mapping
+      codepoint.code              = $1.hex
+      #codepoint.name              = $2
+      #codepoint.category          = $3
+      codepoint.combining_class   = Integer($4)
+      #codepoint.bidi_class        = $5
+      codepoint.decomp_type       = $7
+      codepoint.decomp_mapping    = ($8=='') ? nil : $8.split.collect { |element| element.hex }
+      #codepoint.bidi_mirrored     = ($13=='Y') ? true : false
+      codepoint.uppercase_mapping = ($16=='') ? 0 : $16.hex
+      codepoint.lowercase_mapping = ($17=='') ? 0 : $17.hex
+      #codepoint.titlecase_mapping = ($18=='') ? nil : $18.hex
+      @ucd.codepoints[codepoint.code] = codepoint
+    end
+
+    def parse_grapheme_break_property(line)
+      if line =~ /^([0-9A-F\.]+)\s*;\s*([\w]+)\s*#/
+        type = $2.downcase.intern
+        @ucd.boundary[type] ||= []
+        if $1.include? '..'
+          parts = $1.split '..'
+          @ucd.boundary[type] << (parts[0].hex..parts[1].hex)
+        else
+          @ucd.boundary[type] << $1.hex
+        end
+      end
+    end
+    
+    def parse_composition_exclusion(line)
+      if line =~ /^([0-9A-F]+)/i
+        @ucd.composition_exclusion << $1.hex
+      end
+    end
+    
+    def parse_cp1252(line)
+      if line =~ /^([0-9A-Fx]+)\s([0-9A-Fx]+)/i
+        @ucd.cp1252[$1.hex] = $2.hex
+      end
+    end
+    
+    def create_composition_map
+      @ucd.codepoints.each do |_, cp|
+        if !cp.nil? and cp.combining_class == 0 and cp.decomp_type.nil? and !cp.decomp_mapping.nil? and cp.decomp_mapping.length == 2 and @ucd[cp.decomp_mapping[0]].combining_class == 0 and !@ucd.composition_exclusion.include?(cp.code)
+          @ucd.composition_map[cp.decomp_mapping[0]] ||= {}
+          @ucd.composition_map[cp.decomp_mapping[0]][cp.decomp_mapping[1]] = cp.code
+        end
+      end
+    end
+
+    def normalize_boundary_map
+      @ucd.boundary.each do |k,v|
+        if [:lf, :cr].include? k
+          @ucd.boundary[k] = v[0]
+        end
+      end
+    end
+  
+    def parse
+      SOURCES.each do |type, url|
+        filename =  File.join(Dir.tmpdir, "#{url.split('/').last}")
+        unless File.exist?(filename)
+          $stderr.puts "Downloading #{url.split('/').last}"
+          File.open(filename, 'wb') do |target|
+            open(url) do |source|
+              source.each_line { |line| target.write line }
+            end
+          end
+        end
+        File.open(filename) do |file|
+          file.each_line { |line| send "parse_#{type}".intern, line }
+        end        
+      end
+      create_composition_map
+      normalize_boundary_map
+    end
+    
+    def dump_to(filename)
+      File.open(filename, 'wb') do |f|
+        f.write Marshal.dump([@ucd.codepoints, @ucd.composition_exclusion, @ucd.composition_map, @ucd.boundary, @ucd.cp1252])
+      end
+    end
+  end
+end
+
+if __FILE__ == $0
+  filename = UnicodeMultibyte::Multibyte::Handlers::UnicodeDatabase.filename
+  generator = UnicodeMultibyte::Multibyte::Handlers::UnicodeTableGenerator.new
+  generator.parse
+  print "Writing to: #{filename}"
+  generator.dump_to filename
+  puts " (#{File.size(filename)} bytes)"
+end

data/lib/unicodechars/multibyte/handlers/passthru_handler.rb

@@ -0,0 +1,9 @@
+# Chars uses this handler when $KCODE is not set to 'UTF8'. Because this handler doesn't define any methods all call
+# will be forwarded to String.
+class UnicodeMultibyte::Multibyte::Handlers::PassthruHandler
+  
+  # Return the original byteoffset
+  def self.translate_offset(string, byte_offset) #:nodoc:
+    byte_offset
+  end
+end