asciidammit 0.0.1 → 0.0.2

data/Gemfile

@@ -6,5 +6,6 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in to_ascii_latex.gemspec
 gemspec
 
-gem'rake'
+gem 'rake'
+gem 'msgpack'
 

data/asciidammit.gemspec

@@ -20,4 +20,5 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "msgpack"
 end

data/lib/asciidammit/version.rb

@@ -1,3 +1,3 @@
 module AsciiDammit
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/asciidammit.rb

@@ -1,45 +1,110 @@
 require 'open-uri'
-require 'yaml'
+require 'csv'
+require 'msgpack'
 
 module AsciiDammit
-  DATABASEFILE = File.join(File.dirname(__FILE__), 'charmap.yaml')
+  DATABASEFILE = File.join(File.dirname(__FILE__), 'charmap.msgpack')
+  MANUAL = {
+    'DIVISION SIGN'             => '/',
+    'COPYRIGHT SIGN'            => '(c)',
+    'SOFT HYPHEN'               => '-',
+    'REGISTERED SIGN'           => '(r)',
+    'DEGREE SIGN'               => 'o',
+    'PLUS-MINUS SIGN'           => '+/-',
+    'MULTIPLICATION SIGN'       => '*',
+    'INVERTED EXCLAMATION MARK' => '!',
+    'ACUTE ACCENT'              => "'",
+    'HYPHEN'                    => '-',
+    'NON-BREAKING HYPHEN'       => '-',
+    'EN DASH'                   => '-',
+    'EM DASH'                   => '-',
+    'HORIZONTAL BAR'            => '-',
+    'HORIZONTAL ELLIPSIS'       => '...',
+  }
+
   def update
-    database = Hash.new({})
-    open('http://www.unicode.org/Public/3.2-Update/UnicodeData-3.2.0.txt').readlines.each{|line|
-      data = line.strip.split(';')
-      data = {
-        :key => data[0],
-        :char => data[0].length == 4 && data[0] >= '0020' && data[0] <= '007E' ? data[0].to_i(16).chr : nil,
-        :name => data[1],
-        :category => data[2],
-        :decomp => data[5].split.select{|v| v =~ /^[0-9A-Z]+$/},
-        :decimal => data[6].length > 0 ? Integer(data[6]) : nil,
-        :numeric => data[8] =~ /^[0-9]+$/ ? Integer(data[8]) : nil
+    database = {}
+    unmatched = []
+    CSV.parse(open('http://www.unicode.org/Public/3.2-Update/UnicodeData-3.2.0.txt').read, :col_sep => ';').each{|row|
+      next if row[1] == '<control>'
+
+      codepoint = {
+        :codepoint => row[0],
+        :char => row[0].length == 4 && row[0] >= '0020' && row[0] <= '007E' ? row[0].to_i(16).chr : nil,
+        :name => row[1],
+        :category => row[2],
+        :decomp => row[5] ? row[5].split.select{|v| v =~ /^[0-9A-F]{4}$/i}.collect{|cp| cp.intern} : nil,
+        :decimal => row[6] ? Integer(row[6]) : (row[7] ? Integer(row[7]) : nil),
+        :numeric => row[8] =~ /^[0-9]+$/ ? Integer(row[8]) : nil,
+        :upcase => row[12] ? row[12].intern : nil,
+        :downcase => row[13] ? row[13].intern : nil,
       }
-      data[:decomp] = nil if data[:decomp].size == 0
 
-      database[data[:key]] = database[data[:name]] = data
+      # sorry guys, but I really don't know what to make of these
+      next if %w{KANNADA MALAYALAM TELUGU TAMIL ORIYA GUJARATI GURMUKHI BENGALI DEVANAGARI THAANA
+                SYRIAC HEBREW COPTIC HANGUL CYRILLIC ARMENIAN ARABIC SINHALA THAI CHEROKEE
+                OGHAM RUNIC TAGALOG HANUNOO BUHID TAGBANWA KHMER MONGOLIAN MEASURED KANGXI
+                LAO MYANMAR GEORGIAN ETHIOPIC TIBETAN}.include?(codepoint[:name].split[0])
+      next if codepoint[:name] =~ /CANADIAN SYLLABICS /
+      next if codepoint[:name] =~ /PHILIPPINE (SINGLE|DOUBLE) PUNCTUATION/
+      next if codepoint[:name] =~ /BOX DRAWINGS /
+      next if codepoint[:name] =~ /MUSICAL SYMBOL /
+      next if codepoint[:name] =~ /CJK COMPATIBILITY /
+
+      if codepoint[:char].nil?
+        name = codepoint[:name].dup.split.reject{|w| %w{TURNED INVERTED CLOSED REVERSED OPEN}.include?(w)}.join(' ')
+
+        codepoint[:char] ||= $2           if name =~ /LATIN CAPITAL (LIGATURE|LETTER) ([A-Z]+)/ && codepoint[:category] == 'Lu'
+        codepoint[:char] ||= $1           if name =~ /LATIN LETTER ([A-Z]+)/ && codepoint[:category] == 'Lu'
+        codepoint[:char] ||= $3.downcase  if name =~ /LATIN( LETTER)? SMALL (LIGATURE|LETTER|CAPITAL|SCRIPT) ([A-Z]+)/ && codepoint[:category] == 'Ll'
+        codepoint[:char] ||= $1.downcase  if name =~ /LATIN LETTER STRETCHED ([A-Z]+)/ && codepoint[:category] == 'Ll'
+        codepoint[:char] ||= ' '          if name =~ /NO-BREAK SPACE/ && %w{Cf Zs}.include?(codepoint[:category])
+        codepoint[:char] ||= $2.downcase  if name =~ /MODIFIER LETTER SMALL( CAPITAL)? ([A-Z])$/ && codepoint[:category] == 'Lm'
+        codepoint[:char] ||= $2.downcase  if name =~ /MODIFIER LETTER SMALL( CAPITAL)? ([A-Z]) / && codepoint[:category] == 'Lm'
+      end
+
+      codepoint[:char] ||= MANUAL[codepoint[:name]]
+        
+      database[codepoint[:codepoint].intern] = database[codepoint[:name]] = codepoint
     }
 
     database.each_pair{|k, v|
-      next unless k =~ /^[0-9A-Z]+$/
+      next unless k.is_a?(Symbol) && !v[:char]
 
       case v[:category]
         when 'Mn'
-          v[:char] = database[v[:name].gsub(/^COMBINING /, '')][:char]
+          v[:char] = database[v[:name].gsub(/^COMBINING /, '')][:char] if database[v[:name].gsub(/^COMBINING /, '')]
 
         when 'Lu', 'Ll'
-          if v[:char].nil?
-            if v[:decomp]
-              v[:char] = database[v[:decomp][0]][:char]
-            else
-              m = v[:name].match(/(CAPITAL|SMALL) LETTER [A-Z]+ (.)$/)
-              v[:char] = database["LATIN #{m[1]} LETTER #{m[2]}"][:char] if m
-            end
+          if v[:decomp]
+            v[:char] = v[:decomp].collect{|c| database[c]}.compact.collect{|c| c[:char]}.join('')
+            v[:char] = nil if v[:char] == ''
+          else
+            names = v[:name].split.reject{|w| %w{LATIN MACRON SHARP WITH STROKE HOOK TOPBAR DOT ABOVE ACUTE}.include?(w)}
+            names.size.downto(1){|n|
+              names.permutation(n).to_a.each{|name|
+                name = "LATIN " + name.join(' ')
+                v[:char] = database[name][:char] if database[name] && database[name][:char]
+                break if v[:char]
+              }
+              break if v[:char]
+            }
           end
 
-        when 'Lo', 'So', 'Po', 'Sc', 'Mc', 'Sm', 'Ps', 'Pe', 'Pd', 'Cs'
-          v[:char] = database[v[:name].gsub(/.* /, '')][:char] unless v[:char]
+        when 'Lm', 'Lo', 'So', 'Po', 'Sc', 'Mc', 'Sm', 'Ps', 'Pe', 'Pd', 'Cs'
+          names = v[:name].split.reject{|w| %w{PARENTHESIZED CIRCLED ELEMENT VERTICAL HORIZONTAL STROKE LEFTWARDS ARROW BAR OVER
+                                                RIGHTWARDS WITH DOT ABOVE BELOW TAG INVERTED LEFT-POINTING RIGHT-POINTING
+                                                QUADRANT UPPER LEFT LOWER RIGHT
+                                                ANGLE SUPERSCRIPT SUBSCRIPT DOWNWARDS UPWARDS}.include?(w)}
+          STDOUT.flush
+          names.size.downto(1){|n|
+            names.permutation(n).to_a.each{|name|
+              name = name.join(' ')
+              v[:char] = database[name][:char] if database[name] && database[name][:char]
+              break if v[:char]
+            }
+            break if v[:char]
+          }
 
         when 'Lt'
           m = v[:name].match(/(CAPITAL|SMALL) LETTER (.) WITH (CAPITAL|SMALL) LETTER (.)/)
@@ -50,7 +115,7 @@ module AsciiDammit
 
         when 'Nl', 'No', 'Nd'
           if v[:decomp]
-            v[:char] = v[:decomp].collect{|c| database[c][:char].to_s}.join('')
+            v[:char] = v[:decomp].collect{|c| database[c]}.compact.collect{|c| c[:char]}.join('')
             v[:char] = nil if v[:char] == ''
           end
           if v[:char].nil?
@@ -65,28 +130,38 @@ module AsciiDammit
         when 'Zs'
           v[:char] = ' '
 
-        when 'Sk', 'Cf', 'Lm', 'Me', 'Pc', 'Cc', 'Co', 'Zl', 'Zp'
-          #
+        when 'Sk', 'Cf', 'Me', 'Pc', 'Cc', 'Co', 'Zl', 'Zp'
+          v[:ignore] = true
 
         else
           raise "Unhandled character category #{v[:category]}"
       end
+
+      if v[:char].nil?
+        v[:char] = database[v[:upcase]][:char].downcase if v[:upcase] && database[v[:upcase]][:char]
+        v[:char] = database[v[:downcase]][:char].upcase if v[:downcase] && database[v[:downcase]][:char]
+      end
+
+      #puts "#{v[:name]} (#{v[:category]})" unless v[:char] || v[:ignore]
+
+      unmatched << v unless v[:char]
     }
-    database.dup.each_pair{|k, v|
-      next unless k =~ /^[0-9A-Z]+$/ && v[:char]
-      database[k.to_i(16)] = v[:char]
+    database.keys.each{|k|
+      next unless k.is_a?(Symbol)
+      database[eval("\"\\u{#{k}}\"")] = database[k][:char]
     }
-    database = database.reject{|k, v| v.is_a?(Hash)}
-    File.open(DATABASEFILE, "wb", :encoding => 'utf-8'){|f| f.write(database.to_yaml) }
+    database[:unmatched] = unmatched
+    puts "#{database[:unmatched].size} unmatched"
+    File.open(DATABASEFILE, 'wb'){|f| f.write(database.reject{|k, v| v.is_a?(Hash)}.to_msgpack) }
   end
   module_function :update
 
   update unless File.exists?(DATABASEFILE)
-  DATABASE = YAML.load_file(DATABASEFILE)
+  DATABASE = File.open(DATABASEFILE, 'rb'){|f| MessagePack.unpack(f.read)}
 end
 
 class String
   def asciidammit
-    self.split(//).collect{|c| AsciiDammit::DATABASE[c[0].ord].to_s}.join('')
+    self.split(//).collect{|c| a = AsciiDammit::DATABASE[c]; raise "Unhandled char #{c.inspect} (#{a.inspect})" unless a; a.to_s}.join('')
   end
 end