textutils 0.5.1 → 0.5.2

data/Rakefile

@@ -18,7 +18,7 @@ Hoe.spec 'textutils' do
   self.history_file = 'History.markdown'
 
   self.extra_deps = [
-    ['logutils', '>= 0.5.0']
+    ['logutils', '~> 0.5']  # e.g. >= 0.5 <= 1.0
   ]
 
   self.licenses = ['Public Domain']

data/lib/textutils/reader/values_reader.rb

@@ -130,31 +130,59 @@ class ValuesReader
       ## todo: add some more
       ## see http://en.wikipedia.org/wiki/List_of_XML_and_HTML_character_entity_references  for more
       
+      ## todo: add unicode codepoint name
+      
       alternatives = [
         ['ß', 'ss'],
         ['æ', 'ae'],
-        ['ä', 'ae'],  
+        ['ä', 'ae'],
+        ['ā', 'a' ],  # e.g. Liepājas
         ['á', 'a' ],  # e.g. Bogotá, Králové
         ['ã', 'a' ],  # e.g  São Paulo
         ['ă', 'a' ],  # e.g. Chișinău
+        ['â', 'a' ],  # e.g  Goiânia
+        ['å', 'a' ],  # e.g. Vålerenga
+        ['ą', 'a' ],  # e.g. Śląsk
+        ['ç', 'c' ],  # e.g. São Gonçalo, Iguaçu, Neftçi
+        ['ć', 'c' ],  # e.g. Budućnost
         ['é', 'e' ],  # e.g. Vélez, Králové
         ['è', 'e' ],  # e.g. Rivières
         ['ê', 'e' ],  # e.g. Grêmio
         ['ě', 'e' ],  # e.g. Budějovice
+        ['ė', 'e' ],  # e.g. Vėtra
+        ['ë', 'e' ],  # e.g. Skënderbeu
+        ['ğ', 'g' ],  # e.g. Qarabağ
         ['ì', 'i' ],  # e.g. Potosì
         ['í', 'i' ],  # e.g. Ústí
+        ['ł', 'l' ],  # e.g. Wisła, Wrocław
         ['ñ', 'n' ],  # e.g. Porteño
         ['ň', 'n' ],  # e.g. Plzeň, Třeboň
-        ['ö', 'oe'],  
+        ['ö', 'oe'],
+        ['ő', 'o' ],  # e.g. Győri
         ['ó', 'o' ],  # e.g. Colón, Łódź, Kraków
+        ['õ', 'o' ],  # e.g. Nõmme
+        ['ø', 'o' ],  # e.g. Fuglafjørdur, København
         ['ř', 'r' ],  # e.g. Třeboň
-        ['ș', 's' ],  # e.g. Chișinău
-        ['ü', 'ue'],  
+        ['ș', 's' ],  # e.g. Chișinău, București
+        ['ş', 's' ],  # e.g. Beşiktaş
+        ['š', 's' ],  # e.g. Košice
+        ['ü', 'ue'],
         ['ú', 'u' ],  # e.g. Fútbol
+        ['ū', 'u' ],  # e.g. Sūduva
+        ['ı', 'u' ],  # e.g. Bakı   # use u?? (Baku) why-why not?
         ['ź', 'z' ],  # e.g. Łódź
+        ['ž', 'z' ],  # e.g. Domžale, Petržalka
+
         ['Č', 'c' ],  # e.g. České
+        ['İ', 'i' ],  # e.g. İnter
+        ['Í', 'i' ],  # e.g. ÍBV
         ['Ł', 'l' ],  # e.g. Łódź
-        ['Ú', 'u' ],  # e.g. Ústí
+        ['Ö', 'oe' ], # e.g. Örebro
+        ['Ś', 's' ],  # e.g. Śląsk
+        ['Š', 's' ],  # e.g. MŠK
+        ['Ş', 's' ],  # e.g. Şüvälan
+        ['Ú', 'u' ],  # e.g. Ústí, Újpest
+        ['Ž', 'z' ]   # e.g. Žilina
       ]
       
       alternatives.each do |alt|

data/lib/textutils/utils.rb

@@ -6,18 +6,61 @@ class File
     text = open( path, 'r:bom|utf-8' ) do |file|
       file.read
     end
-    ### for convenience for now convert fancy dash to "plain" dash
-    ## todo: check char codes for "fancy" dash alternatives
-    text.gsub!( /—|–/ ) do |_|
-      puts "*** warning: convert fancy dash to 'plain' dash in file >#{path}<"
-      ### exit 1   #### do NOT tolerate!! cleanup dashes for now
-      '-'
-    end
+
+    # NB: for convenience: convert fancy unicode dashes/hyphens to plain ascii hyphen-minus
+    text = convert_unicode_dashes_to_plain_ascii( text, path: path )
+
     text
   end
 end # class File
 
+
+# NB:
+#  U_HYPHEN_MINUS is standard ascii hyphen/minus e.g. -
+#
+#  see en.wikipedia.org/wiki/Dash
+
+U_HYPHEN              = "\u2010"  # unambigous hyphen
+U_NON_BREAKING_HYPHEN = "\u2011"  # unambigous non-breaking hyphen
+U_MINUS               = "\u2212"  # unambigous minus sign (html => &minus;)
+U_NDASH               = "\u2013"  # ndash (html => &ndash; ascii => --)
+U_MDASH               = "\u2014"  # mdash (html => &mdash; ascii => ---)
+
+
+  def convert_unicode_dashes_to_plain_ascii( text, opts = {} )
+    
+    text = text.gsub( /(#{U_HYPHEN}|#{U_NON_BREAKING_HYPHEN}|#{U_MINUS}|#{U_NDASH}|#{U_MDASH})/ ) do |_|
+
+      # puts "found U+#{'%04X' % $1.ord} (#{$1})"
+
+      msg = ''
+
+      if $1 == U_HYPHEN
+        msg << "found hyhpen U+2010 (#{$1})"
+      elsif $1 == U_NON_BREAKING_HYPHEN
+        msg << "found non_breaking_hyhpen U+2011 (#{$1})"
+      elsif $1 == U_MINUS
+        msg << "found minus U+2212 (#{$1})"
+      elsif $1 == U_NDASH
+        msg << "found ndash U+2013 (#{$1})"
+      elsif $1 == U_MDASH
+        msg << "found mdash U+2014 (#{$1})"
+      else
+        msg << "found unknown unicode dash U+#{'%04X' % $1.ord} (#{$1})"
+      end
+
+      msg << " in file >#{opts[:path]}<"   if opts[:path]
+      msg << "; converting to plain ascii hyphen_minus (-)"
   
+      puts "*** warning: #{msg}"
+
+      '-'
+    end
+
+    text
+  end # method convert_unicode_dashes_to_plain_ascii
+
+
   ############
   ### fix/todo: share helper for all text readers/parsers- where to put it?  
   ###

data/lib/textutils/version.rb

@@ -1,7 +1,6 @@
 
 module TextUtils
 
-  VERSION = '0.5.1'
+  VERSION = '0.5.2'
 
 end   # module TextUtils
-

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: textutils
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,22 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-02-27 00:00:00.000000000 Z
+date: 2013-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logutils
-  requirement: &75610580 !ruby/object:Gem::Requirement
+  requirement: &84567730 !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ! '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: 0.5.0
+        version: '0.5'
   type: :runtime
   prerelease: false
-  version_requirements: *75610580
+  version_requirements: *84567730
 - !ruby/object:Gem::Dependency
   name: rdoc
-  requirement: &75610310 !ruby/object:Gem::Requirement
+  requirement: &84567510 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -32,10 +32,10 @@ dependencies:
         version: '3.10'
   type: :development
   prerelease: false
-  version_requirements: *75610310
+  version_requirements: *84567510
 - !ruby/object:Gem::Dependency
   name: hoe
-  requirement: &75610040 !ruby/object:Gem::Requirement
+  requirement: &84567260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -43,7 +43,7 @@ dependencies:
         version: '3.3'
   type: :development
   prerelease: false
-  version_requirements: *75610040
+  version_requirements: *84567260
 description: textutils - Text Filters, Helpers, Readers and More
 email: webslideshow@googlegroups.com
 executables: []