asciify 0.1.0

data/lib/asciify.rb

@@ -0,0 +1,107 @@
+#!/usr/bin/ruby
+
+require 'iconv'
+require 'enumerator'
+require 'yaml'
+
+
+class Asciify
+  Intermediate = "UCS-4"
+  PackFormat = "N*"
+
+  class Mapping
+    
+    # converts an UTF-8 string to an array of unicode codepoints
+    #
+    def from_utf8(str)
+      Iconv.new(Intermediate,"UTF-8").iconv(str).unpack(PackFormat)
+    end
+    
+    # define a mapping from Unicode codepoints to ASCII chars
+    # +language+ can be a path to a yaml file which contains the
+    # mappings as a Hash
+    #
+    # If +language+ is a symbol, it refers to a builtin mapping
+    #
+    def initialize(language = :default, replacement = "?")
+      
+      if Symbol === language
+        path = "#{File.dirname(__FILE__)}/mappings/#{language}.yaml"
+      else
+        path = language
+      end
+      
+      h = YAML.load_file(path)
+      i = Iconv.new("UCS-4","UTF-8")
+      
+      # use the default replacement if the hash 
+      @map = Hash.new( i.iconv(replacement).unpack(PackFormat) )
+        
+      # the mappings file is UTF-8, recode to UCS-4
+      h.each { |k,v|
+        @map[*i.iconv(k).unpack(PackFormat)] = i.iconv(v).unpack(PackFormat)
+      }
+      
+      @map
+    end
+
+    def [](codepoint)
+      @map[codepoint]
+    end
+
+  end
+
+  class HTMLEntities < Mapping
+    
+    # mapping from Unicode codepoints to numeric HTML entities
+    #
+    #   Asciify.new(Asciify::HTMLEntities).convert("\303\244") #=> "&#228;"
+    #
+    def [](codepoint)
+      from_utf8 "&##{codepoint};"
+    end
+  end
+  
+  def initialize(replacement = "?", target = "ASCII", source = "UTF-8")
+    @from_input_enc = Iconv.new(Intermediate, source)
+    @to_output_enc = Iconv.new(target, Intermediate)
+
+    if String === replacement
+      r = @from_input_enc.iconv(replacement).unpack(PackFormat)
+      @mapping = Hash.new(r)
+    else
+      @mapping = replacement
+    end
+  end
+
+  def convert(str)
+    u16s = @from_input_enc.iconv(str)
+  
+    s = u16s.unpack(PackFormat).collect { |codepoint|
+      codepoint < 128 ? codepoint : @mapping[codepoint]
+    }.flatten.compact.pack(PackFormat)
+  
+    return @to_output_enc.iconv(s)
+  end
+
+end
+
+class String
+  
+  # removes all characters which are not part of ascii
+  # and replaces them with +replacement+
+  #
+  # +replacement+ is supposed to be the same encoding as +source+
+  #
+  def asciify(*args)
+    Asciify.new(*args).convert(self)
+  end
+
+  def ascii?
+    self.to_enum(:each_byte).all? { |b| b < 128 }
+  end
+end
+
+if __FILE__ == $0
+end
+

data/lib/mappings/default.yaml

@@ -0,0 +1,27 @@
+--- 
+"“": '"'
+"”": '"'
+"‘": "'"
+"’": "'"
+"„": '"'
+"〝": '"'
+"〞": '"'
+"»": ">>"
+"«": "<<"
+"ä": "ae"
+"ö": "oe"
+"ü": "ue"
+"Ä": "Ae"
+"Ö": "Oe"
+"Ü": "Ue"
+"ß": "ss"
+"æ": "ae"
+"Æ": "AE"
+"œ": "oe"
+"Œ": "OE"
+"€": "EUR"
+"½": "1/2"
+"¼": "1/4"
+"¾": "3/4"
+"©": "(c)"
+"®": "(r)"

metadata

@@ -0,0 +1,41 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.8.11
+specification_version: 1
+name: asciify
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+date: 2006-01-24 00:00:00 +01:00
+summary: Tool to strip non-ASCII characters from a string and replace them with something else
+require_paths: 
+  - lib
+email: levin@grundeis.net
+homepage: http://levinalex.de/ruby/asciify
+rubyforge_project: 
+description: 
+autorequire: 
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+    - 
+      - ">"
+      - !ruby/object:Gem::Version 
+        version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+authors: 
+  - Levin Alexander
+files: 
+  - lib/asciify.rb
+  - lib/mappings
+  - lib/mappings/default.yaml
+test_files: []
+rdoc_options: []
+extra_rdoc_files: []
+executables: []
+extensions: []
+requirements: []
+dependencies: []