mosaheh 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in mosaheh.gemspec
+gemspec

data/README.md

@@ -0,0 +1,46 @@
+Mosaheh
+=======
+
+Mosaheh, Arabic for corrector, is a ruby lib which can repair UTF-8 Arabic (U+0600 - U+06FF) text that has been mistakenly saved as single-byte latin1 encoding (cp1252). 
+
+Usage
+-----
+    
+    # encoding: UTF-8
+    require 'mosaheh'
+
+    encoder = Mosaheh::Encoder.new
+    encoder.repair 'عربي' # => عربي
+
+When to use
+-----------
+
+The biggest usecase for it is to repair Arabic data stored in MySql databases with the wrong encoding.
+
+Imagine that you have a MySql database which has the correct collation for a utf-8 data, but the MySql driver was sending the data as latin1 (default in PHPs PDO for example). 
+In this case Mosaheh will be your savior, especially if you have a lot of invaluable data.
+
+LICENSE
+-------
+
+(The MIT License)
+
+> Copyright (C) 2011 by Maher Sallam
+> 
+> Permission is hereby granted, free of charge, to any person obtaining a copy
+> of this software and associated documentation files (the "Software"), to deal
+> in the Software without restriction, including without limitation the rights
+> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+> copies of the Software, and to permit persons to whom the Software is
+> furnished to do so, subject to the following conditions:
+> 
+> The above copyright notice and this permission notice shall be included in
+> all copies or substantial portions of the Software.
+> 
+> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+> THE SOFTWARE.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/mosaheh/encoder.rb

@@ -0,0 +1,118 @@
+# encoding: UTF-8
+
+class Mosaheh::Encoder
+    
+  # Arabic UTF-8 Block, from U+0600 to U+06FF seperated using '_*_'
+  AR = '؀_*_؁_*_؂_*_؃_*_؄_*_؅_*_؆_*_؇_*_؈_*_؉_*_؊_*_؋_*_،_*_؍_*_؎_*_؏_*_ؐ_*_ؑ_*_ؒ_*_ؓ_*_ؔ_*_ؕ_*_ؖ_*_ؗ_*_ؘ_*_ؙ_*_ؚ_*_؛_*_؜_*_؝_*_؞_*_؟_*_ؠ_*_ء_*_آ_*_أ_*_ؤ_*_إ_*_ئ_*_ا_*_ب_*_ة_*_ت_*_ث_*_ج_*_ح_*_خ_*_د_*_ذ_*_ر_*_ز_*_س_*_ش_*_ص_*_ض_*_ط_*_ظ_*_ع_*_غ_*_ػ_*_ؼ_*_ؽ_*_ؾ_*_ؿ_*_ـ_*_ف_*_ق_*_ك_*_ل_*_م_*_ن_*_ه_*_و_*_ى_*_ي_*_ً_*_ٌ_*_ٍ_*_َ_*_ُ_*_ِ_*_ّ_*_ْ_*_ٓ_*_ٔ_*_ٕ_*_ٖ_*_ٗ_*_٘_*_ٙ_*_ٚ_*_ٛ_*_ٜ_*_ٝ_*_ٞ_*_ٟ_*_٠_*_١_*_٢_*_٣_*_٤_*_٥_*_٦_*_٧_*_٨_*_٩_*_٪_*_٫_*_٬_*_٭_*_ٮ_*_ٯ_*_ٰ_*_ٱ_*_ٲ_*_ٳ_*_ٴ_*_ٵ_*_ٶ_*_ٷ_*_ٸ_*_ٹ_*_ٺ_*_ٻ_*_ټ_*_ٽ_*_پ_*_ٿ_*_ڀ_*_ځ_*_ڂ_*_ڃ_*_ڄ_*_څ_*_چ_*_ڇ_*_ڈ_*_ډ_*_ڊ_*_ڋ_*_ڌ_*_ڍ_*_ڎ_*_ڏ_*_ڐ_*_ڑ_*_ڒ_*_ړ_*_ڔ_*_ڕ_*_ږ_*_ڗ_*_ژ_*_ڙ_*_ښ_*_ڛ_*_ڜ_*_ڝ_*_ڞ_*_ڟ_*_ڠ_*_ڡ_*_ڢ_*_ڣ_*_ڤ_*_ڥ_*_ڦ_*_ڧ_*_ڨ_*_ک_*_ڪ_*_ګ_*_ڬ_*_ڭ_*_ڮ_*_گ_*_ڰ_*_ڱ_*_ڲ_*_ڳ_*_ڴ_*_ڵ_*_ڶ_*_ڷ_*_ڸ_*_ڹ_*_ں_*_ڻ_*_ڼ_*_ڽ_*_ھ_*_ڿ_*_ۀ_*_ہ_*_ۂ_*_ۃ_*_ۄ_*_ۅ_*_ۆ_*_ۇ_*_ۈ_*_ۉ_*_ۊ_*_ۋ_*_ی_*_ۍ_*_ێ_*_ۏ_*_ې_*_ۑ_*_ے_*_ۓ_*_۔_*_ە_*_ۖ_*_ۗ_*_ۘ_*_ۙ_*_ۚ_*_ۛ_*_ۜ_*_۝_*_۞_*_۟_*_۠_*_ۡ_*_ۢ_*_ۣ_*_ۤ_*_ۥ_*_ۦ_*_ۧ_*_ۨ_*_۩_*_۪_*_۫_*_۬_*_ۭ_*_ۮ_*_ۯ_*_۰_*_۱_*_۲_*_۳_*_۴_*_۵_*_۶_*_۷_*_۸_*_۹_*_ۺ_*_ۻ_*_ۼ_*_۽_*_۾_*_ۿ'
+  
+  # U+0600 to U+06FF encoded using cp1252 
+  BROKEN_AR = 'Ø€_*_؁_*_Ø‚_*_؃_*_Ø„_*_Ø…_*_؆_*_؇_*_؈_*_؉_*_ØŠ_*_Ø‹_*_ØŒ_*_؍_*_ØŽ_*_؏_*_ؐ_*_Ø‘_*_Ø’_*_Ø“_*_Ø”_*_Ø•_*_Ø–_*_Ø—_*_ؘ_*_Ø™_*_Øš_*_Ø›_*_Øœ_*_؝_*_Øž_*_ØŸ_*_Ø _*_Ø¡_*_Ø¢_*_Ø£_*_ؤ_*_Ø¥_*_ئ_*_ا_*_ب_*_Ø©_*_ت_*_Ø«_*_ج_*_Ø­_*_Ø®_*_د_*_Ø°_*_ر_*_ز_*_س_*_Ø´_*_ص_*_ض_*_Ø·_*_ظ_*_ع_*_غ_*_Ø»_*_ؼ_*_ؽ_*_ؾ_*_Ø¿_*_Ù€_*_ف_*_Ù‚_*_Ùƒ_*_Ù„_*_Ù…_*_Ù†_*_Ù‡_*_Ùˆ_*_Ù‰_*_ÙŠ_*_Ù‹_*_ÙŒ_*_ٍ_*_ÙŽ_*_ُ_*_ِ_*_Ù‘_*_Ù’_*_Ù“_*_Ù”_*_Ù•_*_Ù–_*_Ù—_*_Ù˜_*_Ù™_*_Ùš_*_Ù›_*_Ùœ_*_ٝ_*_Ùž_*_ÙŸ_*_Ù _*_Ù¡_*_Ù¢_*_Ù£_*_Ù¤_*_Ù¥_*_Ù¦_*_Ù§_*_Ù¨_*_Ù©_*_Ùª_*_Ù«_*_Ù¬_*_Ù­_*_Ù®_*_Ù¯_*_Ù°_*_Ù±_*_Ù²_*_Ù³_*_Ù´_*_Ùµ_*_Ù¶_*_Ù·_*_Ù¸_*_Ù¹_*_Ùº_*_Ù»_*_Ù¼_*_Ù½_*_Ù¾_*_Ù¿_*_Ú€_*_ځ_*_Ú‚_*_Úƒ_*_Ú„_*_Ú…_*_Ú†_*_Ú‡_*_Úˆ_*_Ú‰_*_ÚŠ_*_Ú‹_*_ÚŒ_*_ڍ_*_ÚŽ_*_ڏ_*_ڐ_*_Ú‘_*_Ú’_*_Ú“_*_Ú”_*_Ú•_*_Ú–_*_Ú—_*_Ú˜_*_Ú™_*_Úš_*_Ú›_*_Úœ_*_ڝ_*_Úž_*_ÚŸ_*_Ú _*_Ú¡_*_Ú¢_*_Ú£_*_Ú¤_*_Ú¥_*_Ú¦_*_Ú§_*_Ú¨_*_Ú©_*_Úª_*_Ú«_*_Ú¬_*_Ú­_*_Ú®_*_Ú¯_*_Ú°_*_Ú±_*_Ú²_*_Ú³_*_Ú´_*_Úµ_*_Ú¶_*_Ú·_*_Ú¸_*_Ú¹_*_Úº_*_Ú»_*_Ú¼_*_Ú½_*_Ú¾_*_Ú¿_*_Û€_*_ہ_*_Û‚_*_Ûƒ_*_Û„_*_Û…_*_Û†_*_Û‡_*_Ûˆ_*_Û‰_*_ÛŠ_*_Û‹_*_ÛŒ_*_ۍ_*_ÛŽ_*_ۏ_*_ې_*_Û‘_*_Û’_*_Û“_*_Û”_*_Û•_*_Û–_*_Û—_*_Û˜_*_Û™_*_Ûš_*_Û›_*_Ûœ_*_۝_*_Ûž_*_ÛŸ_*_Û _*_Û¡_*_Û¢_*_Û£_*_Û¤_*_Û¥_*_Û¦_*_Û§_*_Û¨_*_Û©_*_Ûª_*_Û«_*_Û¬_*_Û­_*_Û®_*_Û¯_*_Û°_*_Û±_*_Û²_*_Û³_*_Û´_*_Ûµ_*_Û¶_*_Û·_*_Û¸_*_Û¹_*_Ûº_*_Û»_*_Û¼_*_Û½_*_Û¾_*_Û¿'
+
+  def initialize options = {}
+    @replace_char = options[:replace_char] || '?'
+
+    generate_mappings_hash
+  end
+
+  def show_mappings_hash
+    i = 0
+    justification = 0;
+    mappings = ''
+    ar = AR.split('_*_')
+
+    @map.each do |broken, good|
+
+      str = "[#{broken.join(', ')}]"
+      
+      justification = str.length if str.length > justification
+      str = str.ljust justification      
+
+      str += ' => '
+      str += "[#{good.join(', ')}]"
+      str += " # #{ar[i]}\n"
+
+      mappings += str
+
+      i += 1
+    end
+
+    puts mappings
+  end
+
+  def repair(str)
+    @broken = str.unpack('C*')
+    @repaired = []
+
+    while @broken.length > 0
+
+      # Try to use the mappings hash first and jump to the
+      # next sequence if we succeed
+      next if sequence_found_in_map
+
+      # Try to handle ASCII chars if they are not a part
+      # of a broken sequence
+      next if byte_not_broken
+
+      handle_unknown_byte
+    end
+
+    result = @repaired.pack('C*')
+    @repaired = []
+    result
+  end
+
+private
+  
+  def generate_mappings_hash
+    @map = {}
+    ar = AR.split('_*_')
+    
+    BROKEN_AR.split('_*_').each_with_index do |c, i| 
+      @map[ c.unpack('C*') ] = ar[i].unpack('C*')
+    end
+  end
+  
+  def sequence_found_in_map
+    (1..4).each do |i|
+      broken_seq = @broken[0..i]
+      if @map.has_key?(broken_seq) 
+        @repaired += @map[broken_seq]
+        @broken.slice!(0, i + 1) 
+        return true
+      end
+    end
+    false
+  end
+
+  def byte_not_broken
+    if ( 
+        @broken.first != 195 && # The byte is not the beginning of a broken sequence
+        @broken.first <  256    # One byte char
+       ) || (
+        @broken.first == 195 &&          # The byte is the beginning of a sequence ...
+        !(152..155).include?(@broken[1]) # ... but the next one is not, so it's not a sequence! 
+       )
+      # Add the byte to the repaired sequence and remove it from the broken one
+      @repaired << @broken.shift 
+    end
+  end
+
+  def handle_unknown_byte
+    case
+    # Handle the case when the last 2 bytes are the beginning of
+    # a broken sequence but it's not found in the mappings hash.
+    # The best guess is that they're 2 one-byte chars.
+    # 
+    # Handles: [195, (152 | 153 | 154 | 155)]
+    when @broken.length == 2
+      @repaired << @broken.first << @broken[1]
+      @broken.slice!(0, 2)
+
+    # If we are here, then we have no idea what is this byte!
+    # We will just use the replace_char to replace the unknown byte_not_broken
+    # in the repaired sequence
+    else
+      @repaired += @replace_char.bytes.to_a
+    end
+  end
+end

data/lib/mosaheh/version.rb

@@ -0,0 +1,4 @@
+# encoding: UTF-8
+module Mosaheh
+  VERSION = "0.0.1"
+end

data/lib/mosaheh.rb

@@ -0,0 +1,6 @@
+# encoding: UTF-8
+require "mosaheh/version"
+
+module Mosaheh
+  autoload :Encoder, 'mosaheh/encoder'
+end

data/mosaheh.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "mosaheh/version"
+
+Gem::Specification.new do |s|
+  s.name        = "mosaheh"
+  s.version     = Mosaheh::VERSION
+  s.authors     = ["Maher Sallam"]
+  s.email       = ["maher@sallam.me"]
+  s.homepage    = ""
+  s.summary     = %q{An Arabic text re-encoder from latin1 (cp1252) to Arabic (UTF-8).}
+  s.description = %q{Mosaheh repairs UTF-8 Arabic (U+0600 - U+06FF) text which has been mistakenly saved as single-byte latin1 encoding (cp1252). The biggest usecase for it is to repair Arabic data stored in MySql databases with the wrong encoding.}
+
+  s.rubyforge_project = "mosaheh"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_development_dependency "rspec"
+end

data/spec/.rspec

@@ -0,0 +1 @@
+--color

data/spec/mosaheh/encoder_spec.rb

@@ -0,0 +1,17 @@
+# encoding: UTF-8
+require File.expand_path('../../spec_helper', __FILE__)
+
+describe Mosaheh::Encoder do
+  let(:encoder) { Mosaheh::Encoder.new }
+
+  describe '#repair' do
+    it 'should repair the whole Arabic unicode codeblock' do
+      encoder.repair(Mosaheh::Encoder::BROKEN_AR).unpack('C*').should eq Mosaheh::Encoder::AR.unpack('C*')
+    end
+
+    it 'should not change ASCII chars' do
+      text = [*0..127].map(&:chr).join
+      encoder.repair(text).should == text
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+# encoding: UTF-8
+
+require 'rubygems'
+require 'rspec'
+
+path = File.expand_path(File.dirname(__FILE__) + "/../lib/")
+$LOAD_PATH.unshift(path) unless $LOAD_PATH.include?(path)
+
+require 'mosaheh' 

metadata

@@ -0,0 +1,70 @@
+--- !ruby/object:Gem::Specification
+name: mosaheh
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Maher Sallam
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-10-31 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &21735460 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *21735460
+description: Mosaheh repairs UTF-8 Arabic (U+0600 - U+06FF) text which has been mistakenly
+  saved as single-byte latin1 encoding (cp1252). The biggest usecase for it is to
+  repair Arabic data stored in MySql databases with the wrong encoding.
+email:
+- maher@sallam.me
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- lib/mosaheh.rb
+- lib/mosaheh/encoder.rb
+- lib/mosaheh/version.rb
+- mosaheh.gemspec
+- spec/.rspec
+- spec/mosaheh/encoder_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: mosaheh
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: An Arabic text re-encoder from latin1 (cp1252) to Arabic (UTF-8).
+test_files: []
+has_rdoc: