sconv 0.1.0

data/README

@@ -0,0 +1,31 @@
+== Sconv version 0.1.0
+
+=== Overview
+The Module sconv provides a convenience layer for character
+encoding conversion. It is included by class String. 
+
+It automatically selects between kconv (for Japanese encodings)
+and iconv (for other encodings world-wide). It provides
+conversion methods directly on Strings, including destructive
+conversions. It provides methods with names of the form
+String#inputEncoding_to_outputEncoding automatically
+for any encoding supported by iconv or kconv.
+
+For further information, please see:
+"A Study Concerning Multilingual Processing in Ruby",
+Takuya SHIMADA, Kazunari ITO, and Martin J. Du"rst,
+Proceedings of the 69th Information Processing Society
+of Japan National Convention, March 2007 (in Japanese)
+
+=== Future Work
+- Write tests and run them
+- Provide support for error messages
+- Provide support for other CES conversions
+- Provide automatic update for list of
+  supported character encodings in ces.rb
+  (depends on locally installed iconv library)
+
+=== Copyright
+Copyright (c) 2007 Takuya Shimada, Martin J. Du"rst
+Licensed under the same terms as Ruby. Absolutely no warranty.
+(see http://www.ruby-lang.org/en/LICENSE.txt)

data/lib/ceslist

@@ -0,0 +1,357 @@
+ANSI_X3.4-1968
+ANSI_X3.4-1986
+ASCII
+CP367
+IBM367
+ISO-IR-6
+ISO646-US
+ISO_646.IRV:1991
+US
+US-ASCII
+CSASCII
+UTF-8
+ISO-10646-UCS-2
+UCS-2
+CSUNICODE
+UCS-2BE
+UNICODE-1-1
+UNICODEBIG
+CSUNICODE11
+UCS-2LE
+UNICODELITTLE
+ISO-10646-UCS-4
+UCS-4
+CSUCS4
+UCS-4BE
+UCS-4LE
+UTF-16
+UTF-16BE
+UTF-16LE
+UTF-32
+UTF-32BE
+UTF-32LE
+UNICODE-1-1-UTF-7
+UTF-7
+CSUNICODE11UTF7
+UCS-2-INTERNAL
+UCS-2-SWAPPED
+UCS-4-INTERNAL
+UCS-4-SWAPPED
+C99
+JAVA
+CP819
+IBM819
+ISO-8859-1
+ISO-IR-100
+ISO_8859-1
+ISO_8859-1:1987
+L1
+LATIN1
+CSISOLATIN1
+ISO-8859-2
+ISO-IR-101
+ISO_8859-2
+ISO_8859-2:1987
+L2
+LATIN2
+CSISOLATIN2
+ISO-8859-3
+ISO-IR-109
+ISO_8859-3
+ISO_8859-3:1988
+L3
+LATIN3
+CSISOLATIN3
+ISO-8859-4
+ISO-IR-110
+ISO_8859-4
+ISO_8859-4:1988
+L4
+LATIN4
+CSISOLATIN4
+CYRILLIC
+ISO-8859-5
+ISO-IR-144
+ISO_8859-5
+ISO_8859-5:1988
+CSISOLATINCYRILLIC
+ARABIC
+ASMO-708
+ECMA-114
+ISO-8859-6
+ISO-IR-127
+ISO_8859-6
+ISO_8859-6:1987
+CSISOLATINARABIC
+ECMA-118
+ELOT_928
+GREEK
+GREEK8
+ISO-8859-7
+ISO-IR-126
+ISO_8859-7
+ISO_8859-7:1987
+CSISOLATINGREEK
+HEBREW
+ISO-8859-8
+ISO-IR-138
+ISO_8859-8
+ISO_8859-8:1988
+CSISOLATINHEBREW
+ISO-8859-9
+ISO-IR-148
+ISO_8859-9
+ISO_8859-9:1989
+L5
+LATIN5
+CSISOLATIN5
+ISO-8859-10
+ISO-IR-157
+ISO_8859-10
+ISO_8859-10:1992
+L6
+LATIN6
+CSISOLATIN6
+ISO-8859-13
+ISO-IR-179
+ISO_8859-13
+L7
+LATIN7
+ISO-8859-14
+ISO-CELTIC
+ISO-IR-199
+ISO_8859-14
+ISO_8859-14:1998
+L8
+LATIN8
+ISO-8859-15
+ISO-IR-203
+ISO_8859-15
+ISO_8859-15:1998
+ISO-8859-16
+ISO-IR-226
+ISO_8859-16
+ISO_8859-16:2000
+KOI8-R
+CSKOI8R
+KOI8-U
+KOI8-RU
+CP1250
+MS-EE
+WINDOWS-1250
+CP1251
+MS-CYRL
+WINDOWS-1251
+CP1252
+MS-ANSI
+WINDOWS-1252
+CP1253
+MS-GREEK
+WINDOWS-1253
+CP1254
+MS-TURK
+WINDOWS-1254
+CP1255
+MS-HEBR
+WINDOWS-1255
+CP1256
+MS-ARAB
+WINDOWS-1256
+CP1257
+WINBALTRIM
+WINDOWS-1257
+CP1258
+WINDOWS-1258
+850
+CP850
+IBM850
+CSPC850MULTILINGUAL
+862
+CP862
+IBM862
+CSPC862LATINHEBREW
+866
+CP866
+IBM866
+CSIBM866
+MAC
+MACINTOSH
+MACROMAN
+CSMACINTOSH
+MACCENTRALEUROPE
+MACICELAND
+MACCROATIAN
+MACROMANIA
+MACCYRILLIC
+MACUKRAINE
+MACGREEK
+MACTURKISH
+MACHEBREW
+MACARABIC
+MACTHAI
+HP-ROMAN8
+R8
+ROMAN8
+CSHPROMAN8
+NEXTSTEP
+ARMSCII-8
+GEORGIAN-ACADEMY
+GEORGIAN-PS
+KOI8-T
+MULELAO-1
+CP1133
+IBM-CP1133
+ISO-IR-166
+TIS-620
+TIS620
+TIS620-0
+TIS620.2529-1
+TIS620.2533-0
+TIS620.2533-1
+CP874
+WINDOWS-874
+VISCII
+VISCII1.1-1
+CSVISCII
+TCVN
+TCVN-5712
+TCVN5712-1
+TCVN5712-1:1993
+ISO-IR-14
+ISO646-JP
+JIS_C6220-1969-RO
+JP
+CSISO14JISC6220RO
+JISX0201-1976
+JIS_X0201
+X0201
+CSHALFWIDTHKATAKANA
+ISO-IR-87
+JIS0208
+JIS_C6226-1983
+JIS_X0208
+JIS_X0208-1983
+JIS_X0208-1990
+X0208
+CSISO87JISX0208
+ISO-IR-159
+JIS_X0212
+JIS_X0212-1990
+JIS_X0212.1990-0
+X0212
+CSISO159JISX02121990
+CN
+GB_1988-80
+ISO-IR-57
+ISO646-CN
+CSISO57GB1988
+CHINESE
+GB_2312-80
+ISO-IR-58
+CSISO58GB231280
+CN-GB-ISOIR165
+ISO-IR-165
+ISO-IR-149
+KOREAN
+KSC_5601
+KS_C_5601-1987
+KS_C_5601-1989
+CSKSC56011987
+EUC-JP
+EUCJP
+EXTENDED_UNIX_CODE_PACKED_FORMAT_FOR_JAPANESE
+CSEUCPKDFMTJAPANESE
+MS_KANJI
+SHIFT-JIS
+SHIFT_JIS
+SJIS
+CSSHIFTJIS
+CP932
+ISO-2022-JP
+CSISO2022JP
+ISO-2022-JP-1
+ISO-2022-JP-2
+CSISO2022JP2
+CN-GB
+EUC-CN
+EUCCN
+GB2312
+CSGB2312
+CP936
+GBK
+GB18030
+ISO-2022-CN
+CSISO2022CN
+ISO-2022-CN-EXT
+HZ
+HZ-GB-2312
+EUC-TW
+EUCTW
+CSEUCTW
+BIG-5
+BIG-FIVE
+BIG5
+BIGFIVE
+CN-BIG5
+CSBIG5
+CP950
+BIG5-HKSCS
+BIG5HKSCS
+EUC-KR
+EUCKR
+CSEUCKR
+CP949
+UHC
+CP1361
+JOHAB
+ISO-2022-KR
+CSISO2022KR
+437
+CP437
+IBM437
+CSPC8CODEPAGE437
+CP737
+CP775
+IBM775
+CSPC775BALTIC
+852
+CP852
+IBM852
+CSPCP852
+CP853
+855
+CP855
+IBM855
+CSIBM855
+857
+CP857
+IBM857
+CSIBM857
+CP858
+860
+CP860
+IBM860
+CSIBM860
+861
+CP-IS
+CP861
+IBM861
+CSIBM861
+863
+CP863
+IBM863
+CSIBM863
+CP864
+IBM864
+CSIBM864
+865
+CP865
+IBM865
+CSIBM865
+869
+CP-GR
+CP869
+IBM869
+CSIBM869
+CP1125
+JIS

data/lib/sconv.rb

@@ -0,0 +1,161 @@
+require 'iconv'
+require 'kconv'
+
+# :include: ../README
+module Sconv 
+  # Convert CES (Character Encoding Scheme) of a string
+  # from "from" code to "to" code
+  # (selects kconv for Japanese encodings, iconv for others)
+  def sconv(from, to)
+    if (from =~ /iso-2022-jp|jis|shift_jis|sjis|euc-jp|utf-8|utf-16/i)&&(to =~ /iso-2022-jp|jis|shift_jis|sjis|euc-jp|utf-8|utf-16/i)
+      
+      in_code = Kconv::JIS   if from =~ /^i|^j/i
+      in_code = Kconv::SJIS  if from =~ /^s/i
+      in_code = Kconv::EUC   if from =~ /^e/i
+      in_code = Kconv::UTF8  if from =~ /8$/ 
+      in_code = Kconv::UTF16 if from =~ /6$/
+      
+      out_code = Kconv::JIS   if to =~ /^i|^j/i
+      out_code = Kconv::SJIS  if to =~ /^s/i
+      out_code = Kconv::EUC   if to =~ /^e/i
+      out_code = Kconv::UTF8  if to =~ /8$/ 
+      out_code = Kconv::UTF16 if to =~ /6$/
+    
+      kconv(out_code, in_code)    
+    
+    elsif from =~ /iso-2022-jp|jis|shift_jis|sjis|euc-jp|utf-8|utf-16/i
+      in_code = Kconv::JIS   if from =~ /^i|^j/i
+      in_code = Kconv::SJIS  if from =~ /^s/i
+      in_code = Kconv::EUC   if from =~ /^e/i
+      in_code = Kconv::UTF8  if from =~ /8$/ 
+      in_code = Kconv::UTF16 if from =~ /6$/
+      
+      out_code = to
+      
+      str_utf8 = self.kconv(Kconv::UTF8, in_code)
+      Iconv.conv(out_code, 'UTF-8', str_utf8)
+      
+    elsif to =~ /iso-2022-jp|jis|shift_jis|sjis|euc-jp|utf-8|utf-16/i
+      in_code = from
+      
+      out_code = Kconv::JIS   if to =~ /^i|^j/i
+      out_code = Kconv::SJIS  if to =~ /^s/i
+      out_code = Kconv::EUC   if to =~ /^e/i
+      out_code = Kconv::UTF8  if to =~ /8$/ 
+      out_code = Kconv::UTF16 if to =~ /6$/
+      
+      Iconv.conv('UTF-8', in_code, self).kconv(out_code, Kconv::UTF8)
+    
+    else
+      in_code  = from
+      out_code = to
+      
+      Iconv.conv(out_code, in_code, self)
+    end  
+  end
+  
+  # Return name of CES
+  # without is not removed hyphens and underlines.
+  def normalize
+    ces = Hash.new
+    open('ceslist','r').read.scan(/\S+\n/).each do |i|
+      ces[i.chomp.delete('\-,_')] = i.chomp
+    end
+    ces.fetch(self)
+  end
+  
+  # Check whether the string is EUC-JP or not
+  # (more Ruby-like method name)
+  def EUC?
+    if iseuc == nil
+      false
+    else
+      true
+    end
+  end
+  
+  # Check whether the string is Shift_JIS or not
+  # (more Ruby-like method name)
+  def SJIS?
+    if issjis == nil
+      false
+    else
+      true
+    end
+  end
+  
+  # Check whether the string is UTF-8 or not
+  # (more Ruby-like method name)
+  def UTF8?
+    if isutf8 == nil
+      false
+    else
+      true
+    end
+  end
+  
+  # Guess the string's CES; return the result as a string
+  def guess_ces
+    case Kconv.guess(self)
+    when 1
+      'ISO-2022-JP'
+    when 2
+      'EUC-JP'
+    when 3
+      'Shift_JIS'
+    when 4
+      'BINARY'
+    when 5
+      'ASCII'
+    when 6
+      'UTF-8'
+    when 8
+      'UTF-16'
+    end
+  end
+end
+
+# === Helper Functions for String Class
+class String
+  include Sconv
+
+  alias sconf_old_method_missing method_missing
+
+  # Provides methods of the form String#inputEncoding_to_outputEnoding
+  # and String#inputEncoding_to_outputEncoding!. Examples:
+  # String#UTF8_to_ISO88591
+  def method_missing (method_name)
+    okay = false
+    if method_name.to_s =~ /^(.+)_to_([^!]+)(!)?$/
+      from_code, to_code, exclamation = $1.normalize, $2.normalize, $3
+      destructive = exclamation == '!'
+      okay = true
+      # test whether we can convert these encodings
+      begin
+        ''.sconv(from_code, to_code)
+      end
+      if okay
+        if destructive
+          self.class.send :define_method, method_name.to_sym do
+            replace self.sconv(from_code, to_code)
+          end
+        else
+          self.class.send :define_method, method_name.to_sym do
+            sconv(from_code, to_code)
+          end
+        end
+        send method_name
+      else
+        sconf_old_method_missing
+      end
+    end
+    
+    if method_name.to_s =~ /^to_(\w+)$/
+      method_called = 'to' + $1.downcase
+      self.class.send :define_method, method_name.to_sym do
+        send method_called.to_sym
+      end
+      send method_name
+    end
+  end
+end

metadata

@@ -0,0 +1,48 @@
+--- !ruby/object:Gem::Specification 
+rubygems_version: 0.9.2
+specification_version: 1
+name: sconv
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+date: 2007-05-28 00:00:00 +09:00
+summary: A convenience layer for character encoding conversion, providing one-stop shopping for Kconv (for Japanese encodings) and Iconv (for other encodings).
+require_paths: 
+- lib
+email: duerst@it.aoyama.ac.jp
+homepage: 
+rubyforge_project: 
+description: 
+autorequire: sconv
+default_executable: 
+bindir: bin
+has_rdoc: true
+required_ruby_version: !ruby/object:Gem::Version::Requirement 
+  requirements: 
+  - - ">"
+    - !ruby/object:Gem::Version 
+      version: 0.0.0
+  version: 
+platform: ruby
+signing_key: 
+cert_chain: 
+post_install_message: 
+authors: 
+- Takuya Shimada, Martin J. Du"rst
+files: 
+- lib/ceslist
+- lib/sconv.rb
+- README
+test_files: []
+
+rdoc_options: []
+
+extra_rdoc_files: 
+- README
+executables: []
+
+extensions: []
+
+requirements: []
+
+dependencies: []
+