mac_japanese 0.0.1

data/lib/mac_japanese/version.rb

@@ -0,0 +1,3 @@
+module MacJapanese
+  VERSION = "0.0.1"
+end

data/mac_japanese.gemspec

@@ -0,0 +1,20 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/mac_japanese/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["labocho"]
+  gem.email         = ["labocho@penguinlab.jp"]
+  gem.description   = %q{Convert MacJapanese string to UTF-8 and vice versa.}
+  gem.summary       = %q{Convert MacJapanese string to UTF-8 and vice versa.}
+  gem.homepage      = "https://github.com/labocho/mac_japanese"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "mac_japanese"
+  gem.require_paths = ["lib"]
+  gem.version       = MacJapanese::VERSION
+  gem.add_development_dependency "rspec", "~>2.11.0"
+  gem.add_development_dependency "guard-rspec", "~>0.7.0"
+  gem.add_development_dependency "ruby-debug19"
+end

data/spec/mac_japanese_spec.rb

@@ -0,0 +1,134 @@
+require "spec_helper"
+require "ruby-debug"
+
+describe MacJapanese do
+  describe ".to_utf8" do
+    [true, false].each do |use_pua|
+      context "user_pua: #{use_pua}" do
+        let(:options) { {} }
+        subject { MacJapanese.to_utf8(@src, options.merge(use_pua: use_pua)) }
+        it "should convert us-ascii chars to utf8" do
+          @src = "foo\n".force_encoding("macjapan")
+          should == "foo\n"
+        end
+        it "should convert additional backslash to utf8" do
+          @src = "\x80".force_encoding("macjapan")
+          should == "\\"
+        end
+        it "should convert halfwidth katakana to utf8" do
+          @src = "\xA7".force_encoding("macjapan")
+          should == "\u{FF67}"
+        end
+        it "should convert hiragana to utf8" do
+          @src = "\x82\x9F".force_encoding("macjapan")
+          should == "\u{3041}"
+        end
+        it "should convert apple additions to utf8" do
+          @src = "\x85\x5E".force_encoding("macjapan")
+          should == "\u{2474}"
+        end
+      end
+    end
+    context "default" do
+      subject { MacJapanese.to_utf8(@src) }
+      it "should expand composed char with pua" do
+        @src = "\x85\xAB".force_encoding("macjapan")
+        should == "\u{F862}\u{0058}\u{0049}\u{0049}\u{0049}"
+      end
+    end
+    context "use_pua: true" do
+      subject { MacJapanese.to_utf8(@src, use_pua: true) }
+      it "should expand composed char with pua" do
+        @src = "\x85\xAB".force_encoding("macjapan")
+        should == "\u{F862}\u{0058}\u{0049}\u{0049}\u{0049}"
+      end
+    end
+    context "use_pua: false" do
+      subject { MacJapanese.to_utf8(@src, use_pua: false) }
+      it "should expand composed char without pua" do
+        @src = "\x85\xAB".force_encoding("macjapan")
+        should == "\u{0058}\u{0049}\u{0049}\u{0049}"
+      end
+    end
+    context "pass another encoding string to .to_utf8" do
+      it "should encode just like passing mac japanese string" do
+        @src = "\x82\x9F"
+        @src.encoding.should == Encoding::ASCII_8BIT
+        MacJapanese.to_utf8(@src).should == "\u{3041}"
+      end
+    end
+  end
+
+  describe ".to_mac_japanese" do
+    let(:options) { {} }
+    subject { MacJapanese.to_mac_japanese(@src, options) }
+    it "should convert us-ascii chars to mac_japanese" do
+      @src = "foo\n"
+      should == "foo\n".force_encoding("macjapan")
+    end
+    it "should convert additional backslash to mac_japanese" do
+      @src = "\\"
+      should == "\x80".force_encoding("macjapan")
+    end
+    it "should convert halfwidth katakana to mac_japanese" do
+      @src = "\u{FF67}"
+      should == "\xA7".force_encoding("macjapan")
+    end
+    it "should convert hiragana to mac_japanese" do
+      @src = "\u{3041}"
+      should == "\x82\x9F".force_encoding("macjapan")
+    end
+    it "should convert hiragana followed by composed characters to mac_japanese" do
+      @src = "\u{3041}\u{F862}\u{0058}\u{0049}\u{0049}\u{0049}"
+      should == "\x82\x9F\x85\xAB".force_encoding("macjapan")
+    end
+    it "should convert apple additions to mac_japanese" do
+      @src = "\u{2474}"
+      should == "\x85\x5E".force_encoding("macjapan")
+    end
+    it "should compose characters with pua" do
+      @src = "\u{F862}\u{0058}\u{0049}\u{0049}\u{0049}"
+      should == "\x85\xAB".force_encoding("macjapan")
+    end
+    it "should not compose characters without pua" do
+      @src = "\u{0058}\u{0049}\u{0049}\u{0049}"
+      should == "XIII".force_encoding("macjapan")
+    end
+    context "pass another encoding string to .to_mac_japanese" do
+      it "should encode to mac japanese string (via utf8)" do
+        @src = "\u{3041}".encode("euc-jp")
+        MacJapanese.to_mac_japanese(@src).should == "\x82\x9F".force_encoding("macjapan")
+      end
+    end
+  end
+
+  context "undef: :replace" do
+    it "should replace undefined mac japanese char" do
+      @src = "foo\xFC\xFCbar".force_encoding("macjapan")
+      MacJapanese.to_utf8(@src, undef: :replace).should == "foo\u{fffd}bar"
+    end
+    it "should replace undefined utf-8 char" do
+      @src = "foo\u{FA11}bar"
+      MacJapanese.to_mac_japanese(@src, undef: :replace).should == "foo?bar"
+    end
+    it "should replace with replace option" do
+      @src = "foo\xFC\xFCbar".force_encoding("macjapan")
+      MacJapanese.to_utf8(@src, undef: :replace, replace: "*").should == "foo*bar"
+    end
+  end
+
+  context "undef: (none)" do
+    it "should raise Encoding::UndefinedConversionError for undefined mac japanese char" do
+      @src = "foo\xFC\xFCbar".force_encoding("macjapan")
+      expect{
+        MacJapanese.to_utf8(@src)
+      }.to raise_error(Encoding::UndefinedConversionError)
+    end
+    it "should raise Encoding::UndefinedConversionError for undefined utf-8 japanese char" do
+      @src = "foo\u{FA11}bar"
+      expect{
+        MacJapanese.to_mac_japanese(@src)
+      }.to raise_error(Encoding::UndefinedConversionError)
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,20 @@
+$LOAD_PATH.unshift "#{File.dirname(__FILE__)}/../lib"
+require "mac_japanese"
+
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end

data/src/generate_conversion_tables.rb

@@ -0,0 +1,133 @@
+require "csv"
+require "open-uri"
+
+ROOT_DIR = File.expand_path "#{File.dirname(__FILE__)}/../"
+
+# convert string to literal like "\x81\x40"
+def hex_literal(string)
+  string.bytes.map{|b|
+    "\\x" + b.to_s(16).upcase.rjust(2, "0")
+  }.join
+end
+
+# convert string to literal like "\u{3041}"
+def unicode_literal(string)
+  string.codepoints.map{|c|
+    "\\u{" + c.to_s(16).upcase.rjust(4, "0") + "}"
+  }.join
+end
+
+def pua?(four_hex_with_0x)
+  case four_hex_with_0x
+  when "\u{F860}", "\u{F861}", "\u{F862}", "\u{F87A}", "\u{F87E}", "\u{F87F}"
+    true
+  else
+    false
+  end
+end
+
+# Make pairs of string literals [[macjapanese, utf8], ...]
+def make_pairs(use_pua = true)
+  pairs = []
+  # Control characters
+  pairs += (0x00..0x1f).map{|i|
+    c = [i].pack("C*")
+    [c, c]
+  }
+  open("#{ROOT_DIR}/src/JAPANESE.txt") do |f|
+    f.lines.each do |line|
+      next if line =~ /^#/ # ignore comment
+      next unless line =~ /^(0x.+)\t(0x.+)\t/ # capture
+      macjp_hex, unicode_hex = $~.captures
+
+      # macjp = macjp_hex[2..-1].chars.each_slice(2).map{|hex| "\\x" + hex.join}.join
+      macjp = macjp_hex[2..-1].chars.each_slice(2).map{|h| h.join.to_i(16)}.pack("C*")
+
+      unicode = unicode_hex.split("+").map{|hex|
+        c = hex.to_i(16).chr("utf-8")
+        next "" if !use_pua && pua?(c) # skip pua
+        c
+      }.join
+
+      pairs.push [macjp, unicode]
+    end
+  end
+  pairs
+end
+
+unless File.exist?("#{ROOT_DIR}/src/JAPANESE.txt")
+  open("#{ROOT_DIR}/src/JAPANESE.txt", "w") do |f|
+    open("http://www.unicode.org/Public/MAPPINGS/VENDORS/APPLE/JAPANESE.TXT") do |g|
+      f.print g.read
+    end
+  end
+end
+
+# Make MacJapanese to UTF-8 table (with PUA)
+path = "#{ROOT_DIR}/lib/mac_japanese/mac_japanese_to_utf8_with_pua.rb"
+puts path
+literal_pairs = make_pairs.map{|m, u| [hex_literal(m), unicode_literal(u)]}
+open(path, "w") do |f|
+  f.puts <<-EOS
+# This file was automatically generated by `rake tables`.
+# Cannot modify directly.
+module MacJapanese
+  MAC_JAPANESE_TO_UTF8_WITH_PUA = Hash[
+    [
+#{literal_pairs.map{|m, u| %{      ["#{m}", "#{u}"]}}.join(",\n")}
+    ].each{|m, u| m.force_encoding(Encoding::MacJapanese)}
+  ]
+end
+EOS
+end
+
+# Make MacJapanese to UTF-8 table (without PUA)
+path = "#{ROOT_DIR}/lib/mac_japanese/mac_japanese_to_utf8_without_pua.rb"
+puts path
+literal_pairs = make_pairs(false).map{|m, u| [hex_literal(m), unicode_literal(u)]}
+open(path, "w") do |f|
+  f.puts <<-EOS
+# This file was automatically generated by `rake tables`.
+# Cannot modify directly.
+module MacJapanese
+  MAC_JAPANESE_TO_UTF8_WITHOUT_PUA = Hash[
+    [
+#{literal_pairs.map{|m, u| %{      ["#{m}", "#{u}"]}}.join(",\n")}
+    ].each{|m, u| m.force_encoding(Encoding::MacJapanese)}
+  ]
+end
+EOS
+end
+
+# Make UTF-8 to MacJapanese table
+path = "#{ROOT_DIR}/lib/mac_japanese/utf8_to_mac_japanese.rb"
+puts path
+literal_pairs = make_pairs.map{|m, u| [hex_literal(m), unicode_literal(u)]}
+open(path, "w") do |f|
+  f.puts <<-EOS
+# This file was automatically generated by `rake tables`.
+# Cannot modify directly.
+module MacJapanese
+  UTF8_TO_MAC_JAPANESE = Hash[
+    [
+#{literal_pairs.map{|m, u| %{      ["#{u}", "#{m}"]}}.join(",\n")}
+    ].each{|u, m| m.force_encoding(Encoding::MacJapanese)}
+  ]
+end
+EOS
+end
+
+# Make UTF-8 single character or decomposed characters regexp
+path = "#{ROOT_DIR}/lib/mac_japanese/decomposed_or_normal_character_regexp.rb"
+puts path
+decomposed_or_single_character_regexp =
+  "/(" + make_pairs.map{|*, u| u}.select{|u| u.size > 1}.map{|u| unicode_literal(u)}.join("|") + "|.)/m"
+open(path,  "w") do |f|
+  f.puts <<-EOS
+# This file was automatically generated by `rake tables`.
+# Cannot modify directly.
+module MacJapanese
+  DECOMPOSED_OR_NORMAL_CHARACTER_REGEXP = #{decomposed_or_single_character_regexp}
+end
+EOS
+end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: mac_japanese
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- labocho
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-09-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.11.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.11.0
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+- !ruby/object:Gem::Dependency
+  name: ruby-debug19
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Convert MacJapanese string to UTF-8 and vice versa.
+email:
+- labocho@penguinlab.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- Guardfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/mac_japanese.rb
+- lib/mac_japanese/decomposed_or_normal_character_regexp.rb
+- lib/mac_japanese/mac_japanese_to_utf8_with_pua.rb
+- lib/mac_japanese/mac_japanese_to_utf8_without_pua.rb
+- lib/mac_japanese/utf8_to_mac_japanese.rb
+- lib/mac_japanese/version.rb
+- mac_japanese.gemspec
+- spec/mac_japanese_spec.rb
+- spec/spec_helper.rb
+- src/generate_conversion_tables.rb
+homepage: https://github.com/labocho/mac_japanese
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Convert MacJapanese string to UTF-8 and vice versa.
+test_files:
+- spec/mac_japanese_spec.rb
+- spec/spec_helper.rb