japanese_names 0.0.1

data/lib/japanese_names/enamdict.rb

@@ -0,0 +1,104 @@
+#!/bin/env ruby
+# encoding: utf-8
+
+module JapaneseNames
+
+  # Query interface for the ENAMDICT file (http://www.csse.monash.edu.au/~jwb/enamdict_doc.html)
+  module Enamdict
+
+    # s - surname (138,500)
+    # p - place-name (99,500)
+    # u - person name, either given or surname, as-yet unclassified (139,000)
+    # g - given name, as-yet not classified by sex (64,600)
+    # f - female given name (106,300)
+    # m - male given name (14,500)
+    NAME_FAM = %w(s p u)
+    NAME_GIV = %w(u g f m)
+    NAME_ANY = NAME_FAM | NAME_GIV
+
+    class << self
+
+      # Public: Matches kanji and/or kana regex strings in the dictionary.
+      #
+      # opts - The Hash options used to match the dictionary (default: {}):
+      #        kanji: Regex to match kanji name (optional)
+      #        kana:  Regex to match kana name (optional)
+      #        flags: Flag or Array of flags to filter the match (optional)
+      #
+      # Returns the dict entries as an Array of Arrays [[kanji, kana, flags], ...]
+      def match(opts={})
+        return [] unless opts[:kanji] || opts[:kana]
+
+        kanji = name_regex opts.delete(:kanji)
+        kana  = name_regex opts.delete(:kana)
+        flags = flags_regex opts.delete(:flags)
+        regex = /^#{kanji}\|#{kana}\|#{flags}$/
+
+        search{|line| line[regex]}
+      end
+
+      # Public: Selects entries in the enamdict based on a block which should
+      # evaluate true or false (typically a regex).
+      #
+      # Returns the dict entries as an Array of Arrays [[kanji, kana, flags], ...]
+      def search(&block)
+        sel = []
+        each_line do |line|
+          if block.call(line)
+            sel << unpack_line(line)
+          end
+        end
+        sel
+      end
+
+      protected
+
+      # Internal: Returns the filepath to the enamdict.min file.
+      def filepath
+        File.join(File.dirname(__FILE__), '../../bin/enamdict.min')
+      end
+
+      # Internal: The memoized dictionary instance.
+      def dict
+        return @dict if @dict
+        @dict = []
+        File.open(self.filepath, 'r:utf-8') do |f|
+          while(line = f.gets) != nil
+            @dict << line[0..-2] # omit trailing newline char
+          end
+        end
+        @dict.freeze
+      end
+
+      # Internal: Calls the given block for each line in the dict.
+      def each_line(&block)
+        dict.each{|line| block.call(line) }
+      end
+
+      # Internal: Formats a line as a 3-tuple Array [kanji, kana, flags]
+      def unpack_line(line)
+        line.split('|')
+      end
+
+      # Internal: Builds regex criteria for name.
+      def name_regex(name)
+        case name
+          when String then name
+          when Array  then "(?:#{name.join('|')})"
+          else '.+?'
+        end
+      end
+
+      # Internal: Builds regex criteria for flags.
+      def flags_regex(flags)
+        if !flags || flags == NAME_ANY
+          '.+?'
+        elsif flags.is_a?(Array)
+          ".*?[#{flags.join}].*?"
+        else
+          flags
+        end
+      end
+    end
+  end
+end

data/lib/japanese_names/parser.rb

@@ -0,0 +1,76 @@
+#!/bin/env ruby
+# encoding: utf-8
+
+module JapaneseNames
+
+  # Provides methods for parsing Japanese name strings.
+  class Parser
+
+    # Given a kanji and kana representation of a name splits into to family/given names.
+    #
+    # The choice to prioritize family name is arbitrary. Further analysis is needed
+    # for whether given or family name should be prioritized.
+    #
+    # Returns Array [[kanji_fam, kanji_giv], [kana_fam, kana_giv]] if there was a match.
+    # Returns nil if there was no match.
+    def split(kanji, kana)
+      split_fam(kanji, kana) || split_giv(kanji, kana)
+    end
+
+    def split_giv(kanji, kana)
+      dict = Enamdict.match(kanji: window_right(kanji))
+      dict.sort!{|x,y| y[0].size <=> x[0].size}
+      kana_match = nil
+      if match = dict.detect{|m| kana_match = kana[/#{hk m[1]}$/]}
+        return [[mask_right(kanji, match[0]), match[0]],[mask_right(kana, kana_match), kana_match]]
+      end
+    end
+
+    def split_fam(kanji, kana)
+      dict = Enamdict.match(kanji: window_left(kanji))
+      dict.sort!{|x,y| y[0].size <=> x[0].size}
+      kana_match = nil
+      if match = dict.detect{|m| kana_match = kana[/^#{hk m[1]}/]}
+        return [[match[0], mask_left(kanji, match[0])],[kana_match, mask_left(kana, kana_match)]]
+      end
+    end
+
+    # TODO: add option to strip honorific '様'
+    # TODO: add option to infer sex (0 = unknown, 1 = male, 2 = female as per ISO/IEC 5218)
+
+    protected
+
+    # Returns a regex string which matches both hiragana and katakana variations of a String.
+    def hk(str)
+      "(?:#{Moji.kata_to_hira(str)}|#{Moji.hira_to_kata(str)})"
+    end
+
+    # Masks a String from the left side and returns the remaining (right) portion of the String.
+    #
+    # Example: mask_left("abcde", "ab") #=> "cde"
+    def mask_left(str, mask)
+      str.gsub(/^#{mask}/, '')
+    end
+
+    # Masks a String from the right side and returns the remaining (left) portion of the String.
+    #
+    # Example:  mask_right("abcde", "de") #=> "abc"
+    def mask_right(str, mask)
+      str.gsub(/#{mask}$/, '')
+    end
+
+    # Given a String, returns an array of progressively smaller substrings anchored on the left side.
+    #
+    # Example: window_left("abcde") #=>  ["abcd", "abc", "ab", "a"]
+    def window_left(str)
+      (0..str.size-2).to_a.reverse.map{|i| str[0..i]}
+    end
+
+    # Given a String, returns an array of progressively smaller substrings anchored on the right side.
+    #
+    # Example: window_right("abcde") #=>  ["bcde", "cde", "de", "e"]
+    def window_right(str)
+      (1..str.size-1).map{|i| str[i..-1]}
+    end
+  end
+end

data/lib/japanese_names/version.rb

@@ -0,0 +1,6 @@
+#!/bin/env ruby
+# encoding: utf-8
+
+module JapaneseNames
+  VERSION = '0.0.1'
+end

data/lib/japanese_names.rb

@@ -0,0 +1,7 @@
+$:.unshift File.dirname(__FILE__)
+
+require 'moji'
+
+require 'japanese_names/version'
+require 'japanese_names/enamdict'
+require 'japanese_names/parser'

data/spec/spec_helper.rb

@@ -0,0 +1,11 @@
+#!/bin/env ruby
+# encoding: utf-8
+
+$:.push File.expand_path('../../lib', __FILE__)
+
+require 'rubygems'
+require 'japanese_names'
+
+RSpec.configure do |config|
+  config.mock_with :rspec
+end

data/spec/unit/enamdict_spec.rb

@@ -0,0 +1,59 @@
+#!/bin/env ruby
+# encoding: utf-8
+
+require 'spec_helper'
+
+describe JapaneseNames::Enamdict do
+
+  subject { JapaneseNames::Enamdict }
+
+  describe '#search' do
+
+    it 'should select only lines which match criteria' do
+      result = subject.search{|line| line =~ /^.+?\|あわのはら\|.+?$/}
+      result.should eq [["粟野原", "あわのはら", "s"]]
+    end
+
+    it 'should select multiple lines' do
+      result = subject.search{|line| line =~ /^.+?\|はしの\|.+?$/}
+      result.should eq [["橋之", "はしの", "p"],
+                        ["橋埜", "はしの", "s"],
+                        ["橋野", "はしの", "s"],
+                        ["端野", "はしの", "s"],
+                        ["箸野", "はしの", "s"]]
+    end
+  end
+
+  describe '#lookup' do
+
+    it 'should match kanji only' do
+      result = subject.match(kanji: '外世子')
+      result.should eq [["外世子", "とよこ", "f"]]
+    end
+
+    it 'should match kana only' do
+      result = subject.match(kana: 'ならしま')
+      result.should eq [["樽島", "ならしま", "u"],
+                        ["奈良島", "ならしま", "s"],
+                        ["楢島", "ならしま", "s"],
+                        ["楢嶋", "ならしま", "s"]]
+    end
+
+    it 'should match both kanji and kana only' do
+      result = subject.match(kanji: '楢二郎', kana: 'ならじろう')
+      result.should eq [["楢二郎", "ならじろう", "m"]]
+    end
+
+    it 'should match flags as String' do
+      result = subject.match(kana: 'ならしま', flags: 's')
+      result.should eq [["奈良島", "ならしま", "s"],
+                        ["楢島", "ならしま", "s"],
+                        ["楢嶋", "ならしま", "s"]]
+    end
+
+    it 'should match flags as Array' do
+      result = subject.match(kana: 'ならしま', flags: ['u','g'])
+      result.should eq [["樽島", "ならしま", "u"]]
+    end
+  end
+end

data/spec/unit/parser_spec.rb

@@ -0,0 +1,42 @@
+#!/bin/env ruby
+# encoding: utf-8
+
+require 'spec_helper'
+
+describe JapaneseNames::Parser do
+
+  subject { JapaneseNames::Parser.new }
+
+  describe '#split' do
+
+    [['上原','望','ウエハラ', 'ノゾミ'],
+     ['樋口','知美','ヒグチ', 'ともみ'],
+     ['堺','雅美','さかい', 'マサミ'],
+     ['中村','幸子','ナカムラ', 'サチコ'],
+     ['秋保','郁子','アキホ', 'いくこ'],
+     ['光野','亜佐子','ミツノ', 'アサコ'],
+     ['熊澤','貴子','クマザワ', 'タカコ']].each do |kanji_fam, kanji_giv, kana_fam, kana_giv|
+      it "should parse #{kanji_fam+kanji_giv} #{kana_fam+kana_giv}" do
+        result = subject.split(kanji_fam+kanji_giv, kana_fam+kana_giv)
+        result.should eq [[kanji_fam, kanji_giv], [kana_fam, kana_giv]]
+      end
+
+      it "should parse #{kanji_fam+kanji_giv} #{kana_fam+kana_giv} by given name" do
+        result = subject.split_giv(kanji_fam+kanji_giv, kana_fam+kana_giv)
+        result.should eq [[kanji_fam, kanji_giv], [kana_fam, kana_giv]]
+      end
+
+      it "should parse #{kanji_fam+kanji_giv} #{kana_fam+kana_giv} by family name" do
+        result = subject.split_fam(kanji_fam+kanji_giv, kana_fam+kana_giv)
+        result.should eq [[kanji_fam, kanji_giv], [kana_fam, kana_giv]]
+      end
+    end
+
+    [['XXX','XXX','XXX', 'XXX']].each do |kanji_fam, kanji_giv, kana_fam, kana_giv|
+      it "should return nil for invalid name #{kanji_fam+kanji_giv} #{kana_fam+kana_giv}" do
+        result = subject.split(kanji_fam+kanji_giv, kana_fam+kana_giv)
+        result.should be_nil
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: japanese_names
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Johnny Shields
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: moji
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: gem-release
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Japanese name parser based on ENAMDICT
+email: johnny.shields@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/enamdict.min
+- lib/japanese_names.rb
+- lib/japanese_names/enamdict.rb
+- lib/japanese_names/parser.rb
+- lib/japanese_names/version.rb
+- spec/spec_helper.rb
+- spec/unit/enamdict_spec.rb
+- spec/unit/parser_spec.rb
+homepage: https://github.com/johnnyshields/japanese_names
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.1
+signing_key: 
+specification_version: 4
+summary: Tools for parsing japanese names
+test_files:
+- spec/spec_helper.rb
+- spec/unit/enamdict_spec.rb
+- spec/unit/parser_spec.rb
+has_rdoc: