eiwa 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e88ea7a5b39e145c4e5115b1e1f0505fcb1f9dd63f1f68c54de89903dabaef67
+  data.tar.gz: 2da703a8cd7daa9c4ade8a5d9090f814bfc8adcdd62fd3dcb381b3aba2bc089c
+SHA512:
+  metadata.gz: 561cfcb808e8cfa05f225c92edfa861eea3b3173141ba133ff00c988b0071c9f162ed0ccd321532b7de36bf5797f6b0b00c408cd919d8657e844612c2c5450cf
+  data.tar.gz: a85484fe894a30e3c6f094332a1ae143cb0af95aec164d470aaf87c9614e86dd9e61fd6b8ed986767afd863afb07f830eaf9160deeca6e76b5edc03240c20232

data/.github/workflows/ruby.yml

@@ -0,0 +1,20 @@
+name: Ruby
+
+on: [push]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Set up Ruby 2.6
+      uses: actions/setup-ruby@v1
+      with:
+        ruby-version: 2.6.x
+    - name: Build and test with Rake
+      run: |
+        gem install bundler
+        bundle install --jobs 4 --retry 3
+        bundle exec rake

data/.gitignore

@@ -0,0 +1,8 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in eiwa.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,53 @@
+PATH
+  remote: .
+  specs:
+    eiwa (0.0.1)
+      nokogiri
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.0)
+    coderay (1.1.2)
+    jaro_winkler (1.5.3)
+    method_source (0.9.2)
+    mini_portile2 (2.4.0)
+    minitest (5.11.3)
+    nokogiri (1.10.4)
+      mini_portile2 (~> 2.4.0)
+    parallel (1.17.0)
+    parser (2.6.4.1)
+      ast (~> 2.4.0)
+    pry (0.12.2)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rainbow (3.0.0)
+    rake (10.5.0)
+    rubocop (0.72.0)
+      jaro_winkler (~> 1.5.1)
+      parallel (~> 1.10)
+      parser (>= 2.6)
+      rainbow (>= 2.2.2, < 4.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 1.7)
+    rubocop-performance (1.4.1)
+      rubocop (>= 0.71.0)
+    ruby-progressbar (1.10.1)
+    standard (0.1.4)
+      rubocop (~> 0.72.0)
+      rubocop-performance (~> 1.4.0)
+    unicode-display_width (1.6.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.17)
+  eiwa!
+  minitest (~> 5.0)
+  pry
+  rake (~> 10.0)
+  standard
+
+BUNDLED WITH
+   1.17.3

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2019 Justin Searls
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,77 @@
+# eiwa / 英和
+
+Parses the Japanese-English version of JMDict, a daily export of the WWWJDIC
+online Japanese dictionary.
+
+## Usage
+
+### Install
+
+Install the gem:
+
+```
+gem install eiwa
+```
+
+Or add it to your `Gemfile`:
+
+```ruby
+gem 'eiwa'
+```
+
+### Download a supported dictionary
+
+Get your hands on a supported dictionary. Right now eiwa only parses
+[JMDict](http://www.edrdg.org/jmdict/j_jmdict.html), which can be fetched from
+the [Monash ftp site](http://ftp.monash.edu/pub/nihongo/00INDEX.html) or with a
+script like this, for the Japanese-English export:
+
+```bash
+curl http://ftp.monash.edu/pub/nihongo/JMdict_e -o jmdict.xml
+```
+
+This file is updated daily, and is essentially an export of all vocabulary on
+the [WWWJDIC application](http://nihongo.monash.edu/cgi-bin/wwwjdic?1C)
+
+### Parse the dictionary
+
+The eiwa gem implements an evented [SAX
+parser](https://en.wikipedia.org/wiki/Simple_API_for_XML) via nokogiri to
+efficiently work through the very large XML file, as loading a full DOM into
+memory is very resource-intensive. In consideration of this, eiwa's parsing
+method provides two modes, one that will return every dictionary entry in an
+array and one that will invoke a provided block with each entry, but which won't
+retain a reference to the entries, allowing Ruby to garbage collect them as it
+goes.
+
+Parsing the dictionary is CPU intensive, and takes about 13 seconds on my 2019
+13" MacBook Pro.
+
+#### Passing a block
+
+If you just want to do some processing on each entry, it probably makes sense to
+invoke the library by passing a block
+
+```ruby
+Eiwa.parse_file("path/to/some.xml", type: :jmdict_e) do |entry|
+  # Do something with that entry
+end
+```
+
+This approach can parse the entire JMDICT-E dictionary in a 15MB Ruby 2.6
+process.
+
+#### Return the results in an array
+
+If you're just going to add all the entries to an array or otherwise retain them
+in memory, you can call the same method without a block, and it will return all
+the entries in an array.
+
+```ruby
+entries = Eiwa.parse_file("path/to/some.xml", type: :jmdict_e)
+```
+
+Note that for the abridged Japanese-English dictionary, this will consume about
+500MB of RAM.
+
+

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/gem_tasks"
+require "rake/testtask"
+require "standard/rake"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/*.rb"]
+end
+
+task default: [:test, "standard:fix"]

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "eiwa"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/eiwa.gemspec

@@ -0,0 +1,28 @@
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "eiwa/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "eiwa"
+  spec.version = Eiwa::VERSION
+  spec.authors = ["Justin Searls"]
+  spec.email = ["searls@gmail.com"]
+
+  spec.summary = "Parses the JMDict Japanese-English dictionary"
+  spec.homepage = "https://github.com/searls/eiwa"
+  spec.license = "MIT"
+
+  spec.files = Dir.chdir(File.expand_path("..", __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|tmp)/}) }
+  end
+  spec.bindir = "exe"
+  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "nokogiri"
+  spec.add_development_dependency "bundler", "~> 1.17"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "standard"
+  spec.add_development_dependency "pry"
+end

data/lib/eiwa.rb

@@ -0,0 +1,15 @@
+require "eiwa/version"
+require "eiwa/parses_jmdict_file"
+
+module Eiwa
+  class Error < StandardError; end
+
+  def self.parse_file(filename, type: :jmdict_e, &each_entry_block)
+    case type
+    when :jmdict_e
+      ParsesJmdictFile.new.call(filename, each_entry_block)
+    else
+      raise Eiwa::Error.new("Unknown file type: #{type}")
+    end
+  end
+end

data/lib/eiwa/jmdict_doc.rb

@@ -0,0 +1,93 @@
+require_relative "tag/entry"
+require_relative "tag/spelling"
+require_relative "tag/reading"
+require_relative "tag/meaning"
+require_relative "tag/entity"
+require_relative "tag/cross_reference"
+require_relative "tag/antonym"
+require_relative "tag/source_language"
+require_relative "tag/definition"
+require_relative "tag/other"
+
+require_relative "jmdict_entities"
+
+module Eiwa
+  TAGS = {
+    "entry" => Tag::Entry,
+    "k_ele" => Tag::Spelling,
+    "r_ele" => Tag::Reading,
+    "sense" => Tag::Meaning,
+    "pos" => Tag::Entity,
+    "misc" => Tag::Entity,
+    "dial" => Tag::Entity,
+    "field" => Tag::Entity,
+    "ke_inf" => Tag::Entity,
+    "re_inf" => Tag::Entity,
+    "xref" => Tag::CrossReference,
+    "ant" => Tag::Antonym,
+    "lsource" => Tag::SourceLanguage,
+    "gloss" => Tag::Definition,
+  }
+
+  class JmdictDoc < Nokogiri::XML::SAX::Document
+    def initialize(each_entry_block)
+      @each_entry_block = each_entry_block
+    end
+
+    def start_document
+    end
+
+    def end_document
+    end
+
+    def start_element(name, attrs)
+      parent = @current
+      @current = (TAGS[name] || Tag::Other).new
+      @current.start(name, attrs, parent)
+    end
+
+    def end_element(name)
+      raise Eiwa::Error.new("Parsing error. Expected <#{@current.tag_name}> to close before <#{name}>") if @current.tag_name != name
+      ending = @current
+      ending.end_self
+      if ending.is_a?(Tag::Entry)
+        @each_entry_block&.call(ending)
+      end
+
+      @current = ending.parent
+      @current&.end_child(ending)
+    end
+
+    def characters(s)
+      @current.add_characters(s)
+    end
+
+    # def comment string
+    #   puts "comment #{string}"
+    # end
+
+    # def warning string
+    #   puts "warning #{string}"
+    # end
+
+    def error(msg)
+      if (matches = msg.match(/Entity '([\S]+)' not defined/))
+        # See: http://github.com/sparklemotion/nokogiri/issues/1926
+        code = matches[1]
+        @current.set_entity(code, JMDICT_ENTITIES[code])
+      elsif msg == "Detected an entity reference loop\n"
+        # Do nothing and hope this does not matter.
+      else
+        raise Eiwa::Error.new("Parsing error: #{msg}")
+      end
+    end
+
+    # def cdata_block string
+    #   puts "cdata_block #{string}"
+    # end
+
+    # def processing_instruction name, content
+    #   puts "processing_instruction #{name}, #{content}"
+    # end
+  end
+end

data/lib/eiwa/jmdict_entities.rb

@@ -0,0 +1,177 @@
+module Eiwa
+  JMDICT_ENTITIES = {
+    "MA" => "martial arts term",
+    "X" => "rude or X-rated term (not displayed in educational software)",
+    "abbr" => "abbreviation",
+    "adj-i" => "adjective (keiyoushi)",
+    "adj-ix" => "adjective (keiyoushi) - yoi/ii class",
+    "adj-na" => "adjectival nouns or quasi-adjectives (keiyodoshi)",
+    "adj-no" => "nouns which may take the genitive case particle `no'",
+    "adj-pn" => "pre-noun adjectival (rentaishi)",
+    "adj-t" => "`taru' adjective",
+    "adj-f" => "noun or verb acting prenominally",
+    "adv" => "adverb (fukushi)",
+    "adv-to" => "adverb taking the `to' particle",
+    "arch" => "archaism",
+    "ateji" => "ateji (phonetic) reading",
+    "aux" => "auxiliary",
+    "aux-v" => "auxiliary verb",
+    "aux-adj" => "auxiliary adjective",
+    "Buddh" => "Buddhist term",
+    "chem" => "chemistry term",
+    "chn" => "children's language",
+    "col" => "colloquialism",
+    "comp" => "computer terminology",
+    "conj" => "conjunction",
+    "cop-da" => "copula",
+    "ctr" => "counter",
+    "derog" => "derogatory",
+    "eK" => "exclusively kanji",
+    "ek" => "exclusively kana",
+    "exp" => "expressions (phrases, clauses, etc.)",
+    "fam" => "familiar language",
+    "fem" => "female term or language",
+    "food" => "food term",
+    "geom" => "geometry term",
+    "gikun" => "gikun (meaning as reading) or jukujikun (special kanji reading)",
+    "hon" => "honorific or respectful (sonkeigo) language",
+    "hum" => "humble (kenjougo) language",
+    "iK" => "word containing irregular kanji usage",
+    "id" => "idiomatic expression",
+    "ik" => "word containing irregular kana usage",
+    "int" => "interjection (kandoushi)",
+    "io" => "irregular okurigana usage",
+    "iv" => "irregular verb",
+    "ling" => "linguistics terminology",
+    "m-sl" => "manga slang",
+    "male" => "male term or language",
+    "male-sl" => "male slang",
+    "math" => "mathematics",
+    "mil" => "military",
+    "n" => "noun (common) (futsuumeishi)",
+    "n-adv" => "adverbial noun (fukushitekimeishi)",
+    "n-suf" => "noun, used as a suffix",
+    "n-pref" => "noun, used as a prefix",
+    "n-t" => "noun (temporal) (jisoumeishi)",
+    "num" => "numeric",
+    "oK" => "word containing out-dated kanji",
+    "obs" => "obsolete term",
+    "obsc" => "obscure term",
+    "ok" => "out-dated or obsolete kana usage",
+    "oik" => "old or irregular kana form",
+    "on-mim" => "onomatopoeic or mimetic word",
+    "pn" => "pronoun",
+    "poet" => "poetical term",
+    "pol" => "polite (teineigo) language",
+    "pref" => "prefix",
+    "proverb" => "proverb",
+    "prt" => "particle",
+    "physics" => "physics terminology",
+    "quote" => "quotation",
+    "rare" => "rare",
+    "sens" => "sensitive",
+    "sl" => "slang",
+    "suf" => "suffix",
+    "uK" => "word usually written using kanji alone",
+    "uk" => "word usually written using kana alone",
+    "unc" => "unclassified",
+    "yoji" => "yojijukugo",
+    "v1" => "Ichidan verb",
+    "v1-s" => "Ichidan verb - kureru special class",
+    "v2a-s" => "Nidan verb with 'u' ending (archaic)",
+    "v4h" => "Yodan verb with `hu/fu' ending (archaic)",
+    "v4r" => "Yodan verb with `ru' ending (archaic)",
+    "v5aru" => "Godan verb - -aru special class",
+    "v5b" => "Godan verb with `bu' ending",
+    "v5g" => "Godan verb with `gu' ending",
+    "v5k" => "Godan verb with `ku' ending",
+    "v5k-s" => "Godan verb - Iku/Yuku special class",
+    "v5m" => "Godan verb with `mu' ending",
+    "v5n" => "Godan verb with `nu' ending",
+    "v5r" => "Godan verb with `ru' ending",
+    "v5r-i" => "Godan verb with `ru' ending (irregular verb)",
+    "v5s" => "Godan verb with `su' ending",
+    "v5t" => "Godan verb with `tsu' ending",
+    "v5u" => "Godan verb with `u' ending",
+    "v5u-s" => "Godan verb with `u' ending (special class)",
+    "v5uru" => "Godan verb - Uru old class verb (old form of Eru)",
+    "vz" => "Ichidan verb - zuru verb (alternative form of -jiru verbs)",
+    "vi" => "intransitive verb",
+    "vk" => "Kuru verb - special class",
+    "vn" => "irregular nu verb",
+    "vr" => "irregular ru verb, plain form ends with -ri",
+    "vs" => "noun or participle which takes the aux. verb suru",
+    "vs-c" => "su verb - precursor to the modern suru",
+    "vs-s" => "suru verb - special class",
+    "vs-i" => "suru verb - included",
+    "kyb" => "Kyoto-ben",
+    "osb" => "Osaka-ben",
+    "ksb" => "Kansai-ben",
+    "ktb" => "Kantou-ben",
+    "tsb" => "Tosa-ben",
+    "thb" => "Touhoku-ben",
+    "tsug" => "Tsugaru-ben",
+    "kyu" => "Kyuushuu-ben",
+    "rkb" => "Ryuukyuu-ben",
+    "nab" => "Nagano-ben",
+    "hob" => "Hokkaido-ben",
+    "vt" => "transitive verb",
+    "vulg" => "vulgar expression or word",
+    "adj-kari" => "`kari' adjective (archaic)",
+    "adj-ku" => "`ku' adjective (archaic)",
+    "adj-shiku" => "`shiku' adjective (archaic)",
+    "adj-nari" => "archaic/formal form of na-adjective",
+    "n-pr" => "proper noun",
+    "v-unspec" => "verb unspecified",
+    "v4k" => "Yodan verb with `ku' ending (archaic)",
+    "v4g" => "Yodan verb with `gu' ending (archaic)",
+    "v4s" => "Yodan verb with `su' ending (archaic)",
+    "v4t" => "Yodan verb with `tsu' ending (archaic)",
+    "v4n" => "Yodan verb with `nu' ending (archaic)",
+    "v4b" => "Yodan verb with `bu' ending (archaic)",
+    "v4m" => "Yodan verb with `mu' ending (archaic)",
+    "v2k-k" => "Nidan verb (upper class) with `ku' ending (archaic)",
+    "v2g-k" => "Nidan verb (upper class) with `gu' ending (archaic)",
+    "v2t-k" => "Nidan verb (upper class) with `tsu' ending (archaic)",
+    "v2d-k" => "Nidan verb (upper class) with `dzu' ending (archaic)",
+    "v2h-k" => "Nidan verb (upper class) with `hu/fu' ending (archaic)",
+    "v2b-k" => "Nidan verb (upper class) with `bu' ending (archaic)",
+    "v2m-k" => "Nidan verb (upper class) with `mu' ending (archaic)",
+    "v2y-k" => "Nidan verb (upper class) with `yu' ending (archaic)",
+    "v2r-k" => "Nidan verb (upper class) with `ru' ending (archaic)",
+    "v2k-s" => "Nidan verb (lower class) with `ku' ending (archaic)",
+    "v2g-s" => "Nidan verb (lower class) with `gu' ending (archaic)",
+    "v2s-s" => "Nidan verb (lower class) with `su' ending (archaic)",
+    "v2z-s" => "Nidan verb (lower class) with `zu' ending (archaic)",
+    "v2t-s" => "Nidan verb (lower class) with `tsu' ending (archaic)",
+    "v2d-s" => "Nidan verb (lower class) with `dzu' ending (archaic)",
+    "v2n-s" => "Nidan verb (lower class) with `nu' ending (archaic)",
+    "v2h-s" => "Nidan verb (lower class) with `hu/fu' ending (archaic)",
+    "v2b-s" => "Nidan verb (lower class) with `bu' ending (archaic)",
+    "v2m-s" => "Nidan verb (lower class) with `mu' ending (archaic)",
+    "v2y-s" => "Nidan verb (lower class) with `yu' ending (archaic)",
+    "v2r-s" => "Nidan verb (lower class) with `ru' ending (archaic)",
+    "v2w-s" => "Nidan verb (lower class) with `u' ending and `we' conjugation (archaic)",
+    "archit" => "architecture term",
+    "astron" => "astronomy, etc. term",
+    "baseb" => "baseball term",
+    "biol" => "biology term",
+    "bot" => "botany term",
+    "bus" => "business term",
+    "econ" => "economics term",
+    "engr" => "engineering term",
+    "finc" => "finance term",
+    "geol" => "geology, etc. term",
+    "law" => "law, etc. term",
+    "mahj" => "mahjong term",
+    "med" => "medicine, etc. term",
+    "music" => "music term",
+    "Shinto" => "Shinto term",
+    "shogi" => "shogi term",
+    "sports" => "sports term",
+    "sumo" => "sumo term",
+    "zool" => "zoology term",
+    "joc" => "jocular, humorous term",
+    "anat" => "anatomical term",
+  }
+end

data/lib/eiwa/parses_jmdict_file.rb

@@ -0,0 +1,21 @@
+require "nokogiri"
+require_relative "jmdict_doc"
+
+module Eiwa
+  class ParsesJmdictFile
+    def call(filename, each_entry_block)
+      if each_entry_block.nil?
+        entries = []
+        each_entry_block ||= ->(e) { entries << e }
+      end
+
+      JmdictDoc.new(each_entry_block).tap do |doc|
+        Nokogiri::XML::SAX::Parser.new(doc).parse_file(filename) do |ctx|
+          ctx.recovery = true
+        end
+      end
+
+      entries
+    end
+  end
+end

data/lib/eiwa/tag/antonym.rb

@@ -0,0 +1,28 @@
+module Eiwa
+  module Tag
+    class Antonym < Any
+      attr_reader :text, :sense_ordinal
+
+      def initialize(text: nil, sense_ordinal: nil)
+        @text = text
+        @sense_ordinal = sense_ordinal
+      end
+
+      def end_self
+        parts = @characters.split("・")
+        @text = parts[0]
+        @sense_ordinal = parts[1]&.to_i
+      end
+
+      def eql?(other)
+        @text == other.text &&
+          @sense_ordinal == other.sense_ordinal
+      end
+      alias == eql?
+
+      def hash
+        @text.hash + @sense_ordinal.hash
+      end
+    end
+  end
+end

data/lib/eiwa/tag/any.rb

@@ -0,0 +1,28 @@
+module Eiwa
+  module Tag
+    class Any
+      attr_reader :tag_name, :characters, :parent
+
+      def start(tag_name, attrs, parent)
+        @tag_name = tag_name
+        @attrs = Hash[attrs]
+        @parent = parent
+      end
+
+      def add_characters(s)
+        @characters ||= ""
+        @characters << s.chomp
+      end
+
+      def end_child(child)
+      end
+
+      def end_self
+      end
+
+      def to_s
+        "<#{@tag_name}>#{@characters}</#{@tag_name}>"
+      end
+    end
+  end
+end

data/lib/eiwa/tag/cross_reference.rb

@@ -0,0 +1,31 @@
+module Eiwa
+  module Tag
+    class CrossReference < Any
+      attr_reader :text, :reading, :sense_ordinal
+
+      def initialize(text: nil, reading: nil, sense_ordinal: nil)
+        @text = text
+        @reading = reading
+        @sense_ordinal = sense_ordinal
+      end
+
+      def end_self
+        parts = @characters.split("・")
+        @text = parts.first
+        @reading = parts[1..-1].find { |part| /[^0-9]/.match(part) }
+        @sense_ordinal = parts.find { |part| /^[0-9]+$/.match(part) }&.to_i
+      end
+
+      def eql?(other)
+        @text == other.text &&
+          @reading == other.reading &&
+          @sense_ordinal == other.sense_ordinal
+      end
+      alias == eql?
+
+      def hash
+        @text.hash + @reading.hash + @sense_ordinal.hash
+      end
+    end
+  end
+end

data/lib/eiwa/tag/definition.rb

@@ -0,0 +1,45 @@
+module Eiwa
+  module Tag
+    class Definition < Any
+      attr_reader :text, :language, :gender, :type
+
+      def initialize(text: nil, language: "eng", gender: nil, type: nil)
+        @text = text
+        @language = language
+        @gender = gender
+        @type = type
+      end
+
+      def end_self
+        @text = @characters
+        @language = @attrs["xml:lang"]
+        @gender = @attrs["g_gend"]
+        @type = @attrs["g_type"]
+      end
+
+      def literal?
+        @type == "lit"
+      end
+
+      def figurative?
+        @type == "fig"
+      end
+
+      def explanation?
+        @type == "expl"
+      end
+
+      def eql?(other)
+        @text == other.text &&
+          @language == other.language &&
+          @gender == other.gender &&
+          @type == other.type
+      end
+      alias == eql?
+
+      def hash
+        @text.hash + @language.hash + @gender.hash + @type.hash
+      end
+    end
+  end
+end

data/lib/eiwa/tag/entity.rb

@@ -0,0 +1,29 @@
+require_relative "any"
+
+module Eiwa
+  module Tag
+    class Entity < Any
+      attr_reader :code, :text
+
+      def initialize(code: nil, text: nil)
+        @code = code
+        @text = text
+      end
+
+      def set_entity(code, text)
+        @code = code
+        @text = text
+      end
+
+      def eql?(other)
+        @code == other.code &&
+          @text == other.text
+      end
+      alias == eql?
+
+      def hash
+        @code.hash + @text.hash
+      end
+    end
+  end
+end

data/lib/eiwa/tag/entry.rb

@@ -0,0 +1,33 @@
+require_relative "any"
+
+module Eiwa
+  module Tag
+    class Entry < Any
+      attr_reader :id, :spellings, :readings, :meanings
+
+      def initialize
+        @spellings = []
+        @readings = []
+        @meanings = []
+      end
+
+      def text
+        (@spellings + @readings).first.text
+      end
+
+      def end_child(child)
+        case child.tag_name
+        when "ent_seq"
+          @id = child.characters.to_i
+        when "k_ele"
+          @spellings << child
+        when "r_ele"
+          @readings << child
+        when "sense"
+          child.trickle_down(@meanings.last) unless @meanings.last.nil?
+          @meanings << child
+        end
+      end
+    end
+  end
+end

data/lib/eiwa/tag/meaning.rb

@@ -0,0 +1,57 @@
+require_relative "any"
+
+module Eiwa
+  module Tag
+    class Meaning < Any
+      attr_reader :parts_of_speech, :definitions, :misc_tags,
+        :cross_references, :restricted_to_readings, :restricted_to_spellings,
+        :antonyms, :fields, :source_languages, :dialects, :comments
+
+      def initialize
+        @parts_of_speech = []
+        @definitions = []
+        @misc_tags = []
+        @cross_references = []
+        @restricted_to_readings = []
+        @restricted_to_spellings = []
+        @antonyms = []
+        @fields = []
+        @source_languages = []
+        @dialects = []
+        @comments = []
+      end
+
+      def end_child(child)
+        case child.tag_name
+        when "pos"
+          @parts_of_speech << child
+        when "gloss"
+          @definitions << child
+        when "misc"
+          @misc_tags << child
+        when "field"
+          @fields << child
+        when "xref"
+          @cross_references << child
+        when "ant"
+          @antonyms << child
+        when "stagr"
+          @restricted_to_readings << child.characters
+        when "stagk"
+          @restricted_to_spellings << child.characters
+        when "lsource"
+          @source_languages << child
+        when "dial"
+          @dialects << child
+        when "s_inf"
+          @comments << child.characters
+        end
+      end
+
+      def trickle_down(previous)
+        @parts_of_speech = previous.parts_of_speech if @parts_of_speech.empty?
+        @misc_tags = previous.misc_tags if @misc_tags.empty?
+      end
+    end
+  end
+end

data/lib/eiwa/tag/other.rb

@@ -0,0 +1,9 @@
+require_relative "any"
+
+module Eiwa
+  module Tag
+    class Other < Any
+      attr_reader :text
+    end
+  end
+end

data/lib/eiwa/tag/reading.rb

@@ -0,0 +1,32 @@
+require_relative "any"
+
+module Eiwa
+  module Tag
+    class Reading < Any
+      attr_reader :text, :frequency_tags, :info_tags
+
+      def initialize
+        @frequency_tags = []
+        @info_tags = []
+        @imprecise_reading = false
+      end
+
+      def imprecise_reading?
+        @imprecise_reading
+      end
+
+      def end_child(child)
+        case child.tag_name
+        when "reb"
+          @text = child.characters
+        when "re_pri"
+          @frequency_tags << child.characters.to_sym
+        when "re_inf"
+          @info_tags << child
+        when "re_nokanji"
+          @imprecise_reading = true
+        end
+      end
+    end
+  end
+end

data/lib/eiwa/tag/source_language.rb

@@ -0,0 +1,33 @@
+module Eiwa
+  module Tag
+    class SourceLanguage < Any
+      attr_reader :text, :language, :wasei, :type
+
+      def initialize(text: nil, language: "eng", wasei: false, type: "full")
+        @text = text
+        @language = language
+        @wasei = wasei
+        @type = type
+      end
+
+      def end_self
+        @text = @characters
+        @language = @attrs["xml:lang"]
+        @wasei = @attrs["ls_wasei"] == "y"
+        @type = @attrs["ls_type"] || "full"
+      end
+
+      def eql?(other)
+        @text == other.text &&
+          @language == other.language &&
+          @wasei == other.wasei &&
+          @type == other.type
+      end
+      alias == eql?
+
+      def hash
+        @text.hash + @language.hash + @wasei.hash + @type.hash
+      end
+    end
+  end
+end

data/lib/eiwa/tag/spelling.rb

@@ -0,0 +1,25 @@
+require_relative "any"
+
+module Eiwa
+  module Tag
+    class Spelling < Any
+      attr_reader :text, :frequency_tags, :info_tags
+
+      def initialize
+        @frequency_tags = []
+        @info_tags = []
+      end
+
+      def end_child(child)
+        case child.tag_name
+        when "keb"
+          @text = child.characters
+        when "ke_pri"
+          @frequency_tags << child.characters.to_sym
+        when "ke_inf"
+          @info_tags << child
+        end
+      end
+    end
+  end
+end

data/lib/eiwa/version.rb

@@ -0,0 +1,3 @@
+module Eiwa
+  VERSION = "0.0.1"
+end

data/script/download_jmdict

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+
+mkdir -p tmp/
+curl http://ftp.monash.edu/pub/nihongo/JMdict_e -o tmp/jmdict.xml

data/script/parse_jmdict

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path("../../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "eiwa"
+
+count = 0
+Eiwa.parse_file("tmp/jmdict.xml", type: :jmdict_e) do |entry|
+  count += 1
+end
+
+puts "Cool. #{count} words"

metadata

@@ -0,0 +1,155 @@
+--- !ruby/object:Gem::Specification
+name: eiwa
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Justin Searls
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2019-09-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- searls@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/ruby.yml"
+- ".gitignore"
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- eiwa.gemspec
+- lib/eiwa.rb
+- lib/eiwa/jmdict_doc.rb
+- lib/eiwa/jmdict_entities.rb
+- lib/eiwa/parses_jmdict_file.rb
+- lib/eiwa/tag/antonym.rb
+- lib/eiwa/tag/any.rb
+- lib/eiwa/tag/cross_reference.rb
+- lib/eiwa/tag/definition.rb
+- lib/eiwa/tag/entity.rb
+- lib/eiwa/tag/entry.rb
+- lib/eiwa/tag/meaning.rb
+- lib/eiwa/tag/other.rb
+- lib/eiwa/tag/reading.rb
+- lib/eiwa/tag/source_language.rb
+- lib/eiwa/tag/spelling.rb
+- lib/eiwa/version.rb
+- script/download_jmdict
+- script/parse_jmdict
+homepage: https://github.com/searls/eiwa
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: Parses the JMDict Japanese-English dictionary
+test_files: []