ruby-jdict 0.0.7 → 0.0.8

data/lib/ruby-jdict/index.rb

@@ -0,0 +1,151 @@
+# encoding: utf-8
+require 'amalgalite'
+require 'fileutils'
+require 'io/console'
+
+module JDict
+  class DictIndex
+    ENTITY_REGEX = /<!ENTITY\s([^ ]*)\s\"(.*)">/
+
+    attr_reader :path
+
+    # Initialize a full-text search index backend for JMdict
+    # @param path [String] path to the dictionary
+    def initialize(path)
+      @dictionary_path = path
+      @index_path = File.dirname(@dictionary_path)
+      @pos_hash = {}
+
+      raise "No dictionary found at path #{@dictionary_path}" unless File.exists? @dictionary_path
+
+      @db_file = File.join(@index_path, "jdict.db")
+      initialize_db(@db_file)
+
+      build_index!
+    end
+
+    def built?
+      @index.first_value_from( "SELECT count(*) from search" ) != 0
+    end
+
+    def delete!
+      @index.close
+      @index = nil
+
+      File.unlink(@db_file) if File.exist?(@db_file)
+
+      initialize_db(@db_file)
+    end
+
+    # Builds the full-text search index
+    # @return [Integer] the number of indexed entries
+    def build_index!(&block)
+      entries_added = do_build_index(&block) unless built?
+
+      #make the hash from abbreviated parts of speech to full definitions
+      @pos_hash ||= build_pos_hash
+
+      entries_added
+    end
+
+    # Returns the search results as an array of +Entry+
+    # @param term [String] the search string
+    # @param language [Symbol] the language to return results in
+    # @return [Array(Entry)] the results of the search
+    def search(term, opts = {})
+      raise "Index not found at path #{@index_path}" unless File.exists? @index_path
+
+      results = []
+
+      query = make_query(term, opts[:exact])
+
+      @index.execute("SELECT sequence_number, kanji, kana, senses, bm25(search) as score FROM search WHERE search MATCH ? LIMIT ?", query, opts[:max_results]) do |row|
+        entry = Entry.from_sql(row)
+        score = 0.0
+
+        is_exact_match = entry.kanji.include?(term) || entry.kana.include?(term)
+        score = 1.0 if is_exact_match
+
+        should_add = !opts[:exact] || (opts[:exact] && is_exact_match)
+
+        # add the result
+        results << [score, entry] if should_add
+      end
+
+      # Sort the results by first column (score) and return only the second column (entry)
+      results.sort_by { |entry| -entry[0] }.map { |entry| entry[1] }
+    end
+
+    # Retrieves the definition of a part-of-speech from its abbreviation
+    # @param pos [String] the abbreviation for the part-of-speech
+    # @return [String] the full description of the part-of-speech
+    def get_pos(pos)
+      build_pos_hash if @pos_hash.empty?
+      @pos_hash[pos_to_sym(pos)]
+    end
+
+    private
+
+    def initialize_db(db_file)
+      @index = Amalgalite::Database.new(db_file)
+      @pos_hash = nil
+
+      create_schema
+    end
+
+    # Creates the SQL schema for the Amalgalite database
+    def create_schema
+      schema = @index.schema
+      unless schema.tables['search']
+        @index.execute_batch <<-SQL
+        CREATE VIRTUAL TABLE search USING fts5(
+            sequence_number,
+            kanji,
+            kana,
+            senses
+        );
+        SQL
+        @index.reload_schema!
+      end
+    end
+
+    def make_query(term, exact)
+      # convert full-width katakana to hiragana
+      # TODO: move to user code
+      # term = Convert.kata_to_hira(term)
+
+      if term.start_with?('seq:')
+        query = "sequence_number : \"#{term[4..-1]}\""
+      else
+        query = "{kanji kana senses} : \"#{term}\""
+        query += "*" unless exact
+      end
+
+      query
+    end
+
+    def do_build_index(&block)
+        indexer = NokogiriDictionaryIndexer.new @dictionary_path
+        entries_added = 0
+
+        @index.transaction do |db_transaction|
+          entries_added = indexer.index(db_transaction, &block)
+        end
+
+        entries_added
+    end
+
+    # Creates the hash of part-of-speech symbols to full definitions from the dictionary
+    def build_pos_hash
+      indexer = NokogiriDictionaryIndexer.new @dictionary_path
+      indexer.parse_parts_of_speech
+    end
+
+    # Converts a part-of-speech entity reference string into a symbol
+    # @param entity [String] the entity reference string
+    # @return [Symbol] the part-of-speech symbol
+    def pos_to_sym(entity)
+      entity.gsub('-', '_').to_sym
+    end
+  end
+end

data/lib/ruby-jdict/indexer/dictionary_indexer.rb

@@ -0,0 +1,28 @@
+require 'amalgalite'
+
+module JDict
+  class DictionaryIndexer
+    attr_reader :parts_of_speech
+
+    def initialize(path)
+      raise "No dictionary path was provided" if path.nil?
+      raise "Dictionary not found at path #{@path}" unless File.exists?(path)
+
+      @path = path
+    end
+
+    def index(db_transaction, &block)
+    end
+
+    def parse_parts_of_speech
+    end
+
+    protected
+
+    def add_entry(db_transaction, entry)
+      db_transaction.prepare("INSERT INTO search( sequence_number, kanji, kana, senses ) VALUES( :sequence_number, :kanji, :kana, :senses );") do |stmt|
+        stmt.execute(entry.to_sql)
+      end
+    end
+  end
+end

data/lib/ruby-jdict/indexer/libxml_dictionary_indexer.rb

@@ -0,0 +1,164 @@
+require 'libxml'
+include LibXML
+
+module JDict
+
+  class LibXMLDictionaryIndexer < DictionaryIndexer
+    def initialize(path)
+      super
+    end
+
+    def index(db_transaction, &block)
+      reader = open_reader(@path)
+
+      # whenever there is a reader error, print its block parameters
+      XML::Error.set_handler { |*args| p args }
+
+      entry_sequence_num, kanji, kana, senses = 0, [], [], []
+      language = nil
+      glosses = {}
+      parts_of_speech = []
+
+      entries_added = 0
+
+      while reader.read
+        yield entries_added, 0 if block_given?
+
+        case reader.node_type
+
+        # start-of-element node
+        when XML::Reader::TYPE_ELEMENT
+          case reader.name
+          when JDict::JMDictConstants::Elements::SEQUENCE
+            entry_sequence_num = reader.next_text.to_i
+
+          # TODO: Raise an exception if reader.next_text.empty? inside the when's
+          #       JMdict shouldn't have any empty elements, I believe.
+          when JDict::JMDictConstants::Elements::KANJI
+            text = reader.next_text
+            kanji << text unless text.empty?
+
+          when JDict::JMDictConstants::Elements::KANA
+            text = reader.next_text
+            kana << text unless text.empty?
+
+          when JDict::JMDictConstants::Elements::GLOSS
+            # Assume the language of the whole sense is the language
+            # of the first gloss (in practice, there is never a gloss
+            # with more than one language)
+            unless language
+              language = reader.node.lang || JMDictConstants::LANGUAGE_DEFAULT
+              language = language.intern
+            end
+            text = reader.next_text
+            glosses << text unless text.empty?
+
+          when JDict::JMDictConstants::Elements::CROSSREFERENCE
+            text = reader.next_text
+          end
+
+        # XML entity references are treated as a different node type
+        # the parent node of the entity reference itself has the actual tag name
+        when XML::Reader::TYPE_ENTITY_REFERENCE
+          if reader.node.parent.name == JDict::JMDictConstants::Elements::PART_OF_SPEECH
+            text = reader.name
+            parts_of_speech << text unless text.empty?
+          end
+
+        when XML::Reader::TYPE_END_ELEMENT
+          case reader.name
+
+          when JDict::JMDictConstants::Elements::SENSE
+            senses << Sense.new(parts_of_speech, glosses, language)
+
+            # clear data for the next sense
+            glosses = {}
+            parts_of_speech = []
+            language = nil
+
+          # we're at the end of the entry element, so index it
+          when JDict::JMDictConstants::Elements::ENTRY
+            raise "No kana found for this entry!" if kana.empty?
+
+            entry = Entry.new(entry_sequence_num, kanji, kana, senses)
+            add_entry(entry)
+
+            # clear data for the next entry
+            kanji, kana, senses = [], [], []
+
+            entries_added += 1
+          end
+        end
+      end
+
+      reader.close
+
+      entries_added
+    end
+
+    def parse_parts_of_speech
+      pos_hash = {}
+      done = false
+      until done
+        reader.read
+        case reader.node_type
+        when XML::Reader::TYPE_DOCUMENT_TYPE
+          # segfaults when attempting this:
+          # cs.each do |child|
+          #   p child.to_s
+          # end
+          doctype_string = reader.node.to_s
+          entities = doctype_string.scan(ENTITY_REGEX)
+          entities.map do |entity|
+            abbrev = entity[0]
+            full = entity[1]
+            sym = pos_to_sym(abbrev)
+            pos_hash[sym] = full
+          end
+          done = true
+        when XML::Reader::TYPE_ELEMENT
+          done = true
+        end
+      end
+
+      reader.close
+
+      printf "\n"
+
+      pos_hash
+    end
+
+    private
+
+    # Creates an XML::Reader object for the given path
+    # @param dictionary_path [String] path to the dictionary file
+    # @return [XML::Reader] the reader for the given dictionary
+    def open_reader(dictionary_path)
+      # open reader
+      reader = nil
+      Dir.chdir(Dir.pwd) do
+        jmdict_path = File.join(dictionary_path)
+        reader = XML::Reader.file(jmdict_path, :encoding => XML::Encoding::UTF_8) # create a reader for JMdict
+        raise "Failed to create XML::Reader for #{dictionary_path}!" if reader.nil?
+      end
+      reader
+    end
+  end
+
+  # Add custom parsing methods to XML::Reader
+  class XML::Reader
+    public
+
+    def next_text
+      while (self.node_type != XML::Reader::TYPE_TEXT and self.read); end
+      self.value
+    end
+
+    def next_entity
+      while (self.node_type != XML::Reader::TYPE_ENTITY and
+             self.node_type != XML::Reader::TYPE_ENTITY_REFERENCE and
+             self.read); end
+      self.value
+    end
+  end
+end

data/lib/ruby-jdict/indexer/nokogiri_dictionary_indexer.rb

@@ -0,0 +1,60 @@
+require 'nokogiri'
+
+module JDict
+  class NokogiriDictionaryIndexer < JDict::DictionaryIndexer
+    def initialize(path)
+      super
+
+      @doc = File.open(path) do |f|
+        Nokogiri::XML(f) { |c| c.strict }
+      end
+    end
+
+    def index(db_transaction, &block)
+      raw = @doc/"./JMdict/entry"
+      total = raw.count
+      entries_added = 0
+
+      raw.each do |entry|
+        yield entries_added, total if block_given?
+
+        sequence_number = entry.at(JDict::JMDictConstants::Elements::SEQUENCE).content.to_i
+        kanji = (entry/JDict::JMDictConstants::Elements::KANJI).map(&:content)
+        kana = (entry/JDict::JMDictConstants::Elements::KANA).map(&:content)
+        senses = (entry/JDict::JMDictConstants::Elements::SENSE).map(&method(:extract_sense))
+
+        entry = Entry.new(sequence_number, kanji, kana, senses)
+        add_entry(db_transaction, entry)
+        entries_added += 1
+      end
+
+      printf "\n"
+
+      entries_added
+    end
+
+    def parse_parts_of_speech
+      {}
+    end
+
+    private
+
+    def extract_sense(e)
+      parts_of_speech = (e/JDict::JMDictConstants::Elements::PART_OF_SPEECH).map(&:inner_html)
+      glosses = (e/JDict::JMDictConstants::Elements::GLOSS).map(&:content)
+
+      # Assume the language of the whole sense is the language
+      # of the first gloss (in practice, there is never a gloss
+      # with more than one language in the official JMDict)
+      first_gloss = e.at(JDict::JMDictConstants::Elements::GLOSS)
+
+      language = if first_gloss
+                     first_gloss.attr("xml:lang")
+                 end
+
+      language ||= "en"
+
+      Sense.new(parts_of_speech, glosses, language)
+    end
+  end
+end

data/lib/ruby-jdict/jdict.rb

@@ -0,0 +1,2 @@
+module JDict
+end

data/lib/ruby-jdict/models/entry.rb

@@ -0,0 +1,64 @@
+#include Constants #XML constants from the dictionary file
+
+# Entries consist of kanji elements, kana elements,
+# general information and sense elements. Each entry must have at
+# least one kana element and one sense element. Others are optional.
+module JDict
+  class Entry
+    attr_accessor :sequence_number, :kanji, :kana, :senses
+    # Create a new Entry
+    #  entry = initialize(kanji, kana, senses)
+    def initialize(sequence_number, kanji, kana, senses)
+      @sequence_number, @kanji, @kana, @senses = sequence_number, kanji, kana, senses
+    end
+
+    # Converts an SQLite row from the index to the Entry format
+    def self.from_sql(row)
+      sequence_number = row["sequence_number"].to_i
+      kanji = row["kanji"].split(", ").map { |k| k = k.force_encoding("UTF-8") }
+      kana = row["kana"].split(", ").map { |k| k = k.force_encoding("UTF-8") }
+      senses = row["senses"].split(SerialConstants::SENSE_SENTINEL).sort.reduce([]) do |arr, txt|
+        arr << Sense.from_sql(txt)
+      end
+      self.new(sequence_number, kanji, kana, senses)
+    end
+
+    # Converts an Entry to a string to be indexed into the SQLite database
+    # @return [String] the serialized string for this Entry
+    def to_sql
+      sense_strings = senses.map(&:to_sql).join(SerialConstants::SENSE_SENTINEL)
+
+      { ':sequence_number' => sequence_number.to_s,
+        ':kanji' => kanji.join(", "),
+        ':kana' => kana.join(", "),
+        ':senses' => sense_strings }
+    end
+
+    # Get an array of +Senses+ for the specified language
+    def senses_by_language(l)
+      senses.select { |s| s.language == l }
+    end
+
+    def to_s
+      str = ""
+      str << "#{kanji_to_s}#{kana_to_s}\n"
+      str << "#{senses_to_s}\n"
+      str
+    end
+
+    def kanji_to_s
+      @kanji.join(', ')
+    end
+
+    def kana_to_s
+      " (#{@kana.join(', ')})" unless @kana.nil?
+    end
+
+    def senses_to_s(delimiter = "\n")
+      list = @senses.map.with_index(1) do |sense, i|
+        "#{i}. #{sense.to_s}"
+      end
+      list.join(delimiter)
+    end
+  end
+end