ruby-jdict 0.0.7 → 0.0.8

data/lib/entry.rb

@@ -1,101 +0,0 @@
-#include Constants #XML constants from the dictionary file
-
-# Entries consist of kanji elements, kana elements,
-# general information and sense elements. Each entry must have at
-# least one kana element and one sense element. Others are optional.
-module JDict
-  class Entry
-
-    attr_accessor :sequence_number, :kanji, :kana, :senses
-    # Create a new Entry
-    #  entry = initialize(kanji, kana, senses)
-    def initialize(sequence_number, kanji, kana, senses)
-      @sequence_number, @kanji, @kana, @senses = sequence_number, kanji, kana, senses
-    end
-
-    KANA_RE = /^kana/
-    SENSE_RE = /^sense/
-    PART_OF_SPEECH_RE = /^\[\[([^\]]+)\]\]\s+/
-
-    MEANING_SENTINEL = '**'
-    PART_OF_SPEECH_SENTINEL = '$$'
-    SENSE_SENTINEL = '%%'
-    LANGUAGE_SENTINEL = '&&'
-    GLOSS_SENTINEL = '@@'
-
-    # Converts an SQLite row from the index to the Entry format
-    def self.from_sql(row)
-      sequence_number = row["sequence_number"].to_i
-      kanji = row["kanji"].split(", ").map { |k| k = k.force_encoding("UTF-8") }
-      kana = row["kana"].split(", ").map { |k| k = k.force_encoding("UTF-8") }
-      senses = []
-      row["senses"].split(SENSE_SENTINEL).sort.each do |txt|
-        ary = txt.scan(PART_OF_SPEECH_RE)
-        if ary.size == 1
-          parts_of_speech = ary[0][0].split(PART_OF_SPEECH_SENTINEL)
-          gloss_strings = txt[(ary.to_s.length-1)..-1]
-        else
-          parts_of_speech = nil
-          gloss_strings = txt[5..-1]
-        end
-
-        gloss_strings = gloss_strings.force_encoding("UTF-8").strip.split(GLOSS_SENTINEL)
-
-        glosses = {}
-        gloss_strings.each do |str|
-          lang, meaning_string = str.split(LANGUAGE_SENTINEL)
-          lang = lang.to_sym
-          meanings = meaning_string.split(MEANING_SENTINEL)
-          (glosses[lang] ||= []) << meanings
-        end
-        glosses_for_lang = glosses[JDict.config.language] || glosses[JDict::JMDictConstants::Languages::ENGLISH]
-        senses << Sense.new(parts_of_speech, glosses_for_lang) # ** is the sentinel sequence
-      end
-      self.new(sequence_number, kanji, kana, senses)
-    end
-
-    # Converts an Entry to a string to be indexed into the SQLite database
-    # @return [String] the serialized string for this Entry
-    def to_sql
-      sense_strings = senses.map do |s|
-        sense = ''
-        sense << "[[#{s.parts_of_speech.join(PART_OF_SPEECH_SENTINEL)}]] " if s.parts_of_speech
-
-        # FIXME: it fails when retrieving entries from an existing index, because only one language is retrieved and the 'lang' field is nil
-        sense << s.glosses.collect { |lang, texts| lang.to_s + LANGUAGE_SENTINEL + texts.join(MEANING_SENTINEL) }.compact.join(GLOSS_SENTINEL)
-      end
-
-      { ':sequence_number' => sequence_number.to_s,
-        ':kanji' => kanji.join(", "),
-        ':kana' => kana.join(", "),
-        ':senses' => sense_strings.join(SENSE_SENTINEL) }
-    end
-
-    # Get an array of +Senses+ for the specified language
-    def senses_by_language(l)
-      senses.select { |s| s.language == l }
-    end
-
-    def to_s
-      str = ""
-      str << "#{kanji_to_s} (#{kana_to_s})\n"
-      str << "#{senses_to_s}\n"
-      str
-    end
-
-    def kanji_to_s
-      @kanji.join(', ')
-    end
-
-    def kana_to_s
-      @kana.join(', ') unless @kana.nil?
-    end
-
-    def senses_to_s(delimiter = "\n")
-      list = @senses.map.with_index(1) do |sense, i|
-        "#{i}. #{sense.to_s}"
-      end
-      list.join(delimiter)
-    end
-  end
-end

data/lib/index.rb

@@ -1,305 +0,0 @@
-# encoding: utf-8
-require 'amalgalite'
-require 'libxml'
-require 'fileutils'
-require 'io/console'
-
-require_relative 'constants' #XML constants from the dictionary file
-
-require_relative 'entry'     #dictionary elements
-require_relative 'kanji'     #...
-require_relative 'kana'      #...
-require_relative 'sense'
-
-include LibXML
-
-module JDict
-  class DictIndex
-
-    LANGUAGE_DEFAULT = JDict::JMDictConstants::Languages::ENGLISH
-    NUM_ENTRIES_TO_INDEX = 50
-    ENTITY_REGEX = /<!ENTITY\s([^ ]*)\s\"(.*)">/
-
-    attr_reader :path
-
-    # Initialize a full-text search index backend for JMdict
-    # @param path [String] path to the dictionary
-    def initialize(path)
-      @dictionary_path = path
-      @index_path = File.dirname(@dictionary_path)
-      @pos_hash = {}
-
-      raise "No dictionary found at path #{@dictionary_path}" unless File.exists? @dictionary_path
-
-      db_file = File.join(@index_path, "fts5.db")
-
-      File.unlink(db_file) if JDict.config.debug && File.exist?(db_file)
-
-      @index = Amalgalite::Database.new(db_file)
-
-      create_schema
-
-      build_index unless built?
-
-      #make the hash from abbreviated parts of speech to full definitions
-      @pos_hash ||= build_pos_hash
-    end
-
-    # Creates the SQL schema for the Amalgalite database
-    def create_schema
-      schema = @index.schema
-      unless schema.tables['search']
-        @index.execute_batch <<-SQL
-        CREATE VIRTUAL TABLE search USING fts5(
-            sequence_number,
-            kanji,
-            kana,
-            senses
-        );
-        SQL
-        @index.reload_schema!
-      end
-    end
-
-    def built?
-      @index.first_value_from( "SELECT count(*) from search" ) != 0
-    end
-
-    def make_query(term, exact)
-      # convert full-width katakana to hiragana
-      # TODO: convert half-width katakana to hiragana
-      term.tr!('ァ-ン','ぁ-ん')
-
-      if term.start_with?('seq:')
-        query = "sequence_number : \"#{term[4..-1]}\""
-      else
-        query = "{kanji kana senses} : \"#{term}\""
-        query += "*" unless exact
-      end
-
-      query
-    end
-
-    # Returns the search results as an array of +Entry+
-    # @param term [String] the search string
-    # @param language [Symbol] the language to return results in
-    # @return [Array(Entry)] the results of the search
-    def search(term, exact=false, language=LANGUAGE_DEFAULT)
-      raise "Index not found at path #{@index_path}" unless File.exists? @index_path
-
-      results = []
-
-      query = make_query(term, exact)
-
-      @index.execute("SELECT sequence_number, kanji, kana, senses, bm25(search) as score FROM search WHERE search MATCH ? LIMIT ?", query, JDict.config.num_results) do |row|
-        entry = Entry.from_sql(row)
-        score = 0.0
-
-        is_exact_match = entry.kanji.include?(term) || entry.kana.include?(term)
-        score = 1.0 if is_exact_match
-
-        should_add = !exact || (exact && is_exact_match)
-
-        # add the result
-        results << [score, entry] if should_add
-      end
-
-      # Sort the results by first column (score) and return only the second column (entry)
-      results.sort_by { |entry| -entry[0] }.map { |entry| entry[1] }
-    end
-
-    # Builds the full-text search index
-    # @param overwrite [Boolean] force a build even if the index path already exists
-    # @param dictionary_path [String] path to the dictionary file
-    # @return [Integer] the number of indexed entries
-    def build_index(overwrite=false, dictionary_path=nil)
-      @dictionary_path = dictionary_path unless dictionary_path.nil?
-      raise "No dictionary path was provided" if @dictionary_path.nil?
-      raise "Dictionary not found at path #{@dictionary_path}" unless File.exists?(@dictionary_path)
-
-      reader = open_reader(@dictionary_path)
-
-      puts "Building index..."
-
-      # whenever there is a reader error, print its block parameters
-      XML::Error.set_handler { |*args| p args }
-
-      # components of an entry
-      entry_sequence_num, kanji, kana, senses = 0, [], [], []
-      glosses = {}
-      parts_of_speech = []
-
-      entries_added = 0
-
-      @index.transaction do |db_transaction|
-
-        # read until the end
-        while reader.read
-
-          # check what type of node we're currently on
-          case reader.node_type
-
-            # start-of-element node
-          when XML::Reader::TYPE_ELEMENT
-            case reader.name
-            when JDict::JMDictConstants::Elements::SEQUENCE
-              entry_sequence_num = reader.next_text.to_i
-
-              # TODO: Raise an exception if reader.next_text.empty? inside the when's
-              #       JMdict shouldn't have any empty elements, I believe.
-            when JDict::JMDictConstants::Elements::KANJI
-              text = reader.next_text
-              kanji << text unless text.empty?
-
-            when JDict::JMDictConstants::Elements::KANA
-              text = reader.next_text
-              kana << text unless text.empty?
-
-            when JDict::JMDictConstants::Elements::GLOSS
-              language = reader.node.lang || LANGUAGE_DEFAULT
-              language = language.intern
-              text = reader.next_text
-              unless text.empty?
-                (glosses[language] ||= []) << text
-              end
-
-            when JDict::JMDictConstants::Elements::CROSSREFERENCE
-              text = reader.next_text
-            end
-
-            # XML entity references are treated as a different node type
-            # the parent node of the entity reference itself has the actual tag name
-          when XML::Reader::TYPE_ENTITY_REFERENCE
-            if reader.node.parent.name == JDict::JMDictConstants::Elements::PART_OF_SPEECH
-              text = reader.name
-              parts_of_speech << text unless text.empty?
-            end
-
-            # end-of-element node
-          when XML::Reader::TYPE_END_ELEMENT
-            case reader.name
-
-            when JDict::JMDictConstants::Elements::SENSE
-              # build sense
-              senses << Sense.new(parts_of_speech, glosses)
-              # glosses.each do |language, texts|
-              #   senses << Sense.new(parts_of_speech,
-              #                       texts.join(', ').strip,
-              #                       language)
-              # end
-
-              # clear data for the next sense
-              glosses = {}
-              parts_of_speech = []
-
-              # we're at the end of the entry element, so index it
-            when JDict::JMDictConstants::Elements::ENTRY
-              raise "No kana found for this entry!" if kana.empty?
-
-              #index
-              insert_data = Entry.new(entry_sequence_num, kanji, kana, senses).to_sql
-
-              db_transaction.prepare("INSERT INTO search( sequence_number, kanji, kana, senses ) VALUES( :sequence_number, :kanji, :kana, :senses );") do |stmt|
-                stmt.execute( insert_data )
-              end
-
-              # clear data for the next entry
-              kanji, kana, senses = [], [], []
-
-              entries_added += 1
-            end
-          end
-        end
-      end
-
-      # puts "#{@index.size} entries indexed"
-
-      # Done reading & indexing
-      reader.close
-      # @index.close
-    end
-
-    def rebuild_index
-    raise "Index already exists at path #{@index_path}" if File.exists? @index_path
-    build_index
-  end
-
-  # Creates an XML::Reader object for the given path
-  # @param dictionary_path [String] path to the dictionary file
-  # @return [XML::Reader] the reader for the given dictionary
-  def open_reader(dictionary_path)
-    # open reader
-    reader = nil
-    Dir.chdir(Dir.pwd) do
-      jmdict_path = File.join(dictionary_path)
-      reader = XML::Reader.file(jmdict_path, :encoding => XML::Encoding::UTF_8) # create a reader for JMdict
-      raise "Failed to create XML::Reader for #{dictionary_path}!" if reader.nil?
-    end
-    reader
-  end
-
-  # Creates the hash of part-of-speech symbols to full definitions from the dictionary
-  def build_pos_hash
-    pos_hash = {}
-    reader = open_reader(@dictionary_path)
-    done = false
-    until done
-      reader.read
-      case reader.node_type
-      when XML::Reader::TYPE_DOCUMENT_TYPE
-        # segfaults when attempting this:
-        # cs.each do |child|
-        #   p child.to_s
-        # end
-        doctype_string = reader.node.to_s
-        entities = doctype_string.scan(ENTITY_REGEX)
-        entities.map do |entity|
-          abbrev = entity[0]
-          full = entity[1]
-          sym = pos_to_sym(abbrev)
-          pos_hash[sym] = full
-        end
-        done = true
-      when XML::Reader::TYPE_ELEMENT
-        done = true
-      end
-    end
-    pos_hash
-  end
-
-  # Converts a part-of-speech entity reference string into a symbol
-  # @param entity [String] the entity reference string
-  # @return [Symbol] the part-of-speech symbol
-  def pos_to_sym(entity)
-    entity.gsub('-', '_').to_sym
-  end
-
-  # Retrieves the definition of a part-of-speech from its abbreviation
-  # @param pos [String] the abbreviation for the part-of-speech
-  # @return [String] the full description of the part-of-speech
-  def get_pos(pos)
-    build_pos_hash if @pos_hash.empty?
-    @pos_hash[pos_to_sym(pos)]
-  end
-end
-
-# Add custom parsing methods to XML::Reader
-class XML::Reader
-
-  public
-  # Get the next text node
-  def next_text
-    # read until a text node
-    while (self.node_type != XML::Reader::TYPE_TEXT and self.read); end
-    self.value
-  end
-  # Get the next entity node
-  def next_entity
-    # read until an entity node
-    while (self.node_type != XML::Reader::TYPE_ENTITY and
-      self.node_type != XML::Reader::TYPE_ENTITY_REFERENCE and
-      self.read); end
-    self.value
-  end
-end
-end

data/lib/jdict.rb

@@ -1,20 +0,0 @@
-require 'configuration'
-require 'dictionaries/jmdict'
-
-module JDict
-  class << self
-    attr_accessor :config
-  end
-
-  def self.config
-    @config ||= Configuration.new
-  end
-
-  def self.reset
-    @config = Configuration.new
-  end
-
-  def self.configure
-    yield(config)
-  end
-end

data/lib/kana.rb

@@ -1,4 +0,0 @@
-module JDict
-  class Kana
-  end
-end

data/lib/kanji.rb

@@ -1,4 +0,0 @@
-module JDict
-  class Kanji
-  end
-end

data/lib/sense.rb

@@ -1,28 +0,0 @@
-# The sense element will record the translational equivalent
-# of the Japanese word, plus other related information. Where there
-# are several distinctly different meanings of the word, multiple
-# sense elements will be employed.
-module JDict
-  class Sense
-    attr_reader :parts_of_speech, :glosses
-    #
-    # Create a new +Sense+
-    def initialize(parts_of_speech, glosses)
-      @parts_of_speech, @glosses = parts_of_speech, glosses
-    end
-
-    def to_s
-      parts_of_speech_to_s(@parts_of_speech) + glosses_to_s(@glosses)
-    end
-
-    private
-
-    def glosses_to_s(glosses)
-      glosses.join('; ')
-    end
-
-    def parts_of_speech_to_s(parts_of_speech)
-      parts_of_speech.nil? ? '' : '[' + parts_of_speech.join(',') + '] '
-    end
-  end
-end