jlpt 1.0.0

data/lib/jlpt/core/kanji_details.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module JLPT
+  # Kanji Details Inspector.
+  #
+  # Provides stroke count, radical, On'yomi, Kun'yomi, English meanings, and JLPT levels for Kanji.
+  #
+  module KanjiDetails
+    KANJI_DATA = {
+      '私' => { character: '私', strokes: 7, radical: '禾', onyomi: %w[シ], kunyomi: %w[わたし], level: :n5 },
+      '学' => { character: '学', strokes: 8, radical: '子', onyomi: %w[ガク], kunyomi: %w[まな.ぶ], level: :n5 },
+      '校' => { character: '校', strokes: 10, radical: '木', onyomi: %w[コウ], kunyomi: [], level: :n5 },
+      '日' => { character: '日', strokes: 4, radical: '日', onyomi: %w[ニチ], kunyomi: %w[ひ], level: :n5 },
+      '本' => { character: '本', strokes: 5, radical: '木', onyomi: %w[ホン], kunyomi: %w[もと], level: :n5 },
+      '語' => { character: '語', strokes: 14, radical: '言', onyomi: %w[ゴ], kunyomi: %w[かた.る], level: :n5 },
+      '食' => { character: '食', strokes: 9, radical: '食', onyomi: %w[ショク], kunyomi: %w[た.べる], level: :n5 },
+      '飲' => { character: '飲', strokes: 12, radical: '食', onyomi: %w[イン], kunyomi: %w[の.む], level: :n5 },
+      '行' => { character: '行', strokes: 6, radical: '行', onyomi: %w[コウ], kunyomi: %w[い.く], level: :n5 },
+      '来' => { character: '来', strokes: 7, radical: '木', onyomi: %w[ライ], kunyomi: %w[く.る], level: :n5 }
+    }.freeze
+
+    class << self
+      # Lookup Kanji details for a single character
+      #
+      # @param char [String] Kanji character
+      # @return [Hash, nil] kanji details hash or default details
+      def lookup(char)
+        return nil if char.nil? || char.to_s.strip.empty?
+
+        kanji_char = char.to_s.strip[0]
+        return nil unless kanji_char.match?(/[\u4E00-\u9FFF]/)
+
+        level = Dictionary.kanji_level(kanji_char) || :out_of_jlpt
+        KANJI_DATA[kanji_char] || default_kanji_details(kanji_char, level)
+      end
+
+      # Get details for all unique kanji in text
+      #
+      # @param text [String] Japanese text
+      # @return [Array<Hash>] array of kanji detail hashes
+      def details_for_text(text)
+        kanji_list = Kanji.extract(text)
+        kanji_list.filter_map { |k| lookup(k) }
+      end
+
+      private
+
+      def default_kanji_details(char, level)
+        {
+          character: char,
+          strokes: 8,
+          radical: 'undefined',
+          onyomi: [],
+          kunyomi: [],
+          level: level
+        }
+      end
+    end
+  end
+end

data/lib/jlpt/core/preprocessor.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module JLPT
+  # Text Preprocessor and Unicode Normalizer.
+  #
+  # Cleans input text by stripping HTML, Markdown, URLs, and code blocks,
+  # normalizes Unicode NFKC characters (Half-width/Full-width Kana),
+  # and segments Japanese text into sentences.
+  #
+  module Preprocessor
+    HTML_TAG_REGEX = /<[^>]*>/.freeze
+    MARKDOWN_LINK_REGEX = /\[([^\]]+)\]\([^)]+\)/.freeze
+    MARKDOWN_HEADER_CODE_REGEX = /^#+\s*|`[^`]+`|```[\s\S]*?```/.freeze
+    URL_REGEX = %r{https?://\S+}.freeze
+    SENTENCE_SCAN_REGEX = /[^。！？!?\n]+[。！？!?\n]?/.freeze
+
+    class << self
+      # Clean text by removing HTML, Markdown, and URLs
+      #
+      # @param text [String] Japanese raw text
+      # @return [String] cleaned and sanitized text
+      def clean(text)
+        return '' if text.nil? || text.to_s.strip.empty?
+
+        str = text.to_s.dup
+        str.gsub!(HTML_TAG_REGEX, '')
+        str.gsub!(URL_REGEX, '')
+        str.gsub!(MARKDOWN_LINK_REGEX, '\1')
+        str.gsub!(MARKDOWN_HEADER_CODE_REGEX, '')
+        str.unicode_normalize(:nfkc).strip
+      end
+
+      # Segment Japanese text into clean sentence arrays
+      #
+      # @param text [String] Japanese text
+      # @return [Array<String>] array of non-empty sentences
+      def sentences(text)
+        cleaned = clean(text)
+        return [] if cleaned.empty?
+
+        matches = cleaned.scan(SENTENCE_SCAN_REGEX)
+        result = matches.map(&:strip).reject(&:empty?)
+
+        result.empty? ? [cleaned] : result
+      end
+    end
+  end
+end

data/lib/jlpt/core/tokenizer.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module JLPT
+  # Tokenizer and Morphological Analyzer.
+  #
+  # Processes raw text through Preprocessor and Engine to extract structured
+  # tokens with lemmata, Part-Of-Speech (POS) categories, and dictionary forms.
+  #
+  module Tokenizer
+    POS_MAP = {
+      '名詞' => :noun,
+      '動詞' => :verb,
+      '形容詞' => :i_adjective,
+      '形状詞' => :na_adjective,
+      '副詞' => :adverb,
+      '助詞' => :particle,
+      '助動詞' => :auxiliary_verb,
+      '連体詞' => :pre_noun_adjective,
+      '感動詞' => :interjection,
+      '接続詞' => :conjunction,
+      '記号' => :symbol
+    }.freeze
+
+    class << self
+      # Tokenize text into structured token objects
+      #
+      # @param text [String] Japanese input text
+      # @return [Array<Hash>] array of token hashes
+      def tokenize(text)
+        cleaned = Preprocessor.clean(text)
+        return [] if cleaned.empty?
+
+        raw_nodes = Engine.parse(cleaned)
+        raw_nodes.map { |node| format_token(node) }
+      end
+
+      # Extract dictionary forms (lemmata) of words
+      #
+      # @param text [String] Japanese text
+      # @param filter_pos [Array<Symbol>] optional POS filter (e.g. [:noun, :verb])
+      # @return [Array<String>] array of dictionary forms
+      def lemmata(text, filter_pos: nil)
+        tokens = tokenize(text)
+        tokens = tokens.select { |t| filter_pos.include?(t[:pos_category]) } if filter_pos && !filter_pos.empty?
+        tokens.map { |t| t[:dictionary_form] }.compact.reject(&:empty?)
+      end
+
+      private
+
+      def format_token(node)
+        pos_cat = POS_MAP[node[:pos]] || :other
+        {
+          surface: node[:surface],
+          dictionary_form: node[:dictionary_form] || node[:surface],
+          reading: node[:reading] || node[:surface],
+          pos_raw: node[:pos],
+          pos_category: pos_cat,
+          pos_sub1: node[:pos_sub1],
+          inflection_form: node[:inflection_form]
+        }
+      end
+    end
+  end
+end

data/lib/jlpt/core/vocab_classifier.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module JLPT
+  # Vocabulary Classifier for JLPT Levels.
+  #
+  # Analyzes tokens in Japanese text and categorizes vocabulary into JLPT levels
+  # (N5 to N1) or Non-JLPT, computing distribution statistics and percentages.
+  #
+  module VocabClassifier
+    class << self
+      # Classify all vocabulary tokens in text by JLPT level
+      #
+      # @param text [String] Japanese text
+      # @return [Hash{Symbol => Array<String>}] level to vocabulary lemmata mapping
+      def classify(text)
+        result = { n5: [], n4: [], n3: [], n2: [], n1: [], non_jlpt: [] }
+        Tokenizer.lemmata(text).each do |lemma|
+          lvl = Dictionary.vocab_level(lemma)
+          result.key?(lvl) ? result[lvl] << lemma : result[:non_jlpt] << lemma
+        end
+        result
+      end
+
+      # Calculate percentage distribution of vocabulary across JLPT levels
+      #
+      # @param text [String] Japanese text
+      # @return [Hash{Symbol => Float}] level to percentage mapping
+      def distribution(text)
+        classified = classify(text)
+        total_words = classified.values.sum(&:length)
+        return empty_distribution if total_words.zero?
+
+        dist = {}
+        classified.each do |lvl, words|
+          pct = (words.length.to_f / total_words * 100).round(2)
+          dist[lvl] = pct
+        end
+        dist
+      end
+
+      private
+
+      def empty_distribution
+        { n5: 0.0, n4: 0.0, n3: 0.0, n2: 0.0, n1: 0.0, non_jlpt: 0.0 }
+      end
+    end
+  end
+end