jlpt 1.0.0

data/lib/jlpt/cli.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require 'optparse'
+require 'json'
+
+module JLPT
+  # Command Line Interface runner for JLPT gem.
+  module CLI
+    VALID_COMMANDS = %w[analyze furigana export batch kanji conjugate simplify stats].freeze
+
+    class << self
+      def run(args = ARGV)
+        options = { format: :text, target_level: :n4 }
+        parser = build_option_parser(options)
+        parser.parse!(args)
+        return 0 if options[:exit_early]
+
+        command = args.shift || 'analyze'
+        return warn_unknown(command) unless VALID_COMMANDS.include?(command.to_s.downcase)
+
+        execute_command(command, read_input(args), options)
+      end
+
+      private
+
+      def build_option_parser(opts_hash)
+        OptionParser.new do |opts|
+          opts.banner = 'Usage: jlpt [command] [options] <text or filepath>'
+          opts.on('-f', '--format FMT', %i[text json html markdown]) { |f| opts_hash[:format] = f }
+          opts.on('-t', '--target LVL', %i[n5 n4 n3 n2 n1]) { |l| opts_hash[:target_level] = l }
+          add_meta_flags(opts, opts_hash)
+        end
+      end
+
+      def add_meta_flags(opts, opts_hash)
+        opts.on('-v', '--version') do
+          puts "jlpt version #{JLPT::VERSION}"
+          opts_hash[:exit_early] = true
+        end
+        opts.on('-h', '--help') do
+          puts opts
+          opts_hash[:exit_early] = true
+        end
+      end
+
+      def warn_unknown(command)
+        warn "Unknown command: #{command}"
+        1
+      end
+
+      def execute_command(command, input, options)
+        cmd = command.to_s.downcase
+        if %w[analyze furigana export].include?(cmd)
+          execute_core_command(cmd, input, options)
+        else
+          execute_module_command(cmd, input, options)
+        end
+      end
+
+      def execute_core_command(cmd, input, options)
+        case cmd
+        when 'analyze'  then output_analysis(JLPT.analyze(input), options[:format])
+        when 'furigana' then puts Furigana.render(input, format: options[:format] == :html ? :html : :markdown)
+        when 'export'   then puts Exporter.to_anki_tsv(input, target_level: options[:target_level])
+        end
+        0
+      end
+
+      def execute_module_command(cmd, input, options)
+        data = parse_module_data(cmd, input)
+        options[:format] == :json ? puts(JSON.pretty_generate(data)) : puts(data.inspect)
+        0
+      end
+
+      def parse_module_data(cmd, input)
+        case cmd
+        when 'batch'     then parse_batch_input(input)
+        when 'kanji'     then KanjiDetails.details_for_text(input)
+        when 'conjugate' then Conjugation.analyze(input)
+        when 'simplify'  then Simplifier.simplify(input)
+        when 'stats'     then CorpusStats.analyze(input)
+        end
+      end
+
+      def parse_batch_input(input)
+        parsed = begin
+          JSON.parse(input)
+        rescue StandardError
+          input
+        end
+        BatchProcessor.process(parsed)
+      end
+
+      def read_input(args)
+        return '' if args.empty? && $stdin.tty?
+        return $stdin.read if args.empty?
+
+        File.exist?(args.first) ? File.read(args.first) : args.join(' ')
+      rescue StandardError
+        ''
+      end
+
+      def output_analysis(result, format)
+        case format
+        when :json     then puts JSON.pretty_generate(result.to_h)
+        when :html     then puts Reporter.render_html(result)
+        when :markdown then puts Reporter.render_markdown(result)
+        else                print_analysis_summary(result)
+        end
+      end
+
+      def print_analysis_summary(result)
+        puts "Recommended Level: #{result.recommended_level.to_s.upcase}"
+        puts "Difficulty Score:  #{result.score} / 100"
+        puts "Word Count:        #{result.word_count}"
+        puts "Sentence Count:    #{result.sentence_count}"
+        puts "Kanji Density:     #{(result.kanji_density * 100).round(1)}%"
+      end
+    end
+  end
+end

data/lib/jlpt/core/analysis_result.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module JLPT
+  # Value Object encapsulating JLPT text analysis results.
+  #
+  # Supports both method dot-notation and Hash subscripting access.
+  #
+  class AnalysisResult
+    attr_reader :text, :recommended_level, :score, :kanji_density,
+                :vocab_distribution, :kanji_breakdown,
+                :sentence_count, :word_count
+
+    def initialize(attrs = {})
+      @text               = attrs.fetch(:text, '')
+      @recommended_level  = attrs.fetch(:recommended_level, :n5)
+      @score              = attrs.fetch(:score, 0.0)
+      @kanji_density      = attrs.fetch(:kanji_density, 0.0)
+      @vocab_distribution = attrs.fetch(:vocab_distribution, {})
+      @kanji_breakdown    = attrs.fetch(:kanji_breakdown, {})
+      @sentence_count     = attrs.fetch(:sentence_count, 0)
+      @word_count         = attrs.fetch(:word_count, 0)
+    end
+
+    def [](key)
+      public_send(key) if respond_to?(key)
+    end
+
+    def to_h
+      {
+        text: @text,
+        recommended_level: @recommended_level,
+        score: @score,
+        kanji_density: @kanji_density,
+        vocab_distribution: @vocab_distribution,
+        kanji_breakdown: @kanji_breakdown,
+        sentence_count: @sentence_count,
+        word_count: @word_count
+      }
+    end
+
+    def inspect
+      "#<JLPT::AnalysisResult level=#{@recommended_level.to_s.upcase} score=#{@score}>"
+    end
+  end
+end

data/lib/jlpt/core/analyzer.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module JLPT
+  # Core Analyzer Engine computing overall text difficulty and JLPT level recommendation.
+  #
+  module Analyzer
+    LEVEL_WEIGHTS = { n5: 1.0, n4: 2.0, n3: 3.0, n2: 4.0, n1: 5.0, non_jlpt: 2.5, out_of_jlpt: 3.5 }.freeze
+
+    class << self
+      # Analyze Japanese text and return structured AnalysisResult
+      #
+      # @param text [String] Japanese text
+      # @return [JLPT::AnalysisResult] result object
+      def analyze(text)
+        cleaned = Preprocessor.clean(text)
+        return AnalysisResult.new(text: text.to_s) if cleaned.empty?
+
+        CacheManager.fetch(cleaned) { build_analysis_result(cleaned) }
+      end
+
+      private
+
+      def build_analysis_result(cleaned)
+        sents = Preprocessor.sentences(cleaned)
+        tokens = Tokenizer.tokenize(cleaned)
+        vocab_dist = VocabClassifier.distribution(cleaned)
+        kanji_bd = Kanji.breakdown(cleaned)
+        score = calculate_score(vocab_dist, kanji_bd)
+
+        AnalysisResult.new(text: cleaned, recommended_level: score_to_level(score), score: score,
+                           kanji_density: Kanji.density(cleaned), vocab_distribution: vocab_dist,
+                           kanji_breakdown: kanji_bd, sentence_count: sents.length, word_count: tokens.length)
+      end
+
+      def calculate_score(v_dist, k_bd)
+        v_score = v_dist.sum { |l, p| (p / 100.0) * (LEVEL_WEIGHTS[l] || 1.0) }
+        k_score = k_bd.values.sum(&:length).positive? ? compute_kanji_score(k_bd) : 1.0
+
+        ((((v_score * 0.6) + (k_score * 0.4)) - 1.0) / 4.0 * 100.0).clamp(0.0, 100.0).round(2)
+      end
+
+      def compute_kanji_score(kanji_bd)
+        total = kanji_bd.values.sum(&:length)
+        kanji_bd.sum { |l, c| (c.length.to_f / total) * (LEVEL_WEIGHTS[l] || 1.0) }
+      end
+
+      def score_to_level(score)
+        case score
+        when 0.0..25.0   then :n5
+        when 25.01..35.0 then :n4
+        when 35.01..41.0 then :n3
+        when 41.01..46.0 then :n2
+        else                  :n1
+        end
+      end
+    end
+  end
+end

data/lib/jlpt/core/cache_manager.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module JLPT
+  # Thread-safe LRU Cache Manager for JLPT Text Analysis.
+  #
+  # Caches analysis results to prevent redundant processing for identical texts.
+  #
+  module CacheManager
+    MUTEX = Mutex.new
+    @enabled = true
+    DEFAULT_CAPACITY = 1000
+
+    class << self
+      # Fetch cached result or compute via block
+      #
+      # @param key [String] cache key (text)
+      # @yield Compute result if not cached
+      # @return [Object] cached or computed value
+      def fetch(key, &block)
+        return yield unless enabled? && key.is_a?(String) && !key.empty?
+
+        MUTEX.synchronize { read_or_write_cache(key, &block) }
+      end
+
+      # Clear the cache store
+      def clear
+        MUTEX.synchronize { store.clear }
+      end
+
+      # Current cache size
+      #
+      # @return [Integer]
+      def size
+        MUTEX.synchronize { store.length }
+      end
+
+      # Check if cache is enabled
+      #
+      # @return [Boolean]
+      def enabled?
+        @enabled = true if @enabled.nil?
+        @enabled
+      end
+
+      # Enable caching
+      def enable!
+        @enabled = true
+      end
+
+      # Disable caching
+      def disable!
+        @enabled = false
+      end
+
+      # Maximum capacity
+      #
+      # @return [Integer]
+      def capacity
+        @capacity ||= DEFAULT_CAPACITY
+      end
+
+      # Set capacity
+      def capacity=(cap)
+        MUTEX.synchronize do
+          @capacity = cap
+          store.shift while store.length > cap
+        end
+      end
+
+      private
+
+      def store
+        @store ||= {}
+      end
+
+      def read_or_write_cache(key)
+        if store.key?(key)
+          val = store.delete(key)
+          store[key] = val
+          return val
+        end
+
+        val = yield
+        store[key] = val
+        store.shift if store.length > capacity
+        val
+      end
+    end
+  end
+end

data/lib/jlpt/core/config.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Main entry point for the JLPT Japanese text analysis library.
+module JLPT
+  # Global Configuration Engine.
+  #
+  # Manages default options such as furigana format, cache capacity,
+  # and fallback behaviors.
+  #
+  class Config
+    attr_accessor :default_furigana_format, :cache_capacity, :mecab_fallback
+
+    def initialize
+      @default_furigana_format = :html
+      @cache_capacity = 1000
+      @mecab_fallback = true
+    end
+
+    # Reset configuration to default values
+    def reset!
+      initialize
+    end
+  end
+
+  class << self
+    # Return global config instance
+    #
+    # @return [JLPT::Config]
+    def config
+      @config ||= Config.new
+    end
+
+    # Configure global JLPT library settings
+    #
+    # @yield [JLPT::Config]
+    def configure
+      yield(config) if block_given?
+    end
+
+    # Reset configuration to defaults
+    def reset_config!
+      config.reset!
+    end
+  end
+end

data/lib/jlpt/core/dictionary.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module JLPT
+  # Thread-safe In-Memory Dictionary Manager for JLPT Vocabulary and Kanji Datasets.
+  #
+  # Loaded from Bluskyo JLPT_Vocabulary datasets (N5 to N1).
+  #
+  module Dictionary
+    DATA_DIR = File.expand_path('../data', __dir__)
+    MUTEX = Mutex.new
+    @loaded = false
+
+    class << self
+      # Look up the JLPT level of a vocabulary word (dictionary form or lemma)
+      #
+      # @param word [String] vocabulary word
+      # @return [Symbol, nil] :n5, :n4, :n3, :n2, :n1 or nil if unknown
+      def vocab_level(word)
+        ensure_loaded!
+        return nil if word.nil? || word.to_s.strip.empty?
+
+        @vocab_index[word.to_s.strip]
+      end
+
+      # Look up reading (furigana) for a vocabulary word
+      #
+      # @param word [String]
+      # @return [String, nil]
+      def reading_for(word)
+        ensure_loaded!
+        return nil if word.nil? || word.to_s.strip.empty?
+
+        @readings_index[word.to_s.strip]
+      end
+
+      # Look up the JLPT level of a single Kanji character
+      #
+      # @param char [String] single Kanji character
+      # @return [Symbol, nil] :n5, :n4, :n3, :n2, :n1 or nil if non-kanji / unknown
+      def kanji_level(char)
+        ensure_loaded!
+        return nil if char.nil? || char.to_s.strip.empty?
+
+        kanji_char = char.to_s.strip[0]
+        @kanji_index[kanji_char]
+      end
+
+      # Check if a word exists in the JLPT vocabulary dataset
+      #
+      # @param word [String]
+      # @return [Boolean]
+      def vocab_exists?(word)
+        !vocab_level(word).nil?
+      end
+
+      # Check if a kanji exists in the JLPT kanji dataset
+      #
+      # @param char [String]
+      # @return [Boolean]
+      def kanji_exists?(char)
+        !kanji_level(char).nil?
+      end
+
+      private
+
+      def ensure_loaded!
+        return if @loaded
+
+        MUTEX.synchronize do
+          return if @loaded
+
+          load_vocab_dataset!
+          load_kanji_dataset!
+          @loaded = true
+        end
+      end
+
+      def load_vocab_dataset!
+        @vocab_index = {}
+        @readings_index = {}
+        vocab_path = File.join(DATA_DIR, 'vocab.json')
+        return unless File.exist?(vocab_path)
+
+        data = JSON.parse(File.read(vocab_path))
+        data.each { |word, entries| index_vocab_entry(word, entries) }
+      end
+
+      def index_vocab_entry(word, entries)
+        return unless entries.is_a?(Array) && (best = select_best_entry(entries))
+
+        @vocab_index[word] = :"n#{best['level'].to_i}"
+        @readings_index[word] = best['reading'] if best['reading']
+      end
+
+      def select_best_entry(entries)
+        return nil if entries.empty?
+
+        entries.find { |e| e['level'].to_i == 5 } || entries.min_by { |e| e['level'].to_i }
+      end
+
+      def load_kanji_dataset!
+        @kanji_index = {}
+        kanji_path = File.join(DATA_DIR, 'kanji.json')
+        return unless File.exist?(kanji_path)
+
+        data = JSON.parse(File.read(kanji_path))
+        data.each { |char, lvl_num| @kanji_index[char] = :"n#{lvl_num}" }
+      end
+    end
+  end
+end

data/lib/jlpt/core/engine.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module JLPT
+  # Morphological Analysis Engine Wrapper for JLPT.
+  #
+  # Uses MeCab (`natto` gem) if installed on system, with automatic fallback
+  # to a pure-Ruby morphological segmentation engine.
+  #
+  module Engine
+    @mecab_available = nil
+
+    class << self
+      # Check if MeCab is available on host system
+      #
+      # @return [Boolean]
+      def mecab_available?
+        return @mecab_available unless @mecab_available.nil?
+
+        @mecab_available = begin
+          require 'natto'
+          !Natto::MeCab.new.nil?
+        rescue LoadError, StandardError
+          false
+        end
+      end
+
+      # Parse Japanese text into structured token node hashes
+      #
+      # @param text [String] Japanese input text
+      # @return [Array<Hash>] token nodes
+      def parse(text)
+        return [] if text.nil? || text.to_s.strip.empty?
+
+        cleaned = Preprocessor.clean(text)
+        return [] if cleaned.empty?
+
+        if mecab_available?
+          parse_with_mecab(Natto::MeCab.new, cleaned)
+        else
+          parse_with_fallback(cleaned)
+        end
+      end
+
+      private
+
+      def parse_with_mecab(mecab, text)
+        nodes = []
+        mecab.parse(text) do |n|
+          next if n.surface.empty? || n.feature.nil?
+
+          nodes << build_mecab_node(n)
+        end
+        nodes
+      end
+
+      def build_mecab_node(node)
+        f = (node.feature || '').split(',')
+        s = node.surface
+        { surface: s }.merge(fetch_pos_features(f), fetch_form_features(f, s))
+      end
+
+      def fetch_pos_features(feat)
+        { pos: feat[0] || '*', pos_sub1: feat[1] || '*', pos_sub2: feat[2] || '*', pos_sub3: feat[3] || '*' }
+      end
+
+      def fetch_form_features(feat, surface)
+        reading = feat[7] || surface
+        { inflection_type: feat[4] || '*', inflection_form: feat[5] || '*',
+          dictionary_form: feat[6] || surface, reading: reading, pronunciation: feat[8] || reading }
+      end
+
+      def parse_with_fallback(text)
+        PureRubyFallback.parse(text)
+      end
+    end
+
+    # Pure Ruby Fallback Engine for environments without MeCab installed
+    module PureRubyFallback
+      KANJI_KATAKANA_REGEX = /[\u4E00-\u9FFF\u30A0-\u30FF]/.freeze
+      HIRAGANA_REGEX = /[\u3040-\u309F]/.freeze
+      WORD_SCAN_REGEX = /[\u4E00-\u9FFF]+|[\u3040-\u309F]+|[\u30A0-\u30FF]+|[a-zA-Z0-9]+|\S/.freeze
+
+      class << self
+        def parse(text)
+          return [] if text.nil? || text.to_s.strip.empty?
+
+          text.scan(WORD_SCAN_REGEX).flat_map { |token| process_fallback_token(token) }
+        end
+
+        private
+
+        def process_fallback_token(token)
+          if token.match?(/^[\u4E00-\u9FFF]{4,}$/) && !Dictionary.vocab_exists?(token)
+            split_kanji_compound(token).map { |sub| build_fallback_node(sub) }
+          else
+            [build_fallback_node(token)]
+          end
+        end
+
+        def split_kanji_compound(token)
+          chunks = []
+          i = 0
+          while i < token.length
+            chunk, step = find_next_chunk(token, i)
+            chunks << chunk
+            i += step
+          end
+          chunks
+        end
+
+        def find_next_chunk(token, idx)
+          s3 = token[idx, 3]
+          return [s3, 3] if s3.length == 3 && Dictionary.vocab_exists?(s3)
+
+          s2 = token[idx, 2]
+          return [s2, 2] if s2.length == 2 && Dictionary.vocab_exists?(s2)
+
+          len = s2.length >= 2 ? 2 : 1
+          [token[idx, len], len]
+        end
+
+        def build_fallback_node(token)
+          {
+            surface: token, pos: detect_pos(token), pos_sub1: '*', pos_sub2: '*',
+            pos_sub3: '*', inflection_type: '*', inflection_form: '*',
+            dictionary_form: token, reading: token, pronunciation: token
+          }
+        end
+
+        def detect_pos(token)
+          if token.match?(KANJI_KATAKANA_REGEX)
+            '名詞'
+          elsif token.match?(HIRAGANA_REGEX)
+            '助詞'
+          else
+            '記号'
+          end
+        end
+      end
+    end
+  end
+end

data/lib/jlpt/core/kanji.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module JLPT
+  # Dedicated Kanji Character Analyzer and Extractor.
+  #
+  # Identifies Kanji characters in Japanese text, maps them to JLPT levels (N5-N1),
+  # calculates kanji density, and extracts unique kanji filtered by level.
+  #
+  module Kanji
+    KANJI_REGEX = /[\u4E00-\u9FFF]/.freeze
+
+    class << self
+      # Extract all Kanji characters from text
+      #
+      # @param text [String] Japanese text
+      # @param level [Symbol, String, nil] optional JLPT level filter (:n5, :n4, :n3, :n2, :n1)
+      # @return [Array<String>] array of unique Kanji characters
+      def extract(text, level: nil)
+        return [] if text.nil? || text.to_s.strip.empty?
+
+        kanjis = Preprocessor.clean(text).scan(KANJI_REGEX).uniq
+        return kanjis if level.nil?
+
+        target_level = level.to_s.downcase.to_sym
+        kanjis.select { |k| Dictionary.kanji_level(k) == target_level }
+      end
+
+      # Calculate Kanji density ratio (Kanji characters / Total characters)
+      #
+      # @param text [String] Japanese text
+      # @return [Float] density ratio between 0.0 and 1.0
+      def density(text)
+        cleaned = Preprocessor.clean(text)
+        return 0.0 if cleaned.empty?
+
+        kanji_count = cleaned.scan(KANJI_REGEX).length
+        (kanji_count.to_f / cleaned.length).round(4)
+      end
+
+      # Detailed breakdown of Kanji by JLPT level
+      #
+      # @param text [String] Japanese text
+      # @return [Hash{Symbol => Array<String>}] breakdown hash mapping levels to kanji arrays
+      def breakdown(text)
+        res = { n5: [], n4: [], n3: [], n2: [], n1: [], out_of_jlpt: [] }
+        extract(text).each do |k|
+          lvl = Dictionary.kanji_level(k)
+          res.key?(lvl) ? res[lvl] << k : res[:out_of_jlpt] << k
+        end
+        res
+      end
+    end
+  end
+end