kabosu 0.6.10-x86_64-linux

data/lib/kabosu/morpheme_list.rb

@@ -0,0 +1,157 @@
+module Kabosu
+  class MorphemeList
+    include Enumerable
+
+    attr_accessor :internal_cost
+
+    def initialize(source_or_morphemes, internal_cost: nil)
+      @source = source_or_morphemes if lazy_source?(source_or_morphemes)
+      @morphemes = @source ? Array.new(@source.size) : source_or_morphemes
+      @internal_cost = internal_cost || (@source&.internal_cost)
+    end
+
+    def each(&block)
+      return enum_for(:each) unless block
+
+      if @source
+        i = 0
+        while i < size
+          block.call(fetch(i))
+          i += 1
+        end
+      else
+        @morphemes.each(&block)
+      end
+    end
+
+    def [](index)
+      return to_a[index] if index.is_a?(Range)
+
+      idx = normalize_index(index)
+      return nil if idx.nil?
+
+      fetch(idx)
+    end
+
+    def first(n = nil)
+      return self[0] unless n
+
+      n = Integer(n)
+      raise ArgumentError, "negative array size" if n.negative?
+
+      limit = [n, size].min
+      (0...limit).map { fetch(_1) }
+    end
+
+    def last(n = nil)
+      return self[-1] unless n
+
+      n = Integer(n)
+      raise ArgumentError, "negative array size" if n.negative?
+
+      start = [size - n, 0].max
+      (start...size).map { fetch(_1) }
+    end
+
+    def size
+      @morphemes.size
+    end
+
+    def empty?
+      @morphemes.empty?
+    end
+
+    def surfaces
+      return @source.surfaces if @source&.respond_to?(:surfaces)
+
+      map(&:surface)
+    end
+
+    def readings
+      map(&:reading_form)
+    end
+
+    def dictionary_forms
+      map(&:dictionary_form)
+    end
+
+    def normalized_forms
+      map(&:normalized_form)
+    end
+
+    def total_costs
+      map(&:total_cost)
+    end
+
+    def synonym_group_ids
+      map(&:synonym_group_ids)
+    end
+
+    # Joins all surfaces back into the original text (no spaces, for Japanese text).
+    def to_text
+      surfaces.join
+    end
+
+    # Filter morphemes by POS. Accepts a PosMatcher or an array pattern.
+    # Returns a new MorphemeList with only matching morphemes.
+    #
+    #   list.select_pos(Kabosu::PosMatcher.nouns)
+    #   list.select_pos(["名詞", "固有名詞"])
+    #
+    def select_pos(matcher_or_pattern)
+      matcher = coerce_to_matcher(matcher_or_pattern)
+      self.class.new(matcher.filter(to_a))
+    end
+
+    # Inverse of select_pos. Returns a new MorphemeList excluding matching morphemes.
+    def reject_pos(matcher_or_pattern)
+      matcher = coerce_to_matcher(matcher_or_pattern)
+      self.class.new(matcher.reject(to_a))
+    end
+
+    def to_a
+      return @morphemes.dup unless @source
+
+      (0...size).map { fetch(_1) }
+    end
+
+    def inspect
+      base = "#<Kabosu::MorphemeList (#{size} morphemes)"
+      base += " cost=#{@internal_cost}" if @internal_cost
+      base + ": #{surfaces.join(" | ")}>"
+    end
+
+    private
+
+    def coerce_to_matcher(matcher_or_pattern)
+      case matcher_or_pattern
+      when PosMatcher
+        matcher_or_pattern
+      when Array
+        PosMatcher.new(matcher_or_pattern)
+      else
+        raise ArgumentError, "expected a PosMatcher or an Array pattern, got #{matcher_or_pattern.class}"
+      end
+    end
+
+    def lazy_source?(obj)
+      obj.respond_to?(:morpheme_at) && obj.respond_to?(:size) && obj.respond_to?(:surfaces)
+    end
+
+    def normalize_index(index)
+      return nil if size.zero?
+
+      idx = Integer(index)
+      idx += size if idx.negative?
+      return nil if idx.negative? || idx >= size
+
+      idx
+    end
+
+    def fetch(idx)
+      return @morphemes[idx] unless @source
+
+      @morphemes[idx] ||= @source.morpheme_at(idx)
+    end
+  end
+end

data/lib/kabosu/pos_matcher.rb

@@ -0,0 +1,119 @@
+module Kabosu
+  class PosMatcher
+    # Build a matcher from POS patterns or a block.
+    #
+    #   # From a block
+    #   PosMatcher.new { |pos| pos[0] == "名詞" }
+    #
+    #   # Single pattern (array of strings; "*" or nil = match anything)
+    #   PosMatcher.new(["名詞", "固有名詞", "*", "*"])
+    #
+    #   # Multiple patterns (matches if any pattern matches)
+    #   PosMatcher.new(["名詞", "固有名詞"], ["動詞", "*", "*", "*"])
+    #
+    def initialize(*patterns, &block)
+      if block
+        raise ArgumentError, "cannot supply both patterns and a block" unless patterns.empty?
+
+        @proc = block
+      elsif patterns.empty?
+        raise ArgumentError, "must supply at least one pattern or a block"
+      else
+        @patterns = patterns.map(&:freeze).freeze
+      end
+
+      freeze
+    end
+
+    # Returns true if the morpheme (or raw POS array) matches this matcher.
+    def match?(morpheme_or_pos)
+      pos = extract_pos(morpheme_or_pos)
+
+      if @proc
+        @proc.call(pos)
+      else
+        @patterns.any? { |pattern| pattern_match?(pattern, pos) }
+      end
+    end
+
+    # Return matching morphemes as an Array.
+    def filter(morphemes)
+      morphemes.select { |m| match?(m) }
+    end
+
+    # Return non-matching morphemes as an Array.
+    def reject(morphemes)
+      morphemes.reject { |m| match?(m) }
+    end
+
+    # Union: matches if either matcher matches.
+    def |(other)
+      a, b = self, other
+      PosMatcher.new { |pos| a.match?(pos) || b.match?(pos) }
+    end
+
+    # Intersection: matches if both matchers match.
+    def &(other)
+      a, b = self, other
+      PosMatcher.new { |pos| a.match?(pos) && b.match?(pos) }
+    end
+
+    # Difference: matches self but not other.
+    def -(other)
+      difference(other)
+    end
+
+    def difference(other)
+      a, b = self, other
+      PosMatcher.new { |pos| a.match?(pos) && !b.match?(pos) }
+    end
+
+    # ── Pre-built matchers ──
+
+    def self.nouns
+      @nouns ||= new(["名詞"])
+    end
+
+    def self.verbs
+      @verbs ||= new(["動詞"])
+    end
+
+    def self.adjectives
+      @adjectives ||= new(["形容詞"])
+    end
+
+    def self.particles
+      @particles ||= new(["助詞"])
+    end
+
+    def self.auxiliary_verbs
+      @auxiliary_verbs ||= new(["助動詞"])
+    end
+
+    def self.adverbs
+      @adverbs ||= new(["副詞"])
+    end
+
+    def self.proper_nouns
+      @proper_nouns ||= new(["名詞", "固有名詞"])
+    end
+
+    private
+
+    def extract_pos(morpheme_or_pos)
+      if morpheme_or_pos.is_a?(Array)
+        morpheme_or_pos
+      elsif morpheme_or_pos.respond_to?(:part_of_speech)
+        morpheme_or_pos.part_of_speech
+      else
+        raise ArgumentError, "expected an Array or an object responding to #part_of_speech"
+      end
+    end
+
+    def pattern_match?(pattern, pos)
+      pattern.each_with_index.all? do |slot, i|
+        slot.nil? || slot == "*" || slot == pos[i]
+      end
+    end
+  end
+end

data/lib/kabosu/release.rake

@@ -0,0 +1,39 @@
+namespace :release do
+  desc "Bump version. Usage: rake release:bump[1.2.3]"
+  task :bump, [:version] do |_t, args|
+    version = args[:version]
+    abort "Usage: rake release:bump[1.2.3]" unless version&.match?(/\A\d+\.\d+\.\d+\z/)
+
+    # Update lib/kabosu/version.rb
+    version_file = File.expand_path("version.rb", __dir__)
+    content = File.read(version_file)
+    new_content = content.sub(/VERSION = ".*"/, %(VERSION = "#{version}"))
+    File.write(version_file, new_content)
+    puts "Updated #{version_file}"
+
+    # Update ext/kabosu/Cargo.toml
+    cargo_file = File.expand_path("../../ext/kabosu/Cargo.toml", __dir__)
+    content = File.read(cargo_file)
+    new_content = content.sub(/^version = ".*"/, %(version = "#{version}"))
+    File.write(cargo_file, new_content)
+    puts "Updated #{cargo_file}"
+
+    puts "\nVersion bumped to #{version}"
+    puts "Next steps:"
+    puts "  jj commit -m 'Bump version to #{version}'"
+    puts "  jj tag set v#{version}"
+    puts "  jj git push --all"
+  end
+end
+
+desc "Build the gem"
+task :build do
+  sh "gem build kabosu.gemspec"
+end
+
+desc "Build and push the gem to RubyGems"
+task release: :build do
+  gemfile = Dir["kabosu-*.gem"].max_by { |f| File.mtime(f) }
+  abort "No .gem file found" unless gemfile
+  sh "gem push #{gemfile}"
+end

data/lib/kabosu/tasks.rake

@@ -0,0 +1,86 @@
+require "kabosu/dict_manager"
+
+namespace :kabosu do
+  # ── Install ──
+
+  desc "Install the core dictionary (default). VERSION=YYYYMMDD to pin a specific release."
+  task :install do
+    Kabosu::DictManager.new.install("core", version: ENV["VERSION"])
+  end
+
+  namespace :install do
+    desc "Install the small dictionary. VERSION=YYYYMMDD to pin a specific release."
+    task :small do
+      Kabosu::DictManager.new.install("small", version: ENV["VERSION"])
+    end
+
+    desc "Install the core dictionary. VERSION=YYYYMMDD to pin a specific release."
+    task :core do
+      Kabosu::DictManager.new.install("core", version: ENV["VERSION"])
+    end
+
+    desc "Install the full dictionary. VERSION=YYYYMMDD to pin a specific release."
+    task :full do
+      Kabosu::DictManager.new.install("full", version: ENV["VERSION"])
+    end
+  end
+
+  # ── Remove ──
+
+  desc "Remove all installed dictionaries, or a specific one with EDITION=small|core|full and/or VERSION=YYYYMMDD"
+  task :remove do
+    Kabosu::DictManager.new.remove(edition: ENV["EDITION"], version: ENV["VERSION"])
+  end
+
+  namespace :remove do
+    desc "Remove the small dictionary."
+    task :small do
+      Kabosu::DictManager.new.remove(edition: "small", version: ENV["VERSION"])
+    end
+
+    desc "Remove the core dictionary."
+    task :core do
+      Kabosu::DictManager.new.remove(edition: "core", version: ENV["VERSION"])
+    end
+
+    desc "Remove the full dictionary."
+    task :full do
+      Kabosu::DictManager.new.remove(edition: "full", version: ENV["VERSION"])
+    end
+  end
+
+  # ── Info ──
+
+  desc "List installed Sudachi dictionaries"
+  task :list do
+    manager = Kabosu::DictManager.new
+    dicts = manager.installed
+
+    if dicts.empty?
+      puts "No dictionaries installed. Run: rake kabosu:install"
+    else
+      puts "Installed dictionaries (#{manager.dir}):"
+      puts
+      dicts.each do |d|
+        size_mb = (File.size(d[:path]).to_f / 1024 / 1024).round(1)
+        puts "  #{d[:version]} / #{d[:edition].ljust(5)} (#{size_mb} MB)"
+        puts "    #{d[:path]}"
+      end
+    end
+  end
+
+  desc "Show available dictionary versions from GitHub"
+  task :versions do
+    Kabosu::DictManager.new.available_versions.each { |v| puts v }
+  end
+
+  desc "Show the path to the best available dictionary. EDITION=small|core|full to be specific."
+  task :path do
+    begin
+      puts Kabosu::DictManager.new.find(edition: ENV["EDITION"])
+    rescue Kabosu::DictManager::DictNotFound => e
+      $stderr.puts e.message
+      exit 1
+    end
+  end
+end

data/lib/kabosu/version.rb

@@ -0,0 +1,3 @@
+module Kabosu
+  VERSION = "0.6.10"
+end

data/lib/kabosu.rb

@@ -0,0 +1,239 @@
+require_relative "kabosu/version"
+require_relative "kabosu/kabosu"
+require_relative "kabosu/dict_manager"
+require_relative "kabosu/pos_matcher"
+require_relative "kabosu/morpheme_list"
+
+module Kabosu
+  class Error < StandardError; end
+  class ConfigError < Error; end
+  class DictionaryError < Error; end
+  class LookupError < Error; end
+  class TokenizationError < Error; end
+  class SentenceSplitError < Error; end
+
+  class SentenceRange
+    attr_reader :start, :end, :text
+
+    def initialize(start, finish, text)
+      @start = start
+      @end = finish
+      @text = text
+    end
+
+    def to_a
+      [@start, @end, @text]
+    end
+  end
+
+  remove_const(:MODE_A) if const_defined?(:MODE_A, false)
+  remove_const(:MODE_B) if const_defined?(:MODE_B, false)
+  remove_const(:MODE_C) if const_defined?(:MODE_C, false)
+  MODE_A = :a
+  MODE_B = :b
+  MODE_C = :c
+  private_constant :TokenBatch if const_defined?(:TokenBatch, false)
+
+  def self.normalize_mode(mode, param_name: "mode")
+    case mode
+    when :a
+      "A"
+    when :b
+      "B"
+    when :c
+      "C"
+    else
+      raise ArgumentError, "invalid #{param_name} #{mode.inspect}; expected :a, :b, or :c"
+    end
+  end
+  private_class_method :normalize_mode
+
+  # ── Dictionary: keyword API and management ──
+
+  class Dictionary
+    class << self
+      alias_method :_new, :new
+
+      def new(config: nil, system_dict: nil, user_dicts: nil)
+        unless config.nil? || config.is_a?(String)
+          raise ArgumentError, "config must be a String or nil"
+        end
+        unless system_dict.nil? || system_dict.is_a?(String)
+          raise ArgumentError, "system_dict must be a String or nil"
+        end
+        unless user_dicts.nil? || user_dicts.is_a?(Array)
+          raise ArgumentError, "user_dicts must be an Array<String> or nil"
+        end
+        if user_dicts&.any? { !_1.is_a?(String) }
+          raise ArgumentError, "user_dicts must contain only String values"
+        end
+
+        if config.nil? && system_dict.nil?
+          raise ArgumentError, "either config or system_dict is required"
+        end
+        _new(config, system_dict, user_dicts)
+      rescue RuntimeError => e
+        raise map_dictionary_init_error(e, config: config, system_dict: system_dict)
+      end
+
+      def install(edition = "core", version: nil)
+        dict_manager.install(edition, version: version)
+      end
+
+      def path(edition: nil)
+        dict_manager.find(edition: edition)
+      end
+
+      def list
+        dict_manager.installed
+      end
+
+      private
+
+      def map_dictionary_init_error(error, config:, system_dict:)
+        message = error.message
+        if config && system_dict.nil?
+          ConfigError.new(message)
+        elsif message.match?(/config|setting\.json|json/i)
+          ConfigError.new(message)
+        else
+          DictionaryError.new(message)
+        end
+      end
+
+      def dict_manager
+        @dict_manager ||= DictManager.new
+      end
+    end
+
+    alias_method :_create, :create
+    alias_method :_lookup, :lookup
+
+    def create(**options)
+      unknown = options.keys - %i[mode fields debug projection]
+      raise ArgumentError, "unknown keyword(s): #{unknown.join(', ')}" unless unknown.empty?
+
+      mode = options.fetch(:mode, MODE_C)
+      fields = options.fetch(:fields, nil)
+      debug = options.fetch(:debug, false)
+      projection = options.fetch(:projection, nil)
+
+      unless fields.nil? || fields.is_a?(Array)
+        raise ArgumentError, "fields must be an Array<String|Symbol> or nil"
+      end
+      if fields&.any? { !(_1.is_a?(String) || _1.is_a?(Symbol)) }
+        raise ArgumentError, "fields must contain only String or Symbol values"
+      end
+      unless debug == true || debug == false
+        raise ArgumentError, "debug must be true or false"
+      end
+
+      unless projection.nil?
+        raise NotImplementedError, "projection is not supported yet"
+      end
+
+      mode_str = Kabosu.__send__(:normalize_mode, mode)
+      _create(mode_str, fields, debug)
+    end
+
+    def lookup(text)
+      unless text.is_a?(String)
+        raise ArgumentError, "text must be a String"
+      end
+      MorphemeList.new(_lookup(text))
+    rescue RuntimeError => e
+      raise LookupError.new(e.message), cause: e
+    end
+  end
+
+  # ── Tokenizer: wrap output in MorphemeList ──
+
+  class Tokenizer
+    alias_method :_tokenize, :tokenize
+
+    def tokenize(text)
+      unless text.is_a?(String)
+        raise ArgumentError, "text must be a String"
+      end
+
+      batch = _tokenize(text)
+      cost = batch.respond_to?(:internal_cost) ? batch.internal_cost : nil
+      MorphemeList.new(batch, internal_cost: cost)
+    rescue RuntimeError => e
+      raise TokenizationError.new(e.message), cause: e
+    end
+  end
+
+  class Morpheme
+    alias_method :_split, :split
+
+    def split(mode: MODE_C, add_single: true)
+      unless add_single == true || add_single == false
+        raise ArgumentError, "add_single must be true or false"
+      end
+      mode_str = Kabosu.__send__(:normalize_mode, mode)
+      MorphemeList.new(_split(mode_str, nil, add_single))
+    rescue RuntimeError => e
+      raise TokenizationError.new(e.message), cause: e
+    end
+  end
+
+  def self.split_sentences(text, limit: nil, with_checker: false, ranges: false, dictionary: nil)
+    unless text.is_a?(String)
+      raise ArgumentError, "text must be a String"
+    end
+    unless limit.nil? || limit.is_a?(Integer)
+      raise ArgumentError, "limit must be an Integer or nil"
+    end
+    if limit && limit < 1
+      raise ArgumentError, "limit must be greater than 0"
+    end
+    unless with_checker == true || with_checker == false
+      raise ArgumentError, "with_checker must be true or false"
+    end
+    unless ranges == true || ranges == false
+      raise ArgumentError, "ranges must be true or false"
+    end
+    unless dictionary.nil? || dictionary.is_a?(String)
+      raise ArgumentError, "dictionary must be a String path or nil"
+    end
+
+    dict_path = nil
+    if with_checker
+      dict_path = dictionary || Dictionary.path
+    end
+
+    if ranges
+      _split_sentences_with_ranges(text, limit, dict_path).map do |(start, finish, sentence)|
+        SentenceRange.new(start, finish, sentence)
+      end
+    else
+      _split_sentences(text, limit, dict_path)
+    end
+  rescue RuntimeError => e
+    raise SentenceSplitError.new(e.message), cause: e
+  end
+
+  # ── Convenience tokenization ──
+
+  # Tokenize text using an explicitly provided tokenizer.
+  #
+  #   dict = Kabosu::Dictionary.new(system_dict: Kabosu::Dictionary.path)
+  #   tok = dict.create(mode: :a)
+  #   Kabosu.tokenize("東京都に住んでいる", tokenizer: tok)
+  #
+  def self.tokenize(text, tokenizer:)
+    unless text.is_a?(String)
+      raise ArgumentError, "text must be a String"
+    end
+    unless tokenizer.is_a?(Tokenizer)
+      raise ArgumentError, "tokenizer must be a Kabosu::Tokenizer"
+    end
+
+    batch = tokenizer.__send__(:_tokenize, text)
+    cost = batch.respond_to?(:internal_cost) ? batch.internal_cost : nil
+    MorphemeList.new(batch, internal_cost: cost)
+  rescue RuntimeError => e
+    raise TokenizationError.new(e.message), cause: e
+  end
+end