kabosu 0.1.0

data/lib/kabosu/dict_manager.rb

@@ -0,0 +1,239 @@
+require "net/http"
+require "uri"
+require "fileutils"
+require "json"
+require "open3"
+
+module Kabosu
+  class DictManager
+    EDITIONS = %w[small core full].freeze
+    EDITION_PRIORITY = %w[full core small].freeze
+    GITHUB_REPO = "WorksApplications/SudachiDict"
+    GITHUB_API = "https://api.github.com"
+
+    class DictNotFound < StandardError; end
+    class DownloadError < StandardError; end
+
+    # Default storage directory: ~/.kabosu/dict/
+    def self.default_dir
+      File.join(Dir.home, ".kabosu", "dict")
+    end
+
+    def initialize(dir: self.class.default_dir)
+      @dir = dir
+    end
+
+    attr_reader :dir
+
+    # ── Install ──
+
+    # Download and extract a dictionary edition.
+    #
+    #   manager.install("small")
+    #   manager.install("core", version: "20260116")
+    #
+    def install(edition = "core", version: nil)
+      edition = validate_edition(edition)
+      version ||= latest_version
+
+      dest_dir = File.join(@dir, "sudachi-dictionary-#{version}")
+      dic_path = File.join(dest_dir, "system_#{edition}.dic")
+
+      if File.exist?(dic_path)
+        $stderr.puts "Already installed: #{dic_path}"
+        return dic_path
+      end
+
+      url = release_asset_url(version, edition)
+      zip_path = File.join(@dir, "sudachi-dictionary-#{version}-#{edition}.zip")
+
+      FileUtils.mkdir_p(@dir)
+      download(url, zip_path)
+      extract(zip_path, @dir)
+      FileUtils.rm_f(zip_path)
+
+      unless File.exist?(dic_path)
+        raise DownloadError, "Expected #{dic_path} after extraction, but file not found"
+      end
+
+      $stderr.puts "Installed: #{dic_path}"
+      dic_path
+    end
+
+    # ── Discovery ──
+
+    # List all installed dictionaries.
+    # Returns an array of hashes: { version:, edition:, path: }
+    def installed
+      results = []
+      return results unless Dir.exist?(@dir)
+
+      Dir.glob(File.join(@dir, "sudachi-dictionary-*")).sort.reverse.each do |version_dir|
+        next unless File.directory?(version_dir)
+
+        version = File.basename(version_dir).sub("sudachi-dictionary-", "")
+        EDITIONS.each do |edition|
+          dic = File.join(version_dir, "system_#{edition}.dic")
+          next unless File.exist?(dic)
+
+          results << { version: version, edition: edition, path: dic }
+        end
+      end
+
+      results
+    end
+
+    # Find the best available dictionary path.
+    # Prefers: latest version, then full > core > small.
+    def find(edition: nil)
+      candidates = installed
+      raise DictNotFound, "No dictionaries installed. Run: rake kabosu:install" if candidates.empty?
+
+      if edition
+        edition = validate_edition(edition)
+        match = candidates.find { |d| d[:edition] == edition }
+        raise DictNotFound, "No #{edition} dictionary installed" unless match
+        return match[:path]
+      end
+
+      # Group by version (already sorted newest-first), pick best edition
+      by_version = candidates.group_by { |d| d[:version] }
+      latest_version_dicts = by_version.values.first
+
+      best = EDITION_PRIORITY.each do |ed|
+        found = latest_version_dicts.find { |d| d[:edition] == ed }
+        break found if found
+      end
+
+      best.is_a?(Hash) ? best[:path] : latest_version_dicts.first[:path]
+    end
+
+    # ── Remove ──
+
+    # Remove a specific dictionary edition, or an entire version.
+    def remove(edition: nil, version: nil)
+      targets = installed
+      targets = targets.select { |d| d[:version] == version } if version
+      targets = targets.select { |d| d[:edition] == edition } if edition
+
+      raise DictNotFound, "No matching dictionary found" if targets.empty?
+
+      targets.each do |d|
+        FileUtils.rm_f(d[:path])
+        $stderr.puts "Removed: #{d[:path]}"
+
+        # Clean up empty version directories
+        version_dir = File.dirname(d[:path])
+        dics_remaining = Dir.glob(File.join(version_dir, "system_*.dic"))
+        if dics_remaining.empty?
+          FileUtils.rm_rf(version_dir)
+          $stderr.puts "Removed empty directory: #{version_dir}"
+        end
+      end
+    end
+
+    # ── Version resolution ──
+
+    # Fetch the latest release tag from GitHub.
+    def latest_version
+      uri = URI("#{GITHUB_API}/repos/#{GITHUB_REPO}/releases/latest")
+      response = http_get(uri, headers: { "Accept" => "application/json" })
+      data = JSON.parse(response.body)
+      tag = data["tag_name"]
+      # Tags are like "v20260116" — strip the "v" prefix
+      tag.sub(/\Av/, "")
+    end
+
+    # List available versions from GitHub releases.
+    def available_versions
+      uri = URI("#{GITHUB_API}/repos/#{GITHUB_REPO}/releases")
+      response = http_get(uri, headers: { "Accept" => "application/json" })
+      JSON.parse(response.body).map { |r| r["tag_name"].sub(/\Av/, "") }
+    end
+
+    private
+
+    def validate_edition(edition)
+      edition = edition.to_s.downcase
+      unless EDITIONS.include?(edition)
+        raise ArgumentError, "Unknown edition '#{edition}'. Must be one of: #{EDITIONS.join(", ")}"
+      end
+      edition
+    end
+
+    def release_asset_url(version, edition)
+      "https://github.com/#{GITHUB_REPO}/releases/download/v#{version}/sudachi-dictionary-#{version}-#{edition}.zip"
+    end
+
+    def download(url, dest)
+      $stderr.puts "Downloading #{url}..."
+      uri = resolve_redirects(URI(url))
+
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
+        http.request(Net::HTTP::Get.new(uri)) do |response|
+          unless response.is_a?(Net::HTTPSuccess)
+            raise DownloadError, "Failed to download: #{response.code} #{response.message}"
+          end
+
+          total = response["Content-Length"]&.to_i
+          written = 0
+
+          File.open(dest, "wb") do |f|
+            response.read_body do |chunk|
+              f.write(chunk)
+              written += chunk.bytesize
+              if total && total > 0
+                pct = (written * 100 / total).clamp(0, 100)
+                $stderr.print "\r  #{(written.to_f / 1024 / 1024).round(1)} / #{(total.to_f / 1024 / 1024).round(1)} MB (#{pct}%)"
+              end
+            end
+          end
+
+          $stderr.puts "\r  #{(written.to_f / 1024 / 1024).round(1)} MB downloaded"
+        end
+      end
+    end
+
+    def resolve_redirects(uri, limit: 5)
+      raise DownloadError, "Too many redirects" if limit == 0
+
+      Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https") do |http|
+        response = http.request(Net::HTTP::Head.new(uri))
+        case response
+        when Net::HTTPRedirection
+          resolve_redirects(URI(response["location"]), limit: limit - 1)
+        else
+          uri
+        end
+      end
+    end
+
+    def http_get(uri, headers: {}, redirect_limit: 5)
+      raise DownloadError, "Too many redirects" if redirect_limit == 0
+
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = (uri.scheme == "https")
+
+      request = Net::HTTP::Get.new(uri)
+      headers.each { |k, v| request[k] = v }
+
+      response = http.request(request)
+
+      case response
+      when Net::HTTPRedirection
+        http_get(URI(response["location"]), headers: headers, redirect_limit: redirect_limit - 1)
+      else
+        response
+      end
+    end
+
+    def extract(zip_path, dest_dir)
+      $stderr.puts "Extracting..."
+      # Use system unzip — available everywhere, handles large files well
+      _stdout, stderr, status = Open3.capture3("unzip", "-o", zip_path, "-d", dest_dir)
+      unless status.success?
+        raise DownloadError, "Failed to extract #{zip_path}: #{stderr}"
+      end
+    end
+  end
+end

data/lib/kabosu/morpheme_list.rb

@@ -0,0 +1,97 @@
+module Kabosu
+  class MorphemeList
+    include Enumerable
+
+    attr_accessor :internal_cost
+
+    def initialize(morphemes, internal_cost: nil)
+      @morphemes = morphemes
+      @internal_cost = internal_cost
+    end
+
+    def each(&block)
+      @morphemes.each(&block)
+    end
+
+    def [](index)
+      @morphemes[index]
+    end
+
+    def first(n = nil)
+      n ? @morphemes.first(n) : @morphemes.first
+    end
+
+    def last(n = nil)
+      n ? @morphemes.last(n) : @morphemes.last
+    end
+
+    def size
+      @morphemes.size
+    end
+
+    def surfaces
+      map(&:surface)
+    end
+
+    def readings
+      map(&:reading_form)
+    end
+
+    def dictionary_forms
+      map(&:dictionary_form)
+    end
+
+    def normalized_forms
+      map(&:normalized_form)
+    end
+
+    def total_costs
+      map(&:total_cost)
+    end
+
+    def synonym_group_ids
+      map(&:synonym_group_ids)
+    end
+
+    # Joins all surfaces back into the original text (no spaces, for Japanese text).
+    def to_text
+      surfaces.join
+    end
+
+    # Filter morphemes by POS. Accepts a PosMatcher or an array pattern.
+    # Returns a new MorphemeList with only matching morphemes.
+    #
+    #   list.select_pos(Kabosu::PosMatcher.nouns)
+    #   list.select_pos(["名詞", "固有名詞"])
+    #
+    def select_pos(matcher_or_pattern)
+      matcher = coerce_to_matcher(matcher_or_pattern)
+      self.class.new(matcher.filter(@morphemes))
+    end
+
+    # Inverse of select_pos. Returns a new MorphemeList excluding matching morphemes.
+    def reject_pos(matcher_or_pattern)
+      matcher = coerce_to_matcher(matcher_or_pattern)
+      self.class.new(matcher.reject(@morphemes))
+    end
+
+    def inspect
+      base = "#<Kabosu::MorphemeList (#{size} morphemes)"
+      base += " cost=#{@internal_cost}" if @internal_cost
+      base + ": #{surfaces.join(" | ")}>"
+    end
+
+    private
+
+    def coerce_to_matcher(matcher_or_pattern)
+      case matcher_or_pattern
+      when PosMatcher
+        matcher_or_pattern
+      when Array
+        PosMatcher.new(matcher_or_pattern)
+      else
+        raise ArgumentError, "expected a PosMatcher or an Array pattern, got #{matcher_or_pattern.class}"
+      end
+    end
+  end
+end

data/lib/kabosu/pos_matcher.rb

@@ -0,0 +1,119 @@
+module Kabosu
+  class PosMatcher
+    # Build a matcher from POS patterns or a block.
+    #
+    #   # From a block
+    #   PosMatcher.new { |pos| pos[0] == "名詞" }
+    #
+    #   # Single pattern (array of strings; "*" or nil = match anything)
+    #   PosMatcher.new(["名詞", "固有名詞", "*", "*"])
+    #
+    #   # Multiple patterns (matches if any pattern matches)
+    #   PosMatcher.new(["名詞", "固有名詞"], ["動詞", "*", "*", "*"])
+    #
+    def initialize(*patterns, &block)
+      if block
+        raise ArgumentError, "cannot supply both patterns and a block" unless patterns.empty?
+
+        @proc = block
+      elsif patterns.empty?
+        raise ArgumentError, "must supply at least one pattern or a block"
+      else
+        @patterns = patterns.map(&:freeze).freeze
+      end
+
+      freeze
+    end
+
+    # Returns true if the morpheme (or raw POS array) matches this matcher.
+    def match?(morpheme_or_pos)
+      pos = extract_pos(morpheme_or_pos)
+
+      if @proc
+        @proc.call(pos)
+      else
+        @patterns.any? { |pattern| pattern_match?(pattern, pos) }
+      end
+    end
+
+    # Return matching morphemes as an Array.
+    def filter(morphemes)
+      morphemes.select { |m| match?(m) }
+    end
+
+    # Return non-matching morphemes as an Array.
+    def reject(morphemes)
+      morphemes.reject { |m| match?(m) }
+    end
+
+    # Union: matches if either matcher matches.
+    def |(other)
+      a, b = self, other
+      PosMatcher.new { |pos| a.match?(pos) || b.match?(pos) }
+    end
+
+    # Intersection: matches if both matchers match.
+    def &(other)
+      a, b = self, other
+      PosMatcher.new { |pos| a.match?(pos) && b.match?(pos) }
+    end
+
+    # Difference: matches self but not other.
+    def -(other)
+      difference(other)
+    end
+
+    def difference(other)
+      a, b = self, other
+      PosMatcher.new { |pos| a.match?(pos) && !b.match?(pos) }
+    end
+
+    # ── Pre-built matchers ──
+
+    def self.nouns
+      @nouns ||= new(["名詞"])
+    end
+
+    def self.verbs
+      @verbs ||= new(["動詞"])
+    end
+
+    def self.adjectives
+      @adjectives ||= new(["形容詞"])
+    end
+
+    def self.particles
+      @particles ||= new(["助詞"])
+    end
+
+    def self.auxiliary_verbs
+      @auxiliary_verbs ||= new(["助動詞"])
+    end
+
+    def self.adverbs
+      @adverbs ||= new(["副詞"])
+    end
+
+    def self.proper_nouns
+      @proper_nouns ||= new(["名詞", "固有名詞"])
+    end
+
+    private
+
+    def extract_pos(morpheme_or_pos)
+      if morpheme_or_pos.is_a?(Array)
+        morpheme_or_pos
+      elsif morpheme_or_pos.respond_to?(:part_of_speech)
+        morpheme_or_pos.part_of_speech
+      else
+        raise ArgumentError, "expected an Array or an object responding to #part_of_speech"
+      end
+    end
+
+    def pattern_match?(pattern, pos)
+      pattern.each_with_index.all? do |slot, i|
+        slot.nil? || slot == "*" || slot == pos[i]
+      end
+    end
+  end
+end

data/lib/kabosu/tasks.rake

@@ -0,0 +1,86 @@
+require "kabosu/dict_manager"
+
+namespace :kabosu do
+  # ── Install ──
+
+  desc "Install the core dictionary (default). VERSION=YYYYMMDD to pin a specific release."
+  task :install do
+    Kabosu::DictManager.new.install("core", version: ENV["VERSION"])
+  end
+
+  namespace :install do
+    desc "Install the small dictionary. VERSION=YYYYMMDD to pin a specific release."
+    task :small do
+      Kabosu::DictManager.new.install("small", version: ENV["VERSION"])
+    end
+
+    desc "Install the core dictionary. VERSION=YYYYMMDD to pin a specific release."
+    task :core do
+      Kabosu::DictManager.new.install("core", version: ENV["VERSION"])
+    end
+
+    desc "Install the full dictionary. VERSION=YYYYMMDD to pin a specific release."
+    task :full do
+      Kabosu::DictManager.new.install("full", version: ENV["VERSION"])
+    end
+  end
+
+  # ── Remove ──
+
+  desc "Remove all installed dictionaries, or a specific one with EDITION=small|core|full and/or VERSION=YYYYMMDD"
+  task :remove do
+    Kabosu::DictManager.new.remove(edition: ENV["EDITION"], version: ENV["VERSION"])
+  end
+
+  namespace :remove do
+    desc "Remove the small dictionary."
+    task :small do
+      Kabosu::DictManager.new.remove(edition: "small", version: ENV["VERSION"])
+    end
+
+    desc "Remove the core dictionary."
+    task :core do
+      Kabosu::DictManager.new.remove(edition: "core", version: ENV["VERSION"])
+    end
+
+    desc "Remove the full dictionary."
+    task :full do
+      Kabosu::DictManager.new.remove(edition: "full", version: ENV["VERSION"])
+    end
+  end
+
+  # ── Info ──
+
+  desc "List installed Sudachi dictionaries"
+  task :list do
+    manager = Kabosu::DictManager.new
+    dicts = manager.installed
+
+    if dicts.empty?
+      puts "No dictionaries installed. Run: rake kabosu:install"
+    else
+      puts "Installed dictionaries (#{manager.dir}):"
+      puts
+      dicts.each do |d|
+        size_mb = (File.size(d[:path]).to_f / 1024 / 1024).round(1)
+        puts "  #{d[:version]} / #{d[:edition].ljust(5)} (#{size_mb} MB)"
+        puts "    #{d[:path]}"
+      end
+    end
+  end
+
+  desc "Show available dictionary versions from GitHub"
+  task :versions do
+    Kabosu::DictManager.new.available_versions.each { |v| puts v }
+  end
+
+  desc "Show the path to the best available dictionary. EDITION=small|core|full to be specific."
+  task :path do
+    begin
+      puts Kabosu::DictManager.new.find(edition: ENV["EDITION"])
+    rescue Kabosu::DictManager::DictNotFound => e
+      $stderr.puts e.message
+      exit 1
+    end
+  end
+end

data/lib/kabosu/version.rb

@@ -0,0 +1,3 @@
+module Kabosu
+  VERSION = "0.1.0"
+end

data/lib/kabosu.rb

@@ -0,0 +1,143 @@
+require_relative "kabosu/version"
+require_relative "kabosu/kabosu"
+require_relative "kabosu/dict_manager"
+require_relative "kabosu/pos_matcher"
+require_relative "kabosu/morpheme_list"
+
+module Kabosu
+  # ── Dictionary.new: keyword API + auto-discovery ──
+
+  class Dictionary
+    class << self
+      alias_method :_new, :new
+
+      def new(config: nil, dict: nil)
+        dict ||= Kabosu.dict_path
+        _new(config, dict)
+      end
+    end
+  end
+
+  # ── Tokenizer: wrap output in MorphemeList ──
+
+  class Tokenizer
+    alias_method :_tokenize, :tokenize
+
+    def tokenize(text)
+      morphemes = _tokenize(text)
+      cost = respond_to?(:internal_cost) ? internal_cost : nil
+      MorphemeList.new(morphemes, internal_cost: cost)
+    end
+
+    if method_defined?(:tokenize_sentences) || instance_methods(false).include?(:tokenize_sentences)
+      alias_method :_tokenize_sentences, :tokenize_sentences
+
+      def tokenize_sentences(text)
+        _tokenize_sentences(text).map { |morphemes| MorphemeList.new(morphemes) }
+      end
+    end
+  end
+
+  # ── StatefulTokenizer: wrap output in MorphemeList ──
+
+  class StatefulTokenizer
+    alias_method :_tokenize, :tokenize
+
+    def tokenize(text)
+      morphemes = _tokenize(text)
+      cost = respond_to?(:internal_cost) ? internal_cost : nil
+      MorphemeList.new(morphemes, internal_cost: cost)
+    end
+
+    if method_defined?(:tokenize_sentences) || instance_methods(false).include?(:tokenize_sentences)
+      alias_method :_tokenize_sentences, :tokenize_sentences
+
+      def tokenize_sentences(text)
+        _tokenize_sentences(text).map { |morphemes| MorphemeList.new(morphemes) }
+      end
+    end
+  end
+
+  # ── Dictionary management ──
+
+  def self.dict_manager
+    @dict_manager ||= DictManager.new
+  end
+
+  def self.install_dictionary(edition = "core", version: nil)
+    dict_manager.install(edition, version: version)
+  end
+
+  def self.dict_path(edition: nil)
+    dict_manager.find(edition: edition)
+  end
+
+  def self.dictionaries
+    dict_manager.installed
+  end
+
+  # ── Convenience tokenization ──
+
+  # Tokenize text using the best available dictionary.
+  #
+  #   Kabosu.tokenize("東京都に住んでいる")
+  #   Kabosu.tokenize("東京都に住んでいる", mode: "A")
+  #   Kabosu.tokenize("東京都に住んでいる", edition: "small")
+  #
+  def self.tokenize(text, mode: "C", edition: nil)
+    tokenizer = cached_tokenizer(edition: edition, mode: mode)
+    tokenizer.tokenize(text)
+  end
+
+  # Tokenize text into sentences. Returns an array of MorphemeList objects,
+  # one per sentence.
+  #
+  #   Kabosu.tokenize_sentences("東京都に住んでいる。大阪も好きだ。")
+  #
+  def self.tokenize_sentences(text, mode: "C", edition: nil)
+    tokenizer = cached_tokenizer(edition: edition, mode: mode)
+    if tokenizer.respond_to?(:_tokenize_sentences)
+      tokenizer.tokenize_sentences(text)
+    elsif tokenizer.respond_to?(:tokenize_sentences)
+      # Fallback if the alias wasn't set up (Rust method not yet available)
+      tokenizer.tokenize_sentences(text).map { |morphemes| MorphemeList.new(morphemes) }
+    else
+      # Final fallback: treat entire text as a single sentence
+      [tokenizer.tokenize(text)]
+    end
+  end
+
+  # Convenience factory for PosMatcher.
+  #
+  #   nouns = Kabosu.pos_matcher("名詞")
+  #   verbs = Kabosu.pos_matcher(["動詞", "*", "*", "*"])
+  #   custom = Kabosu.pos_matcher { |pos| pos[0] == "動詞" }
+  #
+  def self.pos_matcher(*patterns, &block)
+    if block
+      PosMatcher.new(&block)
+    elsif patterns.length == 1 && patterns[0].is_a?(String)
+      # Shorthand: single string is treated as first POS element
+      PosMatcher.new([patterns[0]])
+    else
+      PosMatcher.new(*patterns)
+    end
+  end
+
+  # Create a StatefulTokenizer for efficient batch processing.
+  # Reuses internal buffers across calls.
+  #
+  #   tok = Kabosu.batch_tokenizer(mode: "A")
+  #   texts.each { |t| results << tok.tokenize(t) }
+  #
+  def self.batch_tokenizer(mode: "C", edition: nil)
+    Dictionary.new(dict: dict_path(edition: edition)).create_stateful(mode)
+  end
+
+  # @api private
+  def self.cached_tokenizer(edition:, mode:)
+    @tokenizers ||= {}
+    @tokenizers[[edition, mode]] ||= Dictionary.new(dict: dict_path(edition: edition)).create(mode)
+  end
+  private_class_method :cached_tokenizer
+end