kotoshu 0.3.0

data/Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]

data/SECURITY.md

@@ -0,0 +1,93 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+Email security@kotoshu.org with details and a reproduction. You will
+receive an acknowledgement within 72 hours. Please do not file public
+issues for security reports.
+
+## Supported Versions
+
+The latest 0.x minor is supported. Older minors are not maintained.
+
+## Threat Model
+
+Kotoshu downloads dictionaries, frequency lists, and (optionally)
+embedding models from public GitHub repositories on first use:
+
+| Resource | Source repo | Cached at |
+|---|---|---|
+| Spelling dictionaries | `kotoshu/dictionaries` | `~/.kotoshu/languages/{code}/` |
+| Kelly frequency lists | `kotoshu/frequency-list-kelly` | `~/.kotoshu/frequency-lists/{code}/` |
+| FastText / ONNX models | `kotoshu/models-fasttext-onnx` | `~/.kotoshu/models/{code}/` |
+
+Downloads flow over HTTPS from `raw.githubusercontent.com`. The threat
+model assumes GitHub serves the bytes the repository owner committed,
+and treats anyone with push access to those repos as trusted.
+
+## Integrity Verification
+
+Each content repo may ship a `manifest.json` at its root listing every
+file with its SHA-256 hash, size, and language/type tags. When a
+manifest is present, every download is verified against it:
+
+1. The manifest is fetched once per cache session.
+2. Each downloaded file's SHA-256 is computed locally and compared to
+   the manifest entry.
+3. On mismatch, `Kotoshu::IntegrityError` is raised with the expected
+   and actual hashes. The download is rejected and the cache is left
+   untouched.
+4. Every verification outcome (verified / unverified / mismatch /
+   missing) is appended to `~/.kotoshu/audit.log` as one JSON object
+   per line.
+
+### Graceful Degradation
+
+When a manifest is **absent** (HTTP 404), verification silently
+downgrades to `"unverified"` status and the download proceeds. This
+preserves forward compatibility with repos that have not yet shipped a
+manifest. The audit log records the difference.
+
+### Strict Mode
+
+`Kotoshu.spellchecker_for(lang, strict: true)` (and the CLI's
+`--strict` flag) re-raise on any optional-resource failure — including
+integrity mismatches on frequency data — instead of silently
+degrading. Spelling-dictionary integrity is always enforced.
+
+## Cache Layout
+
+The cache is written under `$KOTOSHU_HOME` (default `~/.kotoshu/`).
+Files are created with the user's default umask. Cache contents are
+not encrypted at rest.
+
+## Audit Log
+
+`~/.kotoshu/audit.log` is append-only, JSON-per-line, and never
+auto-rotated. Operators in multi-user environments should rotate it
+via logrotate or equivalent. To inspect:
+
+[source,bash]
+----
+cat ~/.kotoshu/audit.log | jq .
+----
+
+To clear the audit log:
+
+[source,ruby]
+----
+Kotoshu::Integrity::AuditLog.new(path: "#{ENV['HOME']}/.kotoshu/audit.log").clear!
+----
+
+## Network Egress
+
+`offline: true` (or `KOTOSHU_OFFLINE=1` or `--offline`) disables all
+network egress and only reads from the on-disk cache. If a required
+resource is not cached, the call raises `Kotoshu::ResourceNotCachedError`
+(CLI exits 3). Use `kotoshu fetch LANGUAGE` to pre-warm the cache in
+environments without outbound network access.
+
+## Scope
+
+This policy covers the kotoshu gem itself. Vulnerabilities in
+dependencies (Thor, suika, onnxruntime) should be reported upstream.

data/examples/01_basic_word_checking.rb

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Example 1: Basic Word Checking
+#
+# This example demonstrates the simplest way to use Kotoshu
+# to check if words are spelled correctly.
+
+require_relative "../lib/kotoshu"
+
+puts "=== Example 1: Basic Word Checking ==="
+puts
+
+# Check if words are correct
+puts "Is 'hello' correct? #{Kotoshu.correct?("hello")}"
+puts "Is 'world' correct? #{Kotoshu.correct?("world")}"
+puts "Is 'helo' correct? #{Kotoshu.correct?("helo")}"
+puts "Is 'Kotoshu' correct? #{Kotoshu.correct?("Kotoshu")}"
+puts
+
+# Get suggestions for misspelled words
+puts "Suggestions for 'helo':"
+suggestions = Kotoshu.suggest("helo")
+puts suggestions.to_words.join(", ")
+puts
+
+puts "Suggestions for 'wrold':"
+suggestions = Kotoshu.suggest("wrold")
+puts suggestions.to_words.join(", ")
+puts
+
+# Check multiple words
+words = %w[hello world test helo wrold]
+puts "Checking multiple words:"
+words.each do |word|
+  status = Kotoshu.correct?(word) ? "✓" : "✗"
+  puts "  #{status} #{word}"
+end

data/examples/02_text_document_checking.rb

@@ -0,0 +1,77 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Example 2: Text and Document Checking
+#
+# This example demonstrates how to check paragraphs and documents
+# for spelling errors and get detailed results.
+
+require_relative "../lib/kotoshu"
+
+puts "=== Example 2: Text and Document Checking ==="
+puts
+
+# Check a paragraph of text
+text = <<~TEXT
+  Hello wrold!
+
+  This is a test document with some misspelled words.
+  We want to see if the spellchcker can find them al.
+
+  Teh quick brown fox jumps over the lazy dog.
+  Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+TEXT
+
+puts "Checking text:"
+puts "-" * 40
+result = Kotoshu.check(text)
+
+puts result
+unless result.success?
+  puts
+  puts "Errors found:"
+  result.each_error do |error|
+    suggestions_str = if error.has_suggestions?
+                        " (did you mean #{error.top_suggestions(3).join(", ")}?)"
+                      else
+                        ""
+                      end
+    puts "  • #{error.word}#{suggestions_str}"
+  end
+end
+
+puts
+puts "=" * 40
+puts
+
+# Check a file
+file_path = "spec/fixtures/documents/with_errors.txt"
+if File.exist?(file_path)
+  puts "Checking file: #{file_path}"
+  puts "-" * 40
+
+  file_result = Kotoshu.check_file(file_path)
+
+  if file_result.success?
+    puts "✓ No errors found (#{file_result.word_count} words checked)"
+  else
+    puts "✗ #{file_result.error_count} error(s) found:"
+    puts
+    file_result.each_unique_error do |word, errors|
+      puts "  • #{word} (appears #{errors.size}x)"
+      first_error = errors.first
+      puts "    Suggestions: #{first_error.top_suggestions(3).join(", ")}" if first_error.has_suggestions?
+    end
+  end
+end
+
+puts
+puts "=" * 40
+puts
+
+# Document result statistics
+puts "Document Statistics:"
+puts "  Word count: #{result.word_count}"
+puts "  Error count: #{result.error_count}"
+puts "  Unique errors: #{result.unique_error_count}"
+puts "  Error summary: #{result.error_summary.inspect}"

data/examples/03_dictionary_backends.rb

@@ -0,0 +1,137 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Example 3: Using Different Dictionary Backends
+#
+# This example demonstrates how to use different dictionary backends
+# including UnixWords, PlainText, Custom, and Hunspell.
+
+require_relative "../lib/kotoshu"
+
+puts "=== Example 3: Dictionary Backends ==="
+puts
+
+# Example 1: UnixWords dictionary
+puts "1. UnixWords Dictionary (System Dictionary)"
+puts "-" * 40
+
+unix_dict = Kotoshu::Dictionary::UnixWords.detect(language_code: "en-US")
+if unix_dict
+  puts "Loaded: #{unix_dict.path}"
+  puts "Words: #{unix_dict.size}"
+  puts "Has 'hello': #{unix_dict.lookup?("hello")}"
+  puts "Has 'Kotoshu': #{unix_dict.lookup?("Kotoshu")}"
+  suggestions = unix_dict.suggest("helo", max_suggestions: 5)
+  puts "Suggestions for 'helo': #{suggestions.join(", ")}"
+else
+  puts "No system dictionary found"
+end
+
+puts
+puts "=" * 40
+puts
+
+# Example 2: PlainText dictionary
+puts "2. PlainText Dictionary"
+puts "-" * 40
+
+plain_dict = Kotoshu::Dictionary::PlainText.from_words(
+  %w[hello world kotoshu ruby spellchecker],
+  language_code: "en"
+)
+
+puts "Created dictionary with #{plain_dict.size} words"
+puts "Has 'hello': #{plain_dict.lookup?("hello")}"
+puts "Has 'ruby': #{plain_dict.lookup?("ruby")}"
+puts "Has 'python': #{plain_dict.lookup?("python")}"
+
+# Add a word dynamically
+plain_dict.add_word("python")
+puts "After adding 'python': #{plain_dict.lookup?("python")}"
+plain_dict.add_word("Kotoshu")
+puts "After adding 'Kotoshu': #{plain_dict.lookup?("Kotoshu")}"
+
+puts
+puts "=" * 40
+puts
+
+# Example 3: Custom dictionary
+puts "3. Custom Dictionary (In-Memory)"
+puts "-" * 40
+
+custom_dict = Kotoshu::Dictionary::Custom.new(
+  words: %w[Kotoshu spellchecker ruby],
+  language_code: "en"
+)
+
+puts "Created custom dictionary"
+puts "Words: #{custom_dict.words.inspect}"
+puts "Size: #{custom_dict.size}"
+puts "Has 'Kotoshu': #{custom_dict.lookup?("Kotoshu")}"
+
+# Merge with another array
+custom_dict.merge(%w[gem library code])
+puts "After merging: #{custom_dict.words.inspect}"
+
+puts
+puts "=" * 40
+puts
+
+# Example 4: Hunspell dictionary (if available)
+puts "4. Hunspell Dictionary"
+puts "-" * 40
+
+hunspell_dic = "dictionaries/hunspell/test/en_US_test.dic"
+hunspell_aff = "dictionaries/hunspell/test/en_US_test.aff"
+
+if File.exist?(hunspell_dic) && File.exist?(hunspell_aff)
+  hunspell_dict = Kotoshu::Dictionary::Hunspell.new(
+    dic_path: hunspell_dic,
+    aff_path: hunspell_aff,
+    language_code: "en-US"
+  )
+
+  puts "Loaded Hunspell dictionary"
+  puts "Words: #{hunspell_dict.size}"
+  puts "Has 'hello': #{hunspell_dict.lookup?("hello")}"
+  puts "Has 'hello' (case-insensitive): #{hunspell_dict.lookup?("HELLO")}"
+  puts "Has 'runs': #{hunspell_dict.lookup?("runs")}"
+  puts "Has 'running': #{hunspell_dict.lookup?("running")}"
+
+  # Show word variants using affix rules
+  puts "\nWord variants for 'run':"
+  variants = hunspell_dict.word_variants("run")
+  puts "  #{variants.inspect}"
+else
+  puts "Hunspell test dictionary not found at:"
+  puts "  #{hunspell_dic}"
+  puts "  #{hunspell_aff}"
+end
+
+puts
+puts "=" * 40
+puts
+
+# Example 5: CSpell dictionary
+puts "5. CSpell Dictionary (Trie-based)"
+puts "-" * 40
+
+cspell_dict = Kotoshu::Dictionary::CSpell.from_words(
+  %w[hello world kotoshu ruby gem],
+  language_code: "en"
+)
+
+puts "Created CSpell dictionary with trie"
+puts "Words: #{cspell_dict.words.inspect}"
+puts "Size: #{cspell_dict.size}"
+puts "Has 'hello': #{cspell_dict.lookup?("hello")}"
+puts "Has prefix 'hel': #{cspell_dict.has_prefix?("hel")}"
+puts "Words with prefix 'hel': #{cspell_dict.words_with_prefix("hel").inspect}"
+
+# Convert to trie
+trie = cspell_dict.trie
+puts "\nTrie structure:"
+puts "  Has 'hello': #{trie.has_word?("hello")}"
+puts "  Has prefix 'wo': #{trie.has_prefix?("wo")}"
+puts "  Words with prefix 'wo': #{trie.words_with_prefix("wo").inspect}"
+puts "  Suggestions for 'he': #{trie.suggestions("he").inspect}"

data/examples/04_trie_data_structure.rb

@@ -0,0 +1,146 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Example 4: Trie Data Structure
+#
+# This example demonstrates how to use the Trie data structure
+# for efficient word lookup and prefix-based operations.
+
+require_relative "../lib/kotoshu"
+
+puts "=== Example 4: Trie Data Structure ==="
+puts
+
+# Build a trie from an array of words
+words = %w[
+  hello help held heap
+  world work word
+  test text toast
+  run running runner
+]
+
+trie = Kotoshu.trie(words)
+
+puts "Built trie with #{words.size} words"
+puts "All words: #{trie.all_words.inspect}"
+puts
+
+# Lookup operations
+puts "Lookup Operations:"
+puts "-" * 20
+puts "Has 'hello': #{trie.has_word?("hello")}"
+puts "Has 'hell': #{trie.has_word?("hell")}"
+puts "Has 'HELLO' (case-sensitive): #{trie.has_word?("HELLO")}"
+puts
+
+# Prefix operations
+puts "Prefix Operations:"
+puts "-" * 20
+puts "Has prefix 'hel': #{trie.has_prefix?("hel")}"
+puts "Has prefix 'wor': #{trie.has_prefix?("wor")}"
+puts "Has prefix 'xyz': #{trie.has_prefix?("xyz")}"
+puts
+
+# Words with prefix
+puts "Words with prefix 'hel': #{trie.words_with_prefix("hel").inspect}"
+puts "Words with prefix 'te': #{trie.words_with_prefix("te").inspect}"
+puts
+
+# Suggestions based on prefix
+puts "Suggestions for 'hel':"
+puts "  #{trie.suggestions("hel", max_results: 10).inspect}"
+puts
+
+puts "Suggestions for 'te':"
+puts "  #{trie.suggestions("te", max_results: 10).inspect}"
+puts
+
+# Traverse the trie
+puts "Traversing trie:"
+puts "-" * 20
+trie.each_word do |word, payload|
+  puts "  #{word} (payload: #{payload.inspect})"
+end
+puts
+
+# Trie builder methods
+puts "Building tries from different sources:"
+puts "-" * 20
+
+# From string
+string_trie = Kotoshu.trie("hello world test")
+puts "From string: #{string_trie.all_words.inspect}"
+
+# From file (if exists)
+test_file = "dictionaries/plain_text/en_US/words.txt"
+if File.exist?(test_file)
+  file_trie = Kotoshu.trie(test_file)
+  puts "From file: loaded #{file_trie.size} words"
+  puts "First 5 words: #{file_trie.all_words.first(5).inspect}"
+end
+
+# Trie set operations
+puts
+puts "Trie Set Operations:"
+puts "-" * 20
+
+trie1 = Kotoshu.trie(%w[hello world test])
+trie2 = Kotoshu.trie(%w[hello world ruby])
+
+puts "Trie 1: #{trie1.all_words.inspect}"
+puts "Trie 2: #{trie2.all_words.inspect}"
+puts
+
+# Union (|)
+union = trie1 | trie2
+puts "Union: #{union.all_words.inspect}"
+
+# Intersection (&)
+intersection = trie1 & trie2
+puts "Intersection: #{intersection.all_words.inspect}"
+
+# Merge (mutating)
+merged = trie1.dup
+merged.merge!(trie2)
+puts "Merged: #{merged.all_words.inspect}"
+
+# Difference
+# Note: Trie doesn't have difference (-) operator, but we can simulate it
+all_words = trie1.all_words | trie2.all_words
+common = trie1.all_words & trie2.all_words
+difference = all_words - common
+puts "Words in only one trie: #{difference.inspect}"
+
+puts
+puts "Trie Statistics:"
+puts "-" * 20
+puts "Total words: #{trie.size}"
+puts "Unique prefixes: #{trie.size}"
+# puts "Max depth: #{trie.max_depth}" # Method not implemented yet
+
+# Advanced: Payload storage
+puts
+puts "Advanced: Payload Storage:"
+puts "-" * 20
+
+payload_trie = Kotoshu::Core::Trie::Builder.new
+payload_trie.add_word("hello", { definition: "a greeting", count: 5 })
+payload_trie.add_word("help", { definition: "assistance", count: 3 })
+payload_trie.add_word("world", { definition: "earth", count: 1 })
+
+payload_trie_obj = payload_trie.build
+
+puts "Word 'hello' payload: #{payload_trie_obj.find_node("hello")&.payload.inspect}"
+puts "Word 'help' payload: #{payload_trie_obj.find_node("help")&.payload.inspect}"
+
+# Convert IndexedDictionary to trie
+puts
+puts "IndexedDictionary to Trie:"
+puts "-" * 20
+
+dict = Kotoshu.dictionary(%w[hello world test])
+trie_from_dict = dict.to_trie
+
+puts "Dictionary words: #{dict.words.inspect}"
+puts "Trie words: #{trie_from_dict.all_words.inspect}"
+puts "Trie has 'hello': #{trie_from_dict.has_word?("hello")}"