stemmers 0.0.1

data/lib/stemmers/stopwords/zu.json

@@ -0,0 +1,31 @@
+[
+  "futhi",
+  "kahle",
+  "kakhulu",
+  "kanye",
+  "khona",
+  "kodwa",
+  "kungani",
+  "kusho",
+  "la",
+  "lakhe",
+  "lapho",
+  "mina",
+  "ngesikhathi",
+  "nje",
+  "phansi",
+  "phezulu",
+  "u",
+  "ukuba",
+  "ukuthi",
+  "ukuze",
+  "uma",
+  "wahamba",
+  "wakhe",
+  "wami",
+  "wase",
+  "wathi",
+  "yakhe",
+  "zakhe",
+  "zonke"
+]

data/lib/stemmers/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Stemmers
+  VERSION = "0.0.1"
+end

data/lib/stemmers.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require_relative "stemmers/version"
+require_relative "stemmers/stemmers"
+
+module Stemmers
+  # Detects the language of the given text.
+  # If the language cannot be detected, it returns nil.
+  #
+  # @param text [String] The text to be analyzed.
+  # @return [String, nil] The detected language code or nil if undetectable.
+  def self.detect_language(text)
+    Bindings.detect_language(text)
+  end
+
+  # Detects if the language is supported by the stemmers.
+  #
+  # @param language [String] The language to check.
+  # @return [Boolean] True if the language is supported, false otherwise.
+  def self.supported_language?(language)
+    Bindings.supported_language?(language)
+  end
+
+  # Stems the given word in the specified language.
+  # If the language is not supported, it raises an `ArgumentError`.
+  #
+  # @param word [String] The word to be stemmed.
+  # @param language [String] The language of the word.
+  # @param lowercase [Boolean] If true, converts the word to lowercase before
+  #                            stemming.
+  # @param normalize [Boolean] If true, removes accents from the word after
+  #                            stemming.
+  # @return [String] The stemmed word.
+  def self.stem_word(word, language:, normalize: false, lowercase: false)
+    word = word.downcase if lowercase
+    stem = Bindings.stem_word(word, language)
+    stem = normalize_word(stem) if normalize
+
+    stem
+  end
+
+  # Stems the given phrase in the specified language.
+  # If the language is not supported, it raises an `ArgumentError`.
+  #
+  # @param phrase [String] The phrase to be stemmed.
+  # @param language [String] The language of the phrase.
+  # @param clean [Boolean] If true, removes stop words before stemming.
+  # @param normalize [Boolean] If true, removes accents from the phrase
+  #                                 after stemming.
+  # @return [Array<String>] An array of stemmed words.
+  def self.stem(phrase, language:, clean: false, normalize: false)
+    words = phrase.downcase.strip.split(/\s+/)
+
+    if clean
+      stop_words = stop_words(language)
+      words = words.reject {|word| stop_words.include?(word) }
+    end
+
+    words.map {|word| stem_word(word, language:, normalize:) }
+  end
+
+  # Returns the stop words for the specified language.
+  # If the language is not supported, an empty list is returned.
+  #
+  # @param language [String] The language for which to retrieve stop words.
+  # @return [Array<String>] An array of stop words.
+  def self.stop_words(language)
+    stop_words_cache[language]
+  end
+
+  # Normalizes a word by removing accents and diacritics.
+  # This is useful for languages where accents do not change the meaning
+  # of the word, such as Portuguese.
+  #
+  # @param word [String] The word to be normalized.
+  # @return [String] The normalized word with accents removed.
+  def self.normalize_word(word)
+    word.unicode_normalize(:nfkd).gsub(/\p{M}/, "")
+  end
+
+  # Returns a cache of stop words loaded from a JSON file.
+  # The cache is initialized only once and reused for subsequent calls.
+  # @return [Hash<String, Array<String>>] A hash mapping language codes to
+  #                                       arrays of stop words.
+  def self.stop_words_cache
+    @stop_words_cache ||= Hash.new do |hash, key|
+      path = File.join(__dir__, "stemmers/stopwords/#{key}.json")
+      hash[key] = File.file?(path) ? JSON.load_file(path) : []
+    end
+  end
+end

data/sig/stemmers.rbs

@@ -0,0 +1,4 @@
+module Stemmers
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: stemmers
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Nando Vieira
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rb_sys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.91
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.91
+description: Bindings some popular snowball stemming algorithms
+email:
+- me@fnando.com
+executables: []
+extensions:
+- ext/stemmers/extconf.rb
+extra_rdoc_files: []
+files:
+- ".rubocop.yml"
+- CHANGELOG.md
+- CODE_OF_CONDUCT.md
+- Cargo.lock
+- Cargo.toml
+- LICENSE.txt
+- README.md
+- Rakefile
+- ext/stemmers/Cargo.toml
+- ext/stemmers/extconf.rb
+- ext/stemmers/src/lib.rs
+- lib/stemmers.rb
+- lib/stemmers/stopwords/af.json
+- lib/stemmers/stopwords/ar.json
+- lib/stemmers/stopwords/bg.json
+- lib/stemmers/stopwords/bn.json
+- lib/stemmers/stopwords/br.json
+- lib/stemmers/stopwords/ca.json
+- lib/stemmers/stopwords/cs.json
+- lib/stemmers/stopwords/da.json
+- lib/stemmers/stopwords/de.json
+- lib/stemmers/stopwords/el.json
+- lib/stemmers/stopwords/en.json
+- lib/stemmers/stopwords/eo.json
+- lib/stemmers/stopwords/es.json
+- lib/stemmers/stopwords/et.json
+- lib/stemmers/stopwords/eu.json
+- lib/stemmers/stopwords/fa.json
+- lib/stemmers/stopwords/fi.json
+- lib/stemmers/stopwords/fr.json
+- lib/stemmers/stopwords/ga.json
+- lib/stemmers/stopwords/gl.json
+- lib/stemmers/stopwords/gu.json
+- lib/stemmers/stopwords/ha.json
+- lib/stemmers/stopwords/he.json
+- lib/stemmers/stopwords/hi.json
+- lib/stemmers/stopwords/hr.json
+- lib/stemmers/stopwords/hu.json
+- lib/stemmers/stopwords/hy.json
+- lib/stemmers/stopwords/id.json
+- lib/stemmers/stopwords/it.json
+- lib/stemmers/stopwords/ja.json
+- lib/stemmers/stopwords/ko.json
+- lib/stemmers/stopwords/ku.json
+- lib/stemmers/stopwords/la.json
+- lib/stemmers/stopwords/lt.json
+- lib/stemmers/stopwords/lv.json
+- lib/stemmers/stopwords/mr.json
+- lib/stemmers/stopwords/ms.json
+- lib/stemmers/stopwords/nl.json
+- lib/stemmers/stopwords/no.json
+- lib/stemmers/stopwords/pl.json
+- lib/stemmers/stopwords/pt.json
+- lib/stemmers/stopwords/ro.json
+- lib/stemmers/stopwords/ru.json
+- lib/stemmers/stopwords/sk.json
+- lib/stemmers/stopwords/sl.json
+- lib/stemmers/stopwords/so.json
+- lib/stemmers/stopwords/st.json
+- lib/stemmers/stopwords/sv.json
+- lib/stemmers/stopwords/sw.json
+- lib/stemmers/stopwords/th.json
+- lib/stemmers/stopwords/tl.json
+- lib/stemmers/stopwords/tr.json
+- lib/stemmers/stopwords/uk.json
+- lib/stemmers/stopwords/ur.json
+- lib/stemmers/stopwords/vi.json
+- lib/stemmers/stopwords/yo.json
+- lib/stemmers/stopwords/zh.json
+- lib/stemmers/stopwords/zu.json
+- lib/stemmers/version.rb
+- sig/stemmers.rbs
+homepage: https://github.com/fnando/stemmers
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/fnando/stemmers
+  source_code_uri: https://github.com/fnando/stemmers
+  changelog_uri: https://github.com/fnando/stemmers
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.3.11
+requirements: []
+rubygems_version: 3.6.7
+specification_version: 4
+summary: Bindings some popular snowball stemming algorithms
+test_files: []