stemmers 0.0.1

data/Cargo.toml

@@ -0,0 +1,7 @@
+# This Cargo.toml is here to let externals tools (IDEs, etc.) know that this is
+# a Rust project. Your extensions dependencies should be added to the Cargo.toml
+# in the ext/ directory.
+
+[workspace]
+members = ["./ext/stemmers"]
+resolver = "2"

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2025 Nando Vieira
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,113 @@
+# Stemmers
+
+Rust bindings for https://whatlang.org and
+https://github.com/testuj-to/tantivy-stemmers for language detection and
+stemming.
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+```bash
+bundle add stemmers
+```
+
+If bundler is not being used to manage dependencies, install the gem by
+executing:
+
+```bash
+gem install stemmers
+```
+
+## Usage
+
+The language detection works in the context of the supported stemmers. If
+language doesn't have a stemmer, then it'll return `nil`.
+
+```ruby
+require "stemmers"
+
+Stemmers.detect_language("Hello there!")
+#=> "en"
+
+Stemmers.detect_language("Olá, mundo!")
+#=> "pt"
+```
+
+To stem a word, you can use the `Stemmers.stem_word(word, **options)` method.
+
+```ruby
+require "stemmers"
+
+Stemmers.stem_word("running", language: "en")
+#=> "run"
+
+Stemmers.stem_word("correndo", language: "pt")
+#=> "corr"
+```
+
+You have a few options when stemming a word with
+`Stemmers.stem_word(input, **options)`:
+
+- `language`: The language to use for stemming. If not provided, it will try to
+  detect the language.
+- `normalize`: If set to `true`, it will normalize the word after stemming. This
+  is useful for languages that have diacritics or special characters.
+- `lowercase`: If set to `true`, it will lowercase the word before stemming
+  (stemming requires lowercase strings, but this is not done automatically to
+  avoid unnecessary transformations when using `Stemmers.stem(phrase)`.
+
+To stem a phrase, you can use `Stemmers.stem(input, **options)`.
+
+```ruby
+require "stemmers"
+
+Stemmers.stem("Testing this phrase", language: "en")
+#=> ["test", "this", "phrase"]
+```
+
+The `Stemmers.stem(input, **options)` method has the following options:
+
+- `language`: The language to use for stemming. If not provided, it will try to
+  detect the language.
+- `normalize`: If set to `true`, it will normalize the word after stemming. This
+  is useful for languages that have diacritics or special characters.
+- `lowercase`: If set to `true`, it will lowercase the word before stemming
+  (stemming requires lowercase strings, but this is not done automatically to
+  avoid unnecessary transformations when using `Stemmers.stem(phrase)`).
+- `clean`: If set to `true`, it will remove stop words from the phrase (beware
+  that you may end up with an empty array). It uses the list of stop words from
+  <https://github.com/stopwords-iso/stopwords-iso> (it's not a great list—it has
+  too much surprising words that shouldn't be in the list, but I couldn't find
+  anything better).
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run
+`rake test` to run the tests. You can also run `bin/console` for an interactive
+prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To
+release a new version, update the version number in `version.rb`, and then run
+`bundle exec rake release`, which will create a git tag for the version, push
+git commits and the created tag, and push the `.gem` file to
+[rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at
+https://github.com/fnando/stemmers. This project is intended to be a safe,
+welcoming space for collaboration, and contributors are expected to adhere to
+the
+[code of conduct](https://github.com/fnando/stemmers/blob/main/CODE_OF_CONDUCT.md).
+
+## License
+
+The gem is available as open source under the terms of the
+[MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Stemmers project's codebases, issue trackers, chat
+rooms and mailing lists is expected to follow the
+[code of conduct](https://github.com/fnando/stemmers/blob/main/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "minitest/test_task"
+
+Minitest::TestTask.create
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+require "rb_sys/extensiontask"
+
+desc "Compile the extension"
+task build: :compile
+
+GEMSPEC = Gem::Specification.load("stemmers.gemspec")
+
+RbSys::ExtensionTask.new("stemmers", GEMSPEC) do |ext|
+  ext.lib_dir = "lib/stemmers"
+end
+
+task default: %i[compile test rubocop]

data/ext/stemmers/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "stemmers"
+version = "0.1.0"
+edition = "2021"
+authors = ["Nando Vieira <me@fnando.com>"]
+license = "MIT"
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+magnus = { version = "0.6.2" }
+rust-stemmers = "1.2.0"
+tantivy-stemmers = { version = "0.4.0", features = ["arabic", "armenian_mkrtchyan", "catalan", "czech_dolamic_light", "danish", "dutch", "estonian_freienthal", "finnish", "french", "german", "greek", "hindi_lightweight", "hungarian", "indonesian_tala", "italian", "lithuanian_jocas", "nepali", "norwegian_bokmal", "polish_yarovoy_unaccented", "portuguese", "romanian", "russian", "spanish", "swedish", "turkish_cilden", "yiddish_urieli"] }
+whatlang = "0.16.4"

data/ext/stemmers/extconf.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("stemmers/stemmers")

data/ext/stemmers/src/lib.rs

@@ -0,0 +1,105 @@
+use magnus::{exception, function, prelude::*, Error, Ruby};
+use tantivy_stemmers::algorithms::{
+    arabic, armenian_mkrtchyan, catalan, czech_dolamic_light, danish, dutch, english_porter_2,
+    estonian_freienthal, finnish, french, german, greek, hindi_lightweight, hungarian,
+    indonesian_tala, italian, lithuanian_jocas, nepali, norwegian_bokmal,
+    polish_yarovoy_unaccented, portuguese, romanian, russian, spanish, swedish, turkish_cilden,
+    yiddish_urieli,
+};
+use whatlang::{detect, Lang};
+
+fn lang_to_code(lang: Lang) -> Option<String> {
+    match lang {
+        Lang::Ara => Some("ar".into()),
+        Lang::Cat => Some("ca".into()),
+        Lang::Ces => Some("cs".into()),
+        Lang::Dan => Some("da".into()),
+        Lang::Deu => Some("de".into()),
+        Lang::Ell => Some("el".into()),
+        Lang::Eng => Some("en".into()),
+        Lang::Est => Some("et".into()),
+        Lang::Fin => Some("fi".into()),
+        Lang::Fra => Some("fr".into()),
+        Lang::Hin => Some("hi".into()),
+        Lang::Hun => Some("hu".into()),
+        Lang::Hye => Some("hy".into()),
+        Lang::Ind => Some("id".into()),
+        Lang::Ita => Some("it".into()),
+        Lang::Lit => Some("lt".into()),
+        Lang::Nep => Some("ne".into()),
+        Lang::Nld => Some("nl".into()),
+        Lang::Nob => Some("no".into()),
+        Lang::Pol => Some("pl".into()),
+        Lang::Por => Some("pt".into()),
+        Lang::Ron => Some("ro".into()),
+        Lang::Rus => Some("ru".into()),
+        Lang::Spa => Some("es".into()),
+        Lang::Swe => Some("sv".into()),
+        Lang::Tam => Some("ta".into()),
+        Lang::Tur => Some("tr".into()),
+        Lang::Yid => Some("yi".into()),
+        _ => None,
+    }
+}
+
+fn detect_language(text: String) -> Option<String> {
+    let Some(info) = detect(&text) else {
+        return None;
+    };
+
+    lang_to_code(info.lang())
+}
+
+fn stem_word(word: String, language: String) -> Result<String, Error> {
+    match language.as_str() {
+        "ar" => Ok(arabic(&word).to_string()),
+        "ca" => Ok(catalan(&word).to_string()),
+        "cs" => Ok(czech_dolamic_light(&word).to_string()),
+        "da" => Ok(danish(&word).to_string()),
+        "de" => Ok(german(&word).to_string()),
+        "el" => Ok(greek(&word).to_string()),
+        "en" => Ok(english_porter_2(&word).to_string()),
+        "es" => Ok(spanish(&word).to_string()),
+        "et" => Ok(estonian_freienthal(&word).to_string()),
+        "fi" => Ok(finnish(&word).to_string()),
+        "fr" => Ok(french(&word).to_string()),
+        "hi" => Ok(hindi_lightweight(&word).to_string()),
+        "hu" => Ok(hungarian(&word).to_string()),
+        "hy" => Ok(armenian_mkrtchyan(&word).to_string()),
+        "id" => Ok(indonesian_tala(&word).to_string()),
+        "it" => Ok(italian(&word).to_string()),
+        "lt" => Ok(lithuanian_jocas(&word).to_string()),
+        "no" => Ok(norwegian_bokmal(&word).to_string()),
+        "ne" => Ok(nepali(&word).to_string()),
+        "nl" => Ok(dutch(&word).to_string()),
+        "pl" => Ok(polish_yarovoy_unaccented(&word).to_string()),
+        "pt" => Ok(portuguese(&word).to_string()),
+        "ro" => Ok(romanian(&word).to_string()),
+        "ru" => Ok(russian(&word).to_string()),
+        "sv" => Ok(swedish(&word).to_string()),
+        "tr" => Ok(turkish_cilden(&word).to_string()),
+        "yi" => Ok(yiddish_urieli(&word).to_string()),
+        _ => Err(Error::new(
+            exception::arg_error(),
+            format!("Unsupported language: {language}"),
+        )),
+    }
+}
+
+fn is_supported_language(language: String) -> bool {
+    [
+        "ar", "ca", "cs", "da", "de", "el", "en", "es", "et", "fi", "fr", "hi", "hu", "hy", "id",
+        "it", "lt", "no", "ne", "nl", "pl", "pt", "ro", "ru", "sv", "tr", "yi",
+    ]
+    .contains(&language.as_str())
+}
+
+#[magnus::init]
+fn init(ruby: &Ruby) -> Result<(), Error> {
+    let root = ruby.define_module("Stemmers")?;
+    let module = root.define_module("Bindings")?;
+    module.define_singleton_method("stem_word", function!(stem_word, 2))?;
+    module.define_singleton_method("detect_language", function!(detect_language, 1))?;
+    module.define_singleton_method("supported_language?", function!(is_supported_language, 1))?;
+    Ok(())
+}

data/lib/stemmers/stopwords/af.json

@@ -0,0 +1,53 @@
+[
+  "'n",
+  "aan",
+  "af",
+  "al",
+  "as",
+  "baie",
+  "by",
+  "daar",
+  "dag",
+  "dat",
+  "die",
+  "dit",
+  "een",
+  "ek",
+  "en",
+  "gaan",
+  "gesê",
+  "haar",
+  "het",
+  "hom",
+  "hulle",
+  "hy",
+  "in",
+  "is",
+  "jou",
+  "jy",
+  "kan",
+  "kom",
+  "ma",
+  "maar",
+  "met",
+  "my",
+  "na",
+  "nie",
+  "om",
+  "ons",
+  "op",
+  "saam",
+  "sal",
+  "se",
+  "sien",
+  "so",
+  "sy",
+  "te",
+  "toe",
+  "uit",
+  "van",
+  "vir",
+  "was",
+  "wat",
+  "ʼn"
+]