mittens 0.2.0 → 0.3.1

data/vendor/snowball/rust/src/snowball/snowball_env.rs

@@ -189,27 +189,69 @@ impl<'a> SnowballEnv<'a> {
         return false;
     }
 
+    pub fn go_in_grouping(&mut self, chars: &[u8], min: u32, max: u32) -> bool {
+        while self.cursor < self.limit {
+            if let Some(chr) = self.current[self.cursor as usize..].chars().next() {
+                let mut ch = chr as u32; //codepoint as integer
+                if ch > max || ch < min {
+                    return true;
+                }
+                ch -= min;
+                if (chars[(ch >> 3) as usize] & (0x1 << (ch & 0x7))) == 0 {
+                    return true;
+                }
+                self.next_char();
+            } else {
+                return false;
+            }
+        }
+        return false;
+    }
+
     pub fn in_grouping_b(&mut self, chars: &[u8], min: u32, max: u32) -> bool {
         if self.cursor <= self.limit_backward {
             return false;
         }
+        let c = self.cursor;
         self.previous_char();
         if let Some(chr) = self.current[self.cursor as usize..].chars().next() {
             let mut ch = chr as u32; //codepoint as integer
-            self.next_char();
             if ch > max || ch < min {
+                self.cursor = c;
                 return false;
             }
             ch -= min;
             if (chars[(ch >> 3) as usize] & (0x1 << (ch & 0x7))) == 0 {
+                self.cursor = c;
                 return false;
             }
-            self.previous_char();
             return true;
         }
         return false;
     }
 
+    pub fn go_in_grouping_b(&mut self, chars: &[u8], min: u32, max: u32) -> bool {
+        while self.cursor > self.limit_backward {
+            let c = self.cursor;
+            self.previous_char();
+            if let Some(chr) = self.current[self.cursor as usize..].chars().next() {
+                let mut ch = chr as u32; //codepoint as integer
+                if ch > max || ch < min {
+                    self.cursor = c;
+                    return true;
+                }
+                ch -= min;
+                if (chars[(ch >> 3) as usize] & (0x1 << (ch & 0x7))) == 0 {
+                    self.cursor = c;
+                    return true;
+                }
+            } else {
+                return false;
+            }
+        }
+        return false;
+    }
+
     pub fn out_grouping(&mut self, chars: &[u8], min: u32, max: u32) -> bool {
         if self.cursor >= self.limit {
             return false;
@@ -229,26 +271,62 @@ impl<'a> SnowballEnv<'a> {
         return false;
     }
 
+    pub fn go_out_grouping(&mut self, chars: &[u8], min: u32, max: u32) -> bool {
+        while self.cursor < self.limit {
+            if let Some(chr) = self.current[self.cursor as usize..].chars().next() {
+                let mut ch = chr as u32; //codepoint as integer
+                if ch <= max && ch >= min {
+                    ch -= min;
+                    if (chars[(ch >> 3) as usize] & (0x1 << (ch & 0x7))) != 0 {
+                        return true;
+                    }
+                }
+                self.next_char();
+            } else {
+                return false;
+            }
+        }
+        return false;
+    }
+
     pub fn out_grouping_b(&mut self, chars: &[u8], min: u32, max: u32) -> bool {
         if self.cursor <= self.limit_backward {
             return false;
         }
+        let c = self.cursor;
         self.previous_char();
         if let Some(chr) = self.current[self.cursor as usize..].chars().next() {
             let mut ch = chr as u32; //codepoint as integer
-            self.next_char();
             if ch > max || ch < min {
-                self.previous_char();
                 return true;
             }
             ch -= min;
             if (chars[(ch >> 3) as usize] & (0x1 << (ch & 0x7))) == 0 {
-                self.previous_char();
                 return true;
             }
+            self.cursor = c;
         }
         return false;
+    }
 
+    pub fn go_out_grouping_b(&mut self, chars: &[u8], min: u32, max: u32) -> bool {
+        while self.cursor > self.limit_backward {
+            let c = self.cursor;
+            self.previous_char();
+            if let Some(chr) = self.current[self.cursor as usize..].chars().next() {
+                let mut ch = chr as u32; //codepoint as integer
+                if ch <= max && ch >= min {
+                    ch -= min;
+                    if (chars[(ch >> 3) as usize] & (0x1 << (ch & 0x7))) != 0 {
+                        self.cursor = c;
+                        return true;
+                    }
+                }
+            } else {
+                return false;
+            }
+        }
+        return false;
     }
 
 

data/vendor/snowball/tests/stemtest.c

@@ -1,5 +1,4 @@
-/* This is a simple program which uses libstemmer to provide a command
- * line interface for stemming using any of the algorithms provided.
+/* Unit tests for handling of cases the vocabularies don't cover.
  */
 
 #include <stdio.h>
@@ -20,16 +19,20 @@ static const struct testcase {
     /* Expected output string (0 means same as input) */
     const char * expect;
 } testcases[] = {
+    // Regression tests for C support code bug decoding 4 byte UTF-8 sequences.
+    // https://github.com/snowballstem/snowball/issues/138
     { "en", 0,
       "a" EMOJI_FACE_THROWING_A_KISS "ing",
       "a" EMOJI_FACE_THROWING_A_KISS "e" },
     { "en", 0, U_40079 "wing", 0 },
     // The Finnish stemmer used to damage numbers ending with two or more of
-    // the same digit: https://github.com/snowballstem/snowball/issues/66
+    // the same digit.  Regression test, applied to all stemmers.
+    // https://github.com/snowballstem/snowball/issues/66
     { 0, 0, "2000", 0 },
     { 0, 0, "999", 0 },
     { 0, 0, "1000000000", 0 },
-    // The Danish stemmer used to damage a number at the end of a word:
+    // The Danish stemmer used to damage a number at the end of a word.
+    // Regression test, applied to all stemmers.
     // https://github.com/snowballstem/snowball/issues/81
     { 0, 0, "space1999", 0 },
     { 0, 0, "hal9000", 0 },

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: mittens
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Andrew Kane
 bindir: bin
 cert_chain: []
-date: 2025-04-03 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies: []
 email: andrew@ankane.org
 executables: []
@@ -25,8 +25,8 @@ files:
 - lib/mittens.rb
 - lib/mittens/version.rb
 - mittens.gemspec
+- vendor/snowball/.github/workflows/ci.yml
 - vendor/snowball/.gitignore
-- vendor/snowball/.travis.yml
 - vendor/snowball/AUTHORS
 - vendor/snowball/CONTRIBUTING.rst
 - vendor/snowball/COPYING
@@ -47,18 +47,19 @@ files:
 - vendor/snowball/algorithms/catalan.sbl
 - vendor/snowball/algorithms/danish.sbl
 - vendor/snowball/algorithms/dutch.sbl
+- vendor/snowball/algorithms/dutch_porter.sbl
 - vendor/snowball/algorithms/english.sbl
+- vendor/snowball/algorithms/esperanto.sbl
+- vendor/snowball/algorithms/estonian.sbl
 - vendor/snowball/algorithms/finnish.sbl
 - vendor/snowball/algorithms/french.sbl
 - vendor/snowball/algorithms/german.sbl
-- vendor/snowball/algorithms/german2.sbl
 - vendor/snowball/algorithms/greek.sbl
 - vendor/snowball/algorithms/hindi.sbl
 - vendor/snowball/algorithms/hungarian.sbl
 - vendor/snowball/algorithms/indonesian.sbl
 - vendor/snowball/algorithms/irish.sbl
 - vendor/snowball/algorithms/italian.sbl
-- vendor/snowball/algorithms/kraaij_pohlmann.sbl
 - vendor/snowball/algorithms/lithuanian.sbl
 - vendor/snowball/algorithms/lovins.sbl
 - vendor/snowball/algorithms/nepali.sbl
@@ -90,7 +91,6 @@ files:
 - vendor/snowball/compiler/header.h
 - vendor/snowball/compiler/space.c
 - vendor/snowball/compiler/syswords.h
-- vendor/snowball/compiler/syswords2.h
 - vendor/snowball/compiler/tokeniser.c
 - vendor/snowball/csharp/.gitignore
 - vendor/snowball/csharp/Snowball/Algorithms/.gitignore
@@ -168,7 +168,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.2
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Stemming for Ruby, powered by Snowball
 test_files: []

data/vendor/snowball/.travis.yml

@@ -1,112 +0,0 @@
-language: minimal
-arch: arm64
-dist: xenial
-env:
-  global: MAKE=make
-matrix:
-  include:
-    - name: "C distribution build"
-      language: c
-      dist: focal
-      compiler: gcc
-      env: CFLAGS_DIST_BUILD='-O2 -Wall -W -std=c90 -Wdeclaration-after-statement -Werror'
-    - name: "C distribution build (clang)"
-      language: c
-      dist: focal
-      compiler: clang
-      env: CFLAGS_DIST_BUILD='-O2 -Wall -W -std=c90 -Wdeclaration-after-statement -Werror'
-    - language: c
-      compiler: gcc
-      env: c_tests=y CFLAGS='-O2 -Wall -W -std=c99 -Werror'
-    - language: c
-      compiler: clang
-      env: c_tests=y CFLAGS='-O2 -Wall -W -std=c99 -Werror'
-    - language: java
-      env: JAVA=java JAVAC=javac
-    - language: go
-      go: "1.8"
-      env: GO=go
-    - language: go
-      dist: bionic
-      go: "1.17"
-      env: GO=go
-    - language: node_js
-      node_js: "node"
-      env: NODE=node
-      dist: focal # seems to be needed for working node
-    - language: rust
-      rust:
-        - stable
-        - beta
-      dist: bionic
-      env: RUST=rust
-    - language: csharp
-      arch: amd64 # csharp doesn't seem to work on arm64
-      env: MCS=mcs
-      dist: bionic
-    - name: Pascal
-      env: FPC=fpc
-      dist: bionic
-      addons:
-        apt:
-          packages:
-            - fpc
-    # The pure Python versions run slowly so we need to thin the testdata
-    # for languages such as Arabic where there's a lot, or else the build
-    # hits the travis time limit.  With pypy, it's enough faster than we
-    # can run the full tests.
-    - language: python
-      python: "3.9"
-      env: PYTHON=python THIN_FACTOR=10
-    - language: python
-      python: "3.7"
-      env: PYTHON=python THIN_FACTOR=10
-    - language: python
-      python: "3.6"
-      env: PYTHON=python THIN_FACTOR=10
-    - language: python
-      python: "pypy3.7-7.3.5"
-      env: PYTHON=python
-      dist: bionic
-    - name: "Ada"
-      env: gprbuild=gprbuild
-      dist: bionic
-      addons:
-        apt:
-          packages:
-            - gnat
-            - gprbuild
-    - os: windows
-      language: c
-      env: c_tests=y MAKE=mingw32-make
-    - os: windows
-      language: go
-      env: GO=go MAKE=mingw32-make
-
-before_install:
-  # Try to check out a branch of the same name from the snowball-data repo
-  # sibling of this snowball repo, so that PRs requiring changes to both can be
-  # CI tested easily.
-  #
-  # If that fails, just use the standard snowball-data repo's default branch.
-  - GH_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH:-$TRAVIS_BRANCH}
-  - GH_REPO_SLUG=${TRAVIS_PULL_REQUEST_SLUG:-$TRAVIS_REPO_SLUG}
-  - GH_REPO_URL=https://github.com/${GH_REPO_SLUG%%/*}/snowball-data.git
-  - echo "Trying branch $GH_BRANCH from $GH_REPO_URL"
-  - git clone --depth=1 -b "$GH_BRANCH" "$GH_REPO_URL" || git clone --depth=1 https://github.com/snowballstem/snowball-data.git
-
-script:
-  # Ensure CC is set for building the compiler in non-C builds.
-  - test -n "$CC" || export CC=gcc
-  - $MAKE CC="$CC"
-  - test -z "$CFLAGS_DIST_BUILD" || { pip install setuptools && $MAKE dist && mkdir tmp && cd tmp && tar xf ../dist/libstemmer_c-*.tar.gz && cd libstemmer_c-* && $MAKE CFLAGS="$CFLAGS_DIST_BUILD" ; }
-  - test -z "$c_tests" || $MAKE check CC="$CC" STEMMING_DATA=snowball-data
-  - test -z "$PYTHON" || $MAKE check_python python="$PYTHON" STEMMING_DATA=snowball-data
-  - test -z "$JAVA" -o -z "$JAVAC" || $MAKE check_java STEMMING_DATA=snowball-data
-  - test -z "$MCS" || $MAKE check_csharp MCS="$MCS" STEMMING_DATA=snowball-data
-  - test -z "$NODE" || $MAKE check_js STEMMING_DATA=snowball-data
-  - test -z "$RUST" || $MAKE check_rust STEMMING_DATA=snowball-data
-  - test -z "$RUST" || $MAKE check_rust STEMMING_DATA=snowball-data
-  - test -z "$GO" || $MAKE check_go STEMMING_DATA=snowball-data
-  - test -z "$FPC" || $MAKE check_pascal STEMMING_DATA=snowball-data
-  - test -z "$gprbuild" || $MAKE check_ada STEMMING_DATA=snowball-data

data/vendor/snowball/algorithms/german2.sbl

@@ -1,145 +0,0 @@
-
-/*
-    Extra rule for -nisse ending added 11 Dec 2009
-*/
-
-routines (
-           prelude postlude
-           mark_regions
-           R1 R2
-           standard_suffix
-)
-
-externals ( stem )
-
-integers ( p1 p2 x )
-
-groupings ( v s_ending st_ending )
-
-stringescapes {}
-
-/* special characters */
-
-stringdef a"   '{U+00E4}'
-stringdef o"   '{U+00F6}'
-stringdef u"   '{U+00FC}'
-stringdef ss   '{U+00DF}'
-
-define v 'aeiouy{a"}{o"}{u"}'
-
-define s_ending  'bdfghklmnrt'
-define st_ending s_ending - 'r'
-
-define prelude as (
-
-    test repeat goto (
-        v [('u'] v <- 'U') or
-           ('y'] v <- 'Y')
-    )
-
-    repeat (
-        [substring] among(
-            '{ss}' (<- 'ss')
-            'ae'   (<- '{a"}')
-            'oe'   (<- '{o"}')
-            'ue'   (<- '{u"}')
-            'qu'   ()
-            ''     (next)
-        )
-    )
-
-)
-
-define mark_regions as (
-
-    $p1 = limit
-    $p2 = limit
-
-    test(hop 3 setmark x)
-
-    gopast v  gopast non-v  setmark p1
-    try($p1 < x  $p1 = x)  // at least 3
-    gopast v  gopast non-v  setmark p2
-
-)
-
-define postlude as repeat (
-
-    [substring] among(
-        'Y'    (<- 'y')
-        'U'    (<- 'u')
-        '{a"}' (<- 'a')
-        '{o"}' (<- 'o')
-        '{u"}' (<- 'u')
-        ''     (next)
-    )
-
-)
-
-backwardmode (
-
-    define R1 as $p1 <= cursor
-    define R2 as $p2 <= cursor
-
-    define standard_suffix as (
-        do (
-            [substring] R1 among(
-                'em' 'ern' 'er'
-                (   delete
-                )
-                'e' 'en' 'es'
-                (   delete
-                    try (['s'] 'nis' delete)
-                )
-                's'
-                (   s_ending delete
-                )
-            )
-        )
-        do (
-            [substring] R1 among(
-                'en' 'er' 'est'
-                (   delete
-                )
-                'st'
-                (   st_ending hop 3 delete
-                )
-            )
-        )
-        do (
-            [substring] R2 among(
-                'end' 'ung'
-                (   delete
-                    try (['ig'] not 'e' R2 delete)
-                )
-                'ig' 'ik' 'isch'
-                (   not 'e' delete
-                )
-                'lich' 'heit'
-                (   delete
-                    try (
-                        ['er' or 'en'] R1 delete
-                    )
-                )
-                'keit'
-                (   delete
-                    try (
-                        [substring] R2 among(
-                            'lich' 'ig'
-                            (   delete
-                            )
-                        )
-                    )
-                )
-            )
-        )
-    )
-)
-
-define stem as (
-    do prelude
-    do mark_regions
-    backwards
-        do standard_suffix
-    do postlude
-)