mittens 0.3.0 → 0.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +8 -0
  3. data/Gemfile +2 -1
  4. data/README.md +2 -2
  5. data/Rakefile +12 -5
  6. data/ext/mittens/extconf.rb +3 -1
  7. data/lib/mittens/version.rb +1 -1
  8. data/vendor/snowball/.github/workflows/ci.yml +144 -17
  9. data/vendor/snowball/.github/workflows/coverage.yml +117 -0
  10. data/vendor/snowball/.github/workflows/runtime-tests.yml +26 -0
  11. data/vendor/snowball/.gitignore +33 -7
  12. data/vendor/snowball/CONTRIBUTING.rst +7 -0
  13. data/vendor/snowball/COPYING +1 -1
  14. data/vendor/snowball/GNUmakefile +736 -298
  15. data/vendor/snowball/NEWS +1394 -7
  16. data/vendor/snowball/README.rst +8 -8
  17. data/vendor/snowball/ada/generate/generate.adb +5 -5
  18. data/vendor/snowball/ada/src/stemmer.adb +177 -188
  19. data/vendor/snowball/ada/src/stemmer.ads +86 -89
  20. data/vendor/snowball/ada/src/stemwords.adb +21 -5
  21. data/vendor/snowball/algorithms/czech.sbl +255 -0
  22. data/vendor/snowball/algorithms/danish.sbl +22 -10
  23. data/vendor/snowball/algorithms/english.sbl +7 -4
  24. data/vendor/snowball/algorithms/esperanto.sbl +6 -7
  25. data/vendor/snowball/algorithms/estonian.sbl +9 -4
  26. data/vendor/snowball/algorithms/finnish.sbl +33 -21
  27. data/vendor/snowball/algorithms/german.sbl +5 -0
  28. data/vendor/snowball/algorithms/indonesian.sbl +115 -96
  29. data/vendor/snowball/algorithms/italian.sbl +26 -1
  30. data/vendor/snowball/algorithms/lithuanian.sbl +13 -13
  31. data/vendor/snowball/algorithms/norwegian.sbl +17 -6
  32. data/vendor/snowball/algorithms/persian.sbl +387 -0
  33. data/vendor/snowball/algorithms/polish.sbl +177 -0
  34. data/vendor/snowball/algorithms/sesotho.sbl +115 -0
  35. data/vendor/snowball/algorithms/turkish.sbl +4 -4
  36. data/vendor/snowball/compiler/analyser.c +2002 -711
  37. data/vendor/snowball/compiler/driver.c +460 -298
  38. data/vendor/snowball/compiler/generator.c +439 -1965
  39. data/vendor/snowball/compiler/generator_ada.c +605 -430
  40. data/vendor/snowball/compiler/generator_c.c +2227 -0
  41. data/vendor/snowball/compiler/generator_csharp.c +365 -274
  42. data/vendor/snowball/compiler/generator_dart.c +1476 -0
  43. data/vendor/snowball/compiler/generator_go.c +348 -270
  44. data/vendor/snowball/compiler/generator_java.c +330 -233
  45. data/vendor/snowball/compiler/generator_js.c +534 -385
  46. data/vendor/snowball/compiler/generator_pascal.c +361 -320
  47. data/vendor/snowball/compiler/generator_php.c +1445 -0
  48. data/vendor/snowball/compiler/generator_python.c +399 -299
  49. data/vendor/snowball/compiler/generator_rust.c +370 -261
  50. data/vendor/snowball/compiler/generator_zig.c +1474 -0
  51. data/vendor/snowball/compiler/header.h +153 -86
  52. data/vendor/snowball/compiler/space.c +121 -63
  53. data/vendor/snowball/compiler/tokeniser.c +286 -182
  54. data/vendor/snowball/compiler/tokens.h +71 -0
  55. data/vendor/snowball/csharp/Snowball/Among.cs +14 -26
  56. data/vendor/snowball/csharp/Snowball/AssemblyInfo.cs +3 -3
  57. data/vendor/snowball/csharp/Snowball/Stemmer.cs +41 -45
  58. data/vendor/snowball/csharp/Stemwords/Program.cs +20 -14
  59. data/vendor/snowball/cxx/.gitignore +4 -0
  60. data/vendor/snowball/cxx/generate_algorithms.pl +87 -0
  61. data/vendor/snowball/cxx/stemmer.h +12 -0
  62. data/vendor/snowball/cxx/stemwords.cxx +176 -0
  63. data/vendor/snowball/cxx/utilities.cxx +2 -0
  64. data/vendor/snowball/dart/.gitignore +4 -0
  65. data/vendor/snowball/dart/analysis_options.yaml +1 -0
  66. data/vendor/snowball/dart/example/test_app.dart +65 -0
  67. data/vendor/snowball/dart/generate_algorithms.pl +21 -0
  68. data/vendor/snowball/dart/lib/snowball.dart +18 -0
  69. data/vendor/snowball/dart/lib/src/snowball.dart +339 -0
  70. data/vendor/snowball/dart/pubspec.yaml +17 -0
  71. data/vendor/snowball/doc/libstemmer_dart_README +37 -0
  72. data/vendor/snowball/doc/libstemmer_js_README +2 -2
  73. data/vendor/snowball/doc/libstemmer_php_README +38 -0
  74. data/vendor/snowball/doc/libstemmer_python_README +3 -3
  75. data/vendor/snowball/examples/stemwords.c +18 -3
  76. data/vendor/snowball/go/README.md +12 -4
  77. data/vendor/snowball/go/env.go +12 -14
  78. data/vendor/snowball/go/stemwords/main.go +2 -1
  79. data/vendor/snowball/java/org/tartarus/snowball/CharArraySequence.java +36 -0
  80. data/vendor/snowball/java/org/tartarus/snowball/SnowballProgram.java +17 -34
  81. data/vendor/snowball/java/org/tartarus/snowball/SnowballStemmer.java +0 -2
  82. data/vendor/snowball/javascript/base-stemmer.js +219 -252
  83. data/vendor/snowball/javascript/stemwords.js +86 -58
  84. data/vendor/snowball/libstemmer/mkmodules.pl +1 -1
  85. data/vendor/snowball/libstemmer/modules.txt +4 -0
  86. data/vendor/snowball/pascal/SnowballProgram.pas +18 -31
  87. data/vendor/snowball/php/base-stemmer.php +453 -0
  88. data/vendor/snowball/php/stemwords.php +25 -0
  89. data/vendor/snowball/python/create_init.py +16 -21
  90. data/vendor/snowball/python/pyproject.toml +3 -0
  91. data/vendor/snowball/python/setup.py +5 -7
  92. data/vendor/snowball/python/snowballstemmer/among.py +1 -2
  93. data/vendor/snowball/python/snowballstemmer/basestemmer.py +24 -33
  94. data/vendor/snowball/python/stemwords.py +72 -63
  95. data/vendor/snowball/runtime/api.c +11 -42
  96. data/vendor/snowball/runtime/api.h +11 -9
  97. data/vendor/snowball/runtime/snowball_runtime.h +110 -0
  98. data/vendor/snowball/runtime/utilities.c +282 -106
  99. data/vendor/snowball/rust/src/main.rs +2 -2
  100. data/vendor/snowball/rust/src/snowball/snowball_env.rs +7 -9
  101. data/vendor/snowball/tests/compilertest +62 -0
  102. data/vendor/snowball/tests/errors/ae-errors.sbl +53 -0
  103. data/vendor/snowball/tests/errors/ae-errors.stderr +8 -0
  104. data/vendor/snowball/tests/errors/bad-dollar.sbl +14 -0
  105. data/vendor/snowball/tests/errors/bad-dollar.stderr +4 -0
  106. data/vendor/snowball/tests/errors/bad-grouping-definition.sbl +9 -0
  107. data/vendor/snowball/tests/errors/bad-grouping-definition.stderr +6 -0
  108. data/vendor/snowball/tests/errors/missing-bra.sbl +6 -0
  109. data/vendor/snowball/tests/errors/missing-bra.stderr +3 -0
  110. data/vendor/snowball/tests/errors/missing-command.sbl +9 -0
  111. data/vendor/snowball/tests/errors/missing-command.stderr +1 -0
  112. data/vendor/snowball/tests/errors/missing-ket-backwardmode.sbl +3 -0
  113. data/vendor/snowball/tests/errors/missing-ket-backwardmode.stderr +1 -0
  114. data/vendor/snowball/tests/errors/missing-ket.sbl +7 -0
  115. data/vendor/snowball/tests/errors/missing-ket.stderr +6 -0
  116. data/vendor/snowball/tests/errors/notdefined.sbl +4 -0
  117. data/vendor/snowball/tests/errors/notdefined.stderr +2 -0
  118. data/vendor/snowball/tests/errors/string-omitted.sbl +5 -0
  119. data/vendor/snowball/tests/errors/string-omitted.stderr +1 -0
  120. data/vendor/snowball/tests/errors/undeclared.sbl +20 -0
  121. data/vendor/snowball/tests/errors/undeclared.stderr +9 -0
  122. data/vendor/snowball/tests/errors/unexpected-token.sbl +24 -0
  123. data/vendor/snowball/tests/errors/unexpected-token.stderr +17 -0
  124. data/vendor/snowball/tests/errors/wrongdirection.sbl +14 -0
  125. data/vendor/snowball/tests/errors/wrongdirection.stderr +1 -0
  126. data/vendor/snowball/tests/runtime/among.sbl +19 -0
  127. data/vendor/snowball/tests/runtime/arithmeticexpr.sbl +86 -0
  128. data/vendor/snowball/tests/runtime/attachinsert.sbl +47 -0
  129. data/vendor/snowball/tests/runtime/booleans.sbl +19 -0
  130. data/vendor/snowball/tests/runtime/externals.sbl +22 -0
  131. data/vendor/snowball/tests/runtime/hop.sbl +12 -0
  132. data/vendor/snowball/tests/runtime/integertests.sbl +60 -0
  133. data/vendor/snowball/tests/runtime/intlimits.sbl +25 -0
  134. data/vendor/snowball/tests/runtime/naming.sbl +20 -0
  135. data/vendor/snowball/tests/runtime/not.sbl +30 -0
  136. data/vendor/snowball/tests/runtime/or.sbl +17 -0
  137. data/vendor/snowball/tests/runtime/repeat.sbl +16 -0
  138. data/vendor/snowball/tests/runtime/setlimit.sbl +24 -0
  139. data/vendor/snowball/tests/runtime/sizelen.sbl +56 -0
  140. data/vendor/snowball/tests/runtime/slice.sbl +27 -0
  141. data/vendor/snowball/tests/runtime/stringdollar.sbl +39 -0
  142. data/vendor/snowball/tests/runtime/strings.sbl +66 -0
  143. data/vendor/snowball/tests/runtime/test.sbl +19 -0
  144. data/vendor/snowball/tests/stemtest.c +41 -1
  145. data/vendor/snowball/tests/syntax/canon.sbl +32 -0
  146. data/vendor/snowball/tests/syntax/canon.stderr +0 -0
  147. data/vendor/snowball/tests/syntax/canon.syntax +57 -0
  148. data/vendor/snowball/tests/syntax/emptyprogram.sbl +2 -0
  149. data/vendor/snowball/tests/syntax/emptyprogram.syntax +0 -0
  150. data/vendor/snowball/tests/syntax/groupings.sbl +11 -0
  151. data/vendor/snowball/tests/syntax/inline.sbl +144 -0
  152. data/vendor/snowball/tests/syntax/inline.stderr +0 -0
  153. data/vendor/snowball/tests/syntax/inline.syntax +121 -0
  154. data/vendor/snowball/tests/syntax/legacy.sbl +17 -0
  155. data/vendor/snowball/tests/syntax/legacy.stderr +5 -0
  156. data/vendor/snowball/tests/syntax/legacy.syntax +19 -0
  157. data/vendor/snowball/tests/syntax/loops.sbl +14 -0
  158. data/vendor/snowball/tests/syntax/loops.stderr +9 -0
  159. data/vendor/snowball/tests/syntax/loops.syntax +29 -0
  160. data/vendor/snowball/tests/syntax/noops.sbl +76 -0
  161. data/vendor/snowball/tests/syntax/noops.stderr +27 -0
  162. data/vendor/snowball/tests/syntax/noops.syntax +135 -0
  163. data/vendor/snowball/tests/syntax/simplifyae.sbl +21 -0
  164. data/vendor/snowball/tests/syntax/simplifyae.stderr +0 -0
  165. data/vendor/snowball/tests/syntax/simplifyae.syntax +39 -0
  166. data/vendor/snowball/tests/syntax/unused.sbl +18 -0
  167. data/vendor/snowball/tests/syntax/unused.stderr +7 -0
  168. data/vendor/snowball/tests/syntax/unused.syntax +3 -0
  169. data/vendor/snowball/zig/env.zig +599 -0
  170. data/vendor/snowball/zig/generate_algorithms.pl +19 -0
  171. data/vendor/snowball/zig/stemwords.zig +123 -0
  172. metadata +102 -4
  173. data/vendor/snowball/compiler/syswords.h +0 -86
  174. data/vendor/snowball/runtime/header.h +0 -62
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 94a898f3a58eb04ef1ec07954623b14eea4161f21d330a3340ce98916c867755
4
- data.tar.gz: 1fd6926ef1e62564b39ba3ea1295831493c4e52ea9a4316c2d06918b3d8d6a50
3
+ metadata.gz: b5d5216e3e6a0e86aa92ab80ec8876e66be4d766d500d82609f166452f28fb08
4
+ data.tar.gz: f5db98b8a25cc9fb2183a2f0010695239a66c82faf00e1ddb34355739087c48b
5
5
  SHA512:
6
- metadata.gz: ffcad206d69a72e22890032b19041af7f11abf4ef5ec7f51e6dce96ae1f2b2b9091e24cdf8a301202354508fd5cde1bd0421163b1351ec4ed133f46e120bd8f8
7
- data.tar.gz: 0ece86c59547de3722aa5bf90afbb87e3f6a9b12c2db6323e56260e7dc3b95061b7118dd732d81ba33e7407b057c695a92aa60d4b66ea6811d6a933983037f26
6
+ metadata.gz: 886dac65964a1b37c5156ff27b7703a71d6acde7f01c9525d1a564636b1fdfa99bf8871b98e8628ed511cdda536e4d28d7e5bebbefdb9b0a67454346e1f571c7
7
+ data.tar.gz: 62a9a20fca62714a28826626183dba6816d11863fb169da43c0fa7d35d2a7978704f4764be63bce5de1986b188566f15f7ec5be5a0d415d9a85fad7285b2f1ac
data/CHANGELOG.md CHANGED
@@ -1,3 +1,11 @@
1
+ ## 0.3.2 (2026-07-10)
2
+
3
+ - Updated Snowball to 3.1.1
4
+
5
+ ## 0.3.1 (2025-12-07)
6
+
7
+ - Fixed installation on FreeBSD
8
+
1
9
  ## 0.3.0 (2025-09-12)
2
10
 
3
11
  - Updated Snowball to 3.0.1
data/Gemfile CHANGED
@@ -4,4 +4,5 @@ gemspec
4
4
 
5
5
  gem "rake"
6
6
  gem "rake-compiler"
7
- gem "minitest", ">= 5"
7
+ gem "minitest"
8
+ gem "ruby_memcheck"
data/README.md CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  Stemming for Ruby, powered by [Snowball](https://github.com/snowballstem/snowball)
4
4
 
5
- :snowflake: Supports 30 languages
5
+ :snowflake: Supports 30+ languages
6
6
 
7
7
  [![Build Status](https://github.com/ankane/mittens/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/mittens/actions)
8
8
 
@@ -36,7 +36,7 @@ Specify the language
36
36
  stemmer = Mittens::Stemmer.new(language: "french")
37
37
  ```
38
38
 
39
- Supports `arabic`, `armenian`, `basque`, `catalan`, `danish`, `dutch`, `dutch_porter`, `english`, `esperanto`, `estonian`, `finnish`, `french`, `german`, `greek`, `hindi`, `hungarian`, `indonesian`, `irish`, `italian`, `lithuanian`, `nepali`, `norwegian`, `porter`, `portuguese`, `romanian`, `russian`, `serbian`, `spanish`, `swedish`, `tamil`, `turkish`, and `yiddish`
39
+ Supports `arabic`, `armenian`, `basque`, `catalan`, `czech`, `danish`, `dutch`, `dutch_porter`, `english`, `esperanto`, `estonian`, `finnish`, `french`, `german`, `greek`, `hindi`, `hungarian`, `indonesian`, `irish`, `italian`, `lithuanian`, `nepali`, `norwegian`, `persian`, `polish`, `porter`, `portuguese`, `romanian`, `russian`, `serbian`, `sesotho`, `spanish`, `swedish`, `tamil`, `turkish`, and `yiddish`
40
40
 
41
41
  ## History
42
42
 
data/Rakefile CHANGED
@@ -1,12 +1,18 @@
1
1
  require "bundler/gem_tasks"
2
2
  require "rake/testtask"
3
3
  require "rake/extensiontask"
4
+ require "ruby_memcheck"
4
5
 
5
- task default: :test
6
- Rake::TestTask.new do |t|
7
- t.libs << "test"
6
+ test_config = lambda do |t|
8
7
  t.pattern = "test/**/*_test.rb"
9
8
  end
9
+ Rake::TestTask.new(&test_config)
10
+
11
+ namespace :test do
12
+ RubyMemcheck::TestTask.new(:valgrind, &test_config)
13
+ end
14
+
15
+ task default: :test
10
16
 
11
17
  Rake::ExtensionTask.new("mittens") do |ext|
12
18
  ext.name = "ext"
@@ -14,8 +20,9 @@ Rake::ExtensionTask.new("mittens") do |ext|
14
20
  end
15
21
 
16
22
  task :remove_ext do
17
- path = "lib/mittens/ext.bundle"
18
- File.unlink(path) if File.exist?(path)
23
+ Dir["lib/mittens/ext.{bundle,so}"].each do |path|
24
+ File.unlink(path)
25
+ end
19
26
  end
20
27
 
21
28
  Rake::Task["build"].enhance [:remove_ext]
@@ -4,7 +4,9 @@ require "open3"
4
4
  vendor = File.expand_path("../../vendor/snowball", __dir__)
5
5
  # CFLAGS from vendor/snowball/GNUmakefile and -fPIC
6
6
  cflags = "-O2 -W -Wall -Wmissing-prototypes -Wmissing-declarations -fPIC"
7
- output, status = Open3.capture2("make", "CFLAGS=#{cflags}", chdir: vendor)
7
+
8
+ make_cmd = RbConfig::CONFIG["host_os"] =~ /bsd|dragonfly/i ? "gmake" : "make"
9
+ output, status = Open3.capture2(make_cmd, "CFLAGS=#{cflags}", chdir: vendor)
8
10
  puts output
9
11
  raise "Command failed" unless status.success?
10
12
 
@@ -1,3 +1,3 @@
1
1
  module Mittens
2
- VERSION = "0.3.0"
2
+ VERSION = "0.3.2"
3
3
  end
@@ -11,10 +11,14 @@ on:
11
11
  - '*.rst'
12
12
  - NEWS
13
13
  pull_request:
14
- branches: master
14
+ branches: main
15
15
  paths-ignore:
16
16
  - '*.rst'
17
17
  - NEWS
18
+ workflow_call:
19
+ inputs:
20
+ runtime-tests: # We reuse this workflow from runtime-tests.yml.
21
+ type: boolean
18
22
 
19
23
  # Allows you to run this workflow manually from the Actions tab
20
24
  workflow_dispatch:
@@ -25,21 +29,33 @@ jobs:
25
29
  matrix:
26
30
  include:
27
31
  - name: "C distribution build"
28
- CFLAGS_DIST_BUILD: '-O2 -Wall -W -std=c90 -Wmissing-prototypes -Wmissing-declarations -Wshadow -Wdeclaration-after-statement -Werror'
32
+ CFLAGS_DIST_BUILD: '-O2 -Wall -W -std=c90 -pedantic -Wmissing-prototypes -Wmissing-declarations -Wshadow -Werror'
29
33
  - name: "C distribution build (clang)"
30
- CFLAGS_DIST_BUILD: '-O2 -Wall -W -std=c90 -Wmissing-prototypes -Wmissing-declarations -Wshadow -Wdeclaration-after-statement -Werror'
34
+ CFLAGS_DIST_BUILD: '-O2 -Wall -W -std=c90 -pedantic -Wmissing-prototypes -Wmissing-declarations -Wshadow -Werror'
31
35
  CC: clang
32
36
  - name: "C"
33
- c_tests: y
34
- WERROR: '-std=c99 -Werror'
35
- - name: "C (clang)"
36
- c_tests: y
37
- WERROR: '-std=c99 -Werror'
37
+ c_tests: check
38
+ WERROR: '-std=c99 -pedantic -Werror'
39
+ - name: "C (clang+sanitisers)"
40
+ # Generate code for all target languages so we exercise the generator
41
+ # code for each target language with sanitisers on.
42
+ c_tests: "generate check"
43
+ # Overflow of unsigned types is defined behaviour, but we don't
44
+ # expect it to happen in the Snowball compiler or C runtime code
45
+ # so enable `unsigned-integer-overflow` too.
46
+ WERROR: '-std=c99 -pedantic -Werror -fsanitize=address,undefined,float-divide-by-zero,local-bounds,nullability,unsigned-integer-overflow -fsanitize-address-use-after-scope -fsanitize-recover=all -fno-omit-frame-pointer'
38
47
  CC: clang
48
+ - name: "C++"
49
+ CXX: g++
50
+ WERROR: '-pedantic -Werror'
39
51
  - name: java
40
52
  JAVA: java
41
53
  JAVAC: javac
42
54
  JAVACFLAGS: '-Xlint:all -Werror'
55
+ - name: dart
56
+ DART: dart
57
+ os: 'ubuntu-24.04'
58
+ dart_sdk: stable
43
59
  - name: go_old
44
60
  os: 'ubuntu-22.04'
45
61
  apt_packages: 'golang-1.13'
@@ -51,15 +67,32 @@ jobs:
51
67
  - name: javascript_node
52
68
  JSRUN: node
53
69
  apt_packages: 'nodejs'
70
+ - name: javascript_deno
71
+ JSRUN: 'deno --allow-read --allow-write'
72
+ apt_packages: 'nodejs'
54
73
  - name: rust
55
74
  RUST: rust
56
75
  apt_packages: 'rustc'
76
+ - name: zig
77
+ ZIG: zig
57
78
  - name: csharp
58
79
  MCS: mcs
59
80
  apt_packages: 'mono-devel'
60
81
  - name: Pascal
61
82
  FPC: fpc
62
83
  apt_packages: 'fpc'
84
+ - name: PHP 8.5
85
+ PHP: php
86
+ apt_extra_repo: ppa:ondrej/php
87
+ apt_packages: 'php8.5-cli php8.5-mbstring'
88
+ - name: PHP 8.4
89
+ PHP: php
90
+ apt_extra_repo: ppa:ondrej/php
91
+ apt_packages: 'php8.4-cli php8.4-mbstring'
92
+ - name: PHP 8.3
93
+ PHP: php
94
+ os: 'ubuntu-24.04'
95
+ apt_packages: 'php8.3-cli php8.3-mbstring'
63
96
  - name: Python 3.8
64
97
  PYTHON_VERSION: 3.8
65
98
  os: 'ubuntu-22.04'
@@ -87,7 +120,7 @@ jobs:
87
120
  apt_packages: 'gnat gprbuild'
88
121
  - name: Windows (C)
89
122
  os: windows-latest
90
- c_tests: y
123
+ c_tests: check
91
124
  ccache: sccache
92
125
  - name: Windows (Go)
93
126
  os: windows-latest
@@ -102,25 +135,27 @@ jobs:
102
135
 
103
136
  env:
104
137
  CC: ${{ matrix.CC || 'gcc' }}
105
- MAKE: ${{ matrix.MAKE || 'make' }}
138
+ MAKE: ${{ matrix.MAKE || 'make' }} --output-sync --jobs=4
106
139
  STEMMING_DATA: 'snowball-data'
140
+ WERROR: ${{ matrix.WERROR }}
107
141
 
108
142
  steps:
109
143
  - name: Checkout
110
- uses: actions/checkout@v4
144
+ uses: actions/checkout@v5
111
145
  with:
112
146
  show-progress: false
113
147
 
114
148
  - name: Checkout data
149
+ if: ${{ ! inputs.runtime-tests }}
115
150
  run: |
116
151
  # Try to check out a branch of the same name from the snowball-data
117
152
  # repo sibling of this snowball repo, so that PRs requiring changes to
118
153
  # both can be CI tested easily.
119
154
  #
120
- # For a PR, GHA will have merged the PR branch into upstream master so
155
+ # For a PR, GHA will have merged the PR branch into upstream main so
121
156
  # we need to similarly merge the snowball-data branch into upstream
122
- # master of the snowball-data repo as there may be changes there
123
- # required by snowball master.
157
+ # main of the snowball-data repo as there may be changes there
158
+ # required by snowball main.
124
159
  #
125
160
  # If there's no such branch (or repo) we just use the standard
126
161
  # snowball-data repo's default branch. If there is such a branch but
@@ -153,16 +188,40 @@ jobs:
153
188
  git clone "$UPSTREAM_REPO_URL"
154
189
  fi
155
190
  fi
191
+ - name: Hook up runtime testing
192
+ if: ${{ inputs.runtime-tests }}
193
+ run: |
194
+ make setup_runtime_tests
156
195
  - name: Install CCache
157
- uses: hendrikmuhs/ccache-action@v1
196
+ uses: hendrikmuhs/ccache-action@v1.2.21
158
197
  with:
159
198
  key: ${{ matrix.name }}
160
199
  variant: ${{ matrix.ccache || 'ccache' }}
200
+ - name: Enable extra Ubuntu package repo
201
+ if: matrix.apt_extra_repo
202
+ run: |
203
+ sudo apt-get -qq remove "php*-cli" "php*-dev"
204
+ sudo add-apt-repository -y ${{ matrix.apt_extra_repo }}
205
+ sudo apt-get update
161
206
  - name: Install Ubuntu packages
162
207
  if: matrix.apt_packages
163
208
  run: |
164
209
  sudo apt-get update
165
210
  sudo apt-get install -y ${{ matrix.apt_packages }}
211
+ - uses: denoland/setup-deno@v2
212
+ if: ${{ startsWith(matrix.JSRUN, 'deno') }}
213
+ with:
214
+ deno-version: vx.x.x
215
+ - name: Install Dart SDK
216
+ if: matrix.dart_sdk
217
+ uses: dart-lang/setup-dart@v1.7.2
218
+ with:
219
+ sdk: ${{ matrix.dart_sdk }}
220
+ - name: Install Zig
221
+ if: matrix.ZIG
222
+ uses: mlugg/setup-zig@v2
223
+ with:
224
+ version: 0.16.0
166
225
  - name: Install mingw64 packages
167
226
  if: matrix.mingw64_packages
168
227
  uses: msys2/setup-msys2@v2
@@ -181,10 +240,21 @@ jobs:
181
240
  tar xf ../dist/libstemmer_c-*.tar.gz
182
241
  cd libstemmer_c-*
183
242
  $MAKE CFLAGS="${{ matrix.CFLAGS_DIST_BUILD }}"
243
+ cd ../..
244
+ rm -rf tmp
184
245
  - name: Test C
185
246
  if: matrix.c_tests
186
- run: $MAKE check CC="$CC"
187
- - uses: actions/setup-python@v5
247
+ run: |
248
+ # Kludge so "generate" target doesn't need ada installed.
249
+ case '${{ matrix.c_tests }}' in
250
+ *generate*) mkdir ada/bin && ln -s /bin/true ada/bin/generate ;;
251
+ esac
252
+ $MAKE ${{ matrix.c_tests }} CC="$CC"
253
+ - name: Test C++
254
+ if: matrix.CXX
255
+ run: |
256
+ $MAKE check_cxx CXX='${{ matrix.CXX }}'
257
+ - uses: actions/setup-python@v6
188
258
  with:
189
259
  python-version: ${{ matrix.PYTHON_VERSION }}
190
260
  if: matrix.PYTHON_VERSION
@@ -194,12 +264,20 @@ jobs:
194
264
  - name: Test Java
195
265
  if: matrix.JAVA && matrix.JAVAC
196
266
  run: $MAKE check_java JAVA="${{ matrix.JAVA }}" JAVAC="${{ matrix.JAVAC }}" JAVACFLAGS="${{ matrix.JAVACFLAGS }}"
267
+ - name: Test Dart
268
+ if: matrix.DART
269
+ run: $MAKE check_dart DART="${{ matrix.DART }}"
197
270
  - name: Test C#
198
271
  if: matrix.MCS
199
272
  run: $MAKE check_csharp MCS="${{ matrix.MCS }}"
200
273
  - name: Test Javascript
201
274
  if: matrix.JSRUN
202
275
  run: $MAKE check_js JSRUN="${{ matrix.JSRUN }}"
276
+ - name: Lint and check Javascript code
277
+ if: ${{ startsWith(matrix.JSRUN, 'deno') }}
278
+ run: |
279
+ deno lint javascript/*.js js_out/*.js
280
+ deno check javascript/*.js js_out/*.js
203
281
  - name: Test Rust
204
282
  if: matrix.RUST
205
283
  run: $MAKE check_rust RUST="${{ matrix.RUST }}"
@@ -208,9 +286,58 @@ jobs:
208
286
  run: |
209
287
  go mod init github.com/snowballstem/snowball
210
288
  $MAKE check_go GO="${{ matrix.GO }}"
289
+ rm -f go.mod
211
290
  - name: Test Pascal
212
291
  if: matrix.FPC
213
292
  run: $MAKE check_pascal FPC="${{ matrix.FPC }}"
293
+ - name: Test PHP
294
+ if: matrix.PHP
295
+ run: |
296
+ ${{ matrix.PHP }} --version
297
+ $MAKE check_php PHP="${{ matrix.PHP }}"
298
+ - name: Test Zig
299
+ if: matrix.ZIG
300
+ run: |
301
+ $MAKE check_zig zig="${{ matrix.ZIG }}"
302
+ rm -rf "$ZIG_LOCAL_CACHE_DIR"
214
303
  - name: Test Ada
215
304
  if: matrix.gprbuild
216
305
  run: $MAKE check_ada gprbuild="${{ matrix.gprbuild }}"
306
+ - name: make clean
307
+ run: |
308
+ rm -rf '${{ env.STEMMING_DATA }}'
309
+ # `.ccache`/`.sccache` is specific to CI. We can't just delete it here
310
+ # or else ccache-action won't save the cache.
311
+ git status --porcelain=v1|grep -v '^?? \.s\?ccache/$'|grep '^' && { echo 'The generated files listed above are not in .gitignore' ; exit 1; }; true
312
+ $MAKE clean ${{ case(inputs.runtime-tests == true, 'clean_runtime_tests', '') }}
313
+ git status --porcelain=v1 --ignored|grep -v '^?? \.s\?ccache/$'|grep '^' && { echo 'The generated files listed above were not removed by `make clean`' ; exit 1; }; true
314
+
315
+ pypi-publish:
316
+ if: >
317
+ github.event_name == 'push' &&
318
+ startsWith(github.ref, 'refs/tags/v') &&
319
+ inputs.runtime-tests != true
320
+ needs:
321
+ - build
322
+ runs-on: ubuntu-latest
323
+ environment:
324
+ name: pypi
325
+ url: https://pypi.org/p/snowballstemmer
326
+ permissions:
327
+ id-token: write
328
+ steps:
329
+ - name: Checkout
330
+ uses: actions/checkout@v6
331
+ with:
332
+ show-progress: false
333
+ - name: Set up Python
334
+ uses: actions/setup-python@v6
335
+ with:
336
+ python-version: '3.14'
337
+ - name: Install build module
338
+ run: pip install build
339
+ - name: Generate dist
340
+ run: make --output-sync --jobs=4 dist_libstemmer_python
341
+ - name: Publish to PyPI
342
+ if: success()
343
+ uses: pypa/gh-action-pypi-publish@release/v1
@@ -0,0 +1,117 @@
1
+ name: Snowball code coverage
2
+
3
+ # Use bash by default on all platforms.
4
+ defaults:
5
+ run:
6
+ shell: bash
7
+
8
+ on:
9
+ push:
10
+ paths:
11
+ - 'algorithms/*.sbl'
12
+ pull_request:
13
+ branches: main
14
+ paths:
15
+ - 'algorithms/*.sbl'
16
+
17
+ # Allows you to run this workflow manually from the Actions tab
18
+ workflow_dispatch:
19
+
20
+ jobs:
21
+ coverage:
22
+ name: "Snowball code coverage"
23
+ runs-on: ubuntu-24.04
24
+
25
+ env:
26
+ CC: gcc
27
+ MAKE: make --output-sync --jobs=4
28
+ STEMMING_DATA: 'snowball-data'
29
+
30
+ steps:
31
+ - name: Checkout
32
+ uses: actions/checkout@v5
33
+ with:
34
+ show-progress: false
35
+
36
+ - name: Checkout data
37
+ run: |
38
+ # Try to check out a branch of the same name from the snowball-data
39
+ # repo sibling of this snowball repo, so that PRs requiring changes to
40
+ # both can be CI tested easily.
41
+ #
42
+ # For a PR, GHA will have merged the PR branch into upstream main so
43
+ # we need to similarly merge the snowball-data branch into upstream
44
+ # main of the snowball-data repo as there may be changes there
45
+ # required by snowball main.
46
+ #
47
+ # If there's no such branch (or repo) we just use the standard
48
+ # snowball-data repo's default branch. If there is such a branch but
49
+ # the merge fails, we treat that as a fatal error.
50
+ UPSTREAM_REPO_URL=https://github.com/snowballstem/snowball-data.git
51
+ if [ -n "$GITHUB_HEAD_REF" ] ; then
52
+ # Pull-request.
53
+ GH_BRANCH=${GITHUB_HEAD_REF}
54
+ GH_REPO_OWNER=${GITHUB_ACTOR}
55
+ GH_REPO_URL=https://github.com/$GH_REPO_OWNER/snowball-data.git
56
+ git clone "$UPSTREAM_REPO_URL"
57
+ cd snowball-data
58
+ git remote add pr "$GH_REPO_URL"
59
+ git config --global user.email "ci@example.org"
60
+ git config --global user.name "CI"
61
+ echo "Trying branch $GH_BRANCH from $GH_REPO_URL"
62
+ if git fetch pr && git branch --track "$GH_BRANCH" pr/"$GH_BRANCH" ; then
63
+ git merge "$GH_BRANCH"
64
+ else
65
+ echo "Falling back to $UPSTREAM_REPO_URL"
66
+ fi
67
+ else
68
+ # Push.
69
+ GH_BRANCH=${GITHUB_REF_NAME}
70
+ GH_REPO_OWNER=${GITHUB_REPOSITORY_OWNER}
71
+ GH_REPO_URL=https://github.com/$GH_REPO_OWNER/snowball-data.git
72
+ echo "Trying branch $GH_BRANCH from $GH_REPO_URL"
73
+ if ! git clone -b "$GH_BRANCH" "$GH_REPO_URL" ; then
74
+ echo "Falling back to $UPSTREAM_REPO_URL"
75
+ git clone "$UPSTREAM_REPO_URL"
76
+ fi
77
+ fi
78
+ - name: Install CCache
79
+ uses: hendrikmuhs/ccache-action@v1.2.21
80
+ with:
81
+ key: ${{ matrix.name }}
82
+ variant: ${{ matrix.ccache || 'ccache' }}
83
+ - name: Build
84
+ run: $MAKE CC="ccache $CC" SNOWBALL_FLAGS=-coverage CPPFLAGS=-DSNOWBALL_COVERAGE
85
+ - name: Generate coverage reports
86
+ run: |
87
+ # Find commit to compare with:
88
+ if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ] ; then
89
+ # Manually triggered - generate coverage for all algorithms.
90
+ ls -1 algorithms|sed 's,\.sbl$,,p;d'
91
+ elif [ -n "$GITHUB_HEAD_REF" ] ; then
92
+ # Pull-request.
93
+ git diff --numstat "$GITHUB_BASE_REF"... algorithms|sed 's,^[0-9].*/\([^ ]*\)\.sbl$,\1,p;d'
94
+ else
95
+ # Push.
96
+ #
97
+ # Currently we just generate coverage for any .sbl sources which
98
+ # changed in the most recent commit. Ideally we'd diff against the
99
+ # branch pre-push but github doesn't seem to provide a way to get
100
+ # that information.
101
+ git show --numstat algorithms|sed 's,^[0-9].*/\([^ ]*\)\.sbl$,\1,p;d'
102
+ fi | while read a ; do
103
+ # This will skip coverage reports for arabic, which currently has far
104
+ # too large a test vocabulary, such that it needs to be gzipped to get
105
+ # github to accept it:
106
+ #
107
+ # https://github.com/snowballstem/snowball-data/issues/31
108
+ #
109
+ # That size means generating a coverage report for it takes something
110
+ # like 20-25 minutes.
111
+ if [ -f "${STEMMING_DATA}/$a/voc.txt" ] ; then
112
+ $MAKE -s "check_utf8_$a" 2> coverage.log
113
+ echo ; echo "=== Coverage report for $a ===" ; echo
114
+ sort -k2,2 -k3,3n -k3,3 -k5,5g coverage.log |
115
+ uniq -c |awk '{$1=sprintf("% 8d",$1-1);print}'
116
+ fi
117
+ done
@@ -0,0 +1,26 @@
1
+ name: Runtime tests
2
+
3
+ # Use bash by default on all platforms.
4
+ defaults:
5
+ run:
6
+ shell: bash
7
+
8
+ on:
9
+ push:
10
+ paths-ignore:
11
+ - '*.rst'
12
+ - NEWS
13
+ pull_request:
14
+ branches: main
15
+ paths-ignore:
16
+ - '*.rst'
17
+ - NEWS
18
+
19
+ # Allows you to run this workflow manually from the Actions tab
20
+ workflow_dispatch:
21
+
22
+ jobs:
23
+ build:
24
+ uses: ./.github/workflows/ci.yml
25
+ with:
26
+ runtime-tests: true
@@ -1,7 +1,19 @@
1
+ .DS_Store
2
+ *.exe
1
3
  *.o
4
+ # Created by `setup_runtime_tests`:
5
+ /overrides.mk
6
+ /tmp_runtime_tests_snowball_data/
7
+ #
8
+ /algorithms.mk
9
+ /snowball
10
+ /stemtest
11
+ /dist
12
+ # Ada
13
+ /ada/algorithms/
2
14
  /ada/bin/
3
15
  /ada/obj/
4
- /algorithms.mk
16
+ # C
5
17
  /libstemmer/libstemmer.c
6
18
  /libstemmer/libstemmer_utf8.c
7
19
  /libstemmer/mkinc.mak
@@ -9,18 +21,32 @@
9
21
  /libstemmer/modules.h
10
22
  /libstemmer/modules_utf8.h
11
23
  /libstemmer.a
12
- /snowball
13
24
  /src_c
14
- /stemtest
15
25
  /stemwords
16
- /dist
26
+ # C#
27
+ /csharp_stemwords
28
+ /csharp/Snowball/Algorithms/*.generated.cs
29
+ # Dart
30
+ /dart/lib/ext/
31
+ /dart/lib/src/algorithms.dart
32
+ # Go
33
+ /go/algorithms/
34
+ /go/stemwords/algorithms.go
35
+ # Java
17
36
  /java/org/tartarus/snowball/ext/
37
+ /java/org/tartarus/snowball/*.class
38
+ # JS
18
39
  /js_out
40
+ # PHP
41
+ /php_out
42
+ # Python
19
43
  /python_check
20
44
  /python_out
21
- *.generated.cs
45
+ # Rust
22
46
  /rust/Cargo.lock
23
47
  /rust/src/snowball/algorithms/*.rs
24
48
  /rust/target/
25
- /go/algorithms/
26
- /go/stemwords/algorithms.go
49
+ # Zig
50
+ /zig/*_stemmer.zig
51
+ /zig/algorithms.zig
52
+ /zig/stemwords
@@ -223,6 +223,13 @@ Key problems to solve
223
223
  backends with special handling here which may be useful to look at
224
224
  include Javascript, Pascal and Python.
225
225
 
226
+ * Although all the stemmers we currently ship define a single Snowball
227
+ `external` named `stem`, this should not be assumed. Also externals
228
+ should be callable from Snowball code as well as externally (see
229
+ the Go and Rust generators for examples of how to handle this - if
230
+ an external is called from Snowball code then it is generated as a
231
+ routine, with a shim external which just tail calls to this routine).
232
+
226
233
  Don't hardcode algorithm names
227
234
  ------------------------------
228
235
 
@@ -1,7 +1,7 @@
1
1
  Copyright (c) 2001, Dr Martin Porter
2
2
  Copyright (c) 2004,2005, Richard Boulton
3
3
  Copyright (c) 2013, Yoshiki Shibukawa
4
- Copyright (c) 2006,2007,2009,2010,2011,2014-2019, Olly Betts
4
+ Copyright (c) 2006-2025, Olly Betts
5
5
  All rights reserved.
6
6
 
7
7
  Redistribution and use in source and binary forms, with or without