mittens 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/Gemfile +2 -1
- data/README.md +2 -2
- data/Rakefile +12 -5
- data/ext/mittens/extconf.rb +3 -1
- data/lib/mittens/version.rb +1 -1
- data/vendor/snowball/.github/workflows/ci.yml +144 -17
- data/vendor/snowball/.github/workflows/coverage.yml +117 -0
- data/vendor/snowball/.github/workflows/runtime-tests.yml +26 -0
- data/vendor/snowball/.gitignore +33 -7
- data/vendor/snowball/CONTRIBUTING.rst +7 -0
- data/vendor/snowball/COPYING +1 -1
- data/vendor/snowball/GNUmakefile +736 -298
- data/vendor/snowball/NEWS +1394 -7
- data/vendor/snowball/README.rst +8 -8
- data/vendor/snowball/ada/generate/generate.adb +5 -5
- data/vendor/snowball/ada/src/stemmer.adb +177 -188
- data/vendor/snowball/ada/src/stemmer.ads +86 -89
- data/vendor/snowball/ada/src/stemwords.adb +21 -5
- data/vendor/snowball/algorithms/czech.sbl +255 -0
- data/vendor/snowball/algorithms/danish.sbl +22 -10
- data/vendor/snowball/algorithms/english.sbl +7 -4
- data/vendor/snowball/algorithms/esperanto.sbl +6 -7
- data/vendor/snowball/algorithms/estonian.sbl +9 -4
- data/vendor/snowball/algorithms/finnish.sbl +33 -21
- data/vendor/snowball/algorithms/german.sbl +5 -0
- data/vendor/snowball/algorithms/indonesian.sbl +115 -96
- data/vendor/snowball/algorithms/italian.sbl +26 -1
- data/vendor/snowball/algorithms/lithuanian.sbl +13 -13
- data/vendor/snowball/algorithms/norwegian.sbl +17 -6
- data/vendor/snowball/algorithms/persian.sbl +387 -0
- data/vendor/snowball/algorithms/polish.sbl +177 -0
- data/vendor/snowball/algorithms/sesotho.sbl +115 -0
- data/vendor/snowball/algorithms/turkish.sbl +4 -4
- data/vendor/snowball/compiler/analyser.c +2002 -711
- data/vendor/snowball/compiler/driver.c +460 -298
- data/vendor/snowball/compiler/generator.c +439 -1965
- data/vendor/snowball/compiler/generator_ada.c +605 -430
- data/vendor/snowball/compiler/generator_c.c +2227 -0
- data/vendor/snowball/compiler/generator_csharp.c +365 -274
- data/vendor/snowball/compiler/generator_dart.c +1476 -0
- data/vendor/snowball/compiler/generator_go.c +348 -270
- data/vendor/snowball/compiler/generator_java.c +330 -233
- data/vendor/snowball/compiler/generator_js.c +534 -385
- data/vendor/snowball/compiler/generator_pascal.c +361 -320
- data/vendor/snowball/compiler/generator_php.c +1445 -0
- data/vendor/snowball/compiler/generator_python.c +399 -299
- data/vendor/snowball/compiler/generator_rust.c +370 -261
- data/vendor/snowball/compiler/generator_zig.c +1474 -0
- data/vendor/snowball/compiler/header.h +153 -86
- data/vendor/snowball/compiler/space.c +121 -63
- data/vendor/snowball/compiler/tokeniser.c +286 -182
- data/vendor/snowball/compiler/tokens.h +71 -0
- data/vendor/snowball/csharp/Snowball/Among.cs +14 -26
- data/vendor/snowball/csharp/Snowball/AssemblyInfo.cs +3 -3
- data/vendor/snowball/csharp/Snowball/Stemmer.cs +41 -45
- data/vendor/snowball/csharp/Stemwords/Program.cs +20 -14
- data/vendor/snowball/cxx/.gitignore +4 -0
- data/vendor/snowball/cxx/generate_algorithms.pl +87 -0
- data/vendor/snowball/cxx/stemmer.h +12 -0
- data/vendor/snowball/cxx/stemwords.cxx +176 -0
- data/vendor/snowball/cxx/utilities.cxx +2 -0
- data/vendor/snowball/dart/.gitignore +4 -0
- data/vendor/snowball/dart/analysis_options.yaml +1 -0
- data/vendor/snowball/dart/example/test_app.dart +65 -0
- data/vendor/snowball/dart/generate_algorithms.pl +21 -0
- data/vendor/snowball/dart/lib/snowball.dart +18 -0
- data/vendor/snowball/dart/lib/src/snowball.dart +339 -0
- data/vendor/snowball/dart/pubspec.yaml +17 -0
- data/vendor/snowball/doc/libstemmer_dart_README +37 -0
- data/vendor/snowball/doc/libstemmer_js_README +2 -2
- data/vendor/snowball/doc/libstemmer_php_README +38 -0
- data/vendor/snowball/doc/libstemmer_python_README +3 -3
- data/vendor/snowball/examples/stemwords.c +18 -3
- data/vendor/snowball/go/README.md +12 -4
- data/vendor/snowball/go/env.go +12 -14
- data/vendor/snowball/go/stemwords/main.go +2 -1
- data/vendor/snowball/java/org/tartarus/snowball/CharArraySequence.java +36 -0
- data/vendor/snowball/java/org/tartarus/snowball/SnowballProgram.java +17 -34
- data/vendor/snowball/java/org/tartarus/snowball/SnowballStemmer.java +0 -2
- data/vendor/snowball/javascript/base-stemmer.js +219 -252
- data/vendor/snowball/javascript/stemwords.js +86 -58
- data/vendor/snowball/libstemmer/mkmodules.pl +1 -1
- data/vendor/snowball/libstemmer/modules.txt +4 -0
- data/vendor/snowball/pascal/SnowballProgram.pas +18 -31
- data/vendor/snowball/php/base-stemmer.php +453 -0
- data/vendor/snowball/php/stemwords.php +25 -0
- data/vendor/snowball/python/create_init.py +16 -21
- data/vendor/snowball/python/pyproject.toml +3 -0
- data/vendor/snowball/python/setup.py +5 -7
- data/vendor/snowball/python/snowballstemmer/among.py +1 -2
- data/vendor/snowball/python/snowballstemmer/basestemmer.py +24 -33
- data/vendor/snowball/python/stemwords.py +72 -63
- data/vendor/snowball/runtime/api.c +11 -42
- data/vendor/snowball/runtime/api.h +11 -9
- data/vendor/snowball/runtime/snowball_runtime.h +110 -0
- data/vendor/snowball/runtime/utilities.c +282 -106
- data/vendor/snowball/rust/src/main.rs +2 -2
- data/vendor/snowball/rust/src/snowball/snowball_env.rs +7 -9
- data/vendor/snowball/tests/compilertest +62 -0
- data/vendor/snowball/tests/errors/ae-errors.sbl +53 -0
- data/vendor/snowball/tests/errors/ae-errors.stderr +8 -0
- data/vendor/snowball/tests/errors/bad-dollar.sbl +14 -0
- data/vendor/snowball/tests/errors/bad-dollar.stderr +4 -0
- data/vendor/snowball/tests/errors/bad-grouping-definition.sbl +9 -0
- data/vendor/snowball/tests/errors/bad-grouping-definition.stderr +6 -0
- data/vendor/snowball/tests/errors/missing-bra.sbl +6 -0
- data/vendor/snowball/tests/errors/missing-bra.stderr +3 -0
- data/vendor/snowball/tests/errors/missing-command.sbl +9 -0
- data/vendor/snowball/tests/errors/missing-command.stderr +1 -0
- data/vendor/snowball/tests/errors/missing-ket-backwardmode.sbl +3 -0
- data/vendor/snowball/tests/errors/missing-ket-backwardmode.stderr +1 -0
- data/vendor/snowball/tests/errors/missing-ket.sbl +7 -0
- data/vendor/snowball/tests/errors/missing-ket.stderr +6 -0
- data/vendor/snowball/tests/errors/notdefined.sbl +4 -0
- data/vendor/snowball/tests/errors/notdefined.stderr +2 -0
- data/vendor/snowball/tests/errors/string-omitted.sbl +5 -0
- data/vendor/snowball/tests/errors/string-omitted.stderr +1 -0
- data/vendor/snowball/tests/errors/undeclared.sbl +20 -0
- data/vendor/snowball/tests/errors/undeclared.stderr +9 -0
- data/vendor/snowball/tests/errors/unexpected-token.sbl +24 -0
- data/vendor/snowball/tests/errors/unexpected-token.stderr +17 -0
- data/vendor/snowball/tests/errors/wrongdirection.sbl +14 -0
- data/vendor/snowball/tests/errors/wrongdirection.stderr +1 -0
- data/vendor/snowball/tests/runtime/among.sbl +19 -0
- data/vendor/snowball/tests/runtime/arithmeticexpr.sbl +86 -0
- data/vendor/snowball/tests/runtime/attachinsert.sbl +47 -0
- data/vendor/snowball/tests/runtime/booleans.sbl +19 -0
- data/vendor/snowball/tests/runtime/externals.sbl +22 -0
- data/vendor/snowball/tests/runtime/hop.sbl +12 -0
- data/vendor/snowball/tests/runtime/integertests.sbl +60 -0
- data/vendor/snowball/tests/runtime/intlimits.sbl +25 -0
- data/vendor/snowball/tests/runtime/naming.sbl +20 -0
- data/vendor/snowball/tests/runtime/not.sbl +30 -0
- data/vendor/snowball/tests/runtime/or.sbl +17 -0
- data/vendor/snowball/tests/runtime/repeat.sbl +16 -0
- data/vendor/snowball/tests/runtime/setlimit.sbl +24 -0
- data/vendor/snowball/tests/runtime/sizelen.sbl +56 -0
- data/vendor/snowball/tests/runtime/slice.sbl +27 -0
- data/vendor/snowball/tests/runtime/stringdollar.sbl +39 -0
- data/vendor/snowball/tests/runtime/strings.sbl +66 -0
- data/vendor/snowball/tests/runtime/test.sbl +19 -0
- data/vendor/snowball/tests/stemtest.c +41 -1
- data/vendor/snowball/tests/syntax/canon.sbl +32 -0
- data/vendor/snowball/tests/syntax/canon.stderr +0 -0
- data/vendor/snowball/tests/syntax/canon.syntax +57 -0
- data/vendor/snowball/tests/syntax/emptyprogram.sbl +2 -0
- data/vendor/snowball/tests/syntax/emptyprogram.syntax +0 -0
- data/vendor/snowball/tests/syntax/groupings.sbl +11 -0
- data/vendor/snowball/tests/syntax/inline.sbl +144 -0
- data/vendor/snowball/tests/syntax/inline.stderr +0 -0
- data/vendor/snowball/tests/syntax/inline.syntax +121 -0
- data/vendor/snowball/tests/syntax/legacy.sbl +17 -0
- data/vendor/snowball/tests/syntax/legacy.stderr +5 -0
- data/vendor/snowball/tests/syntax/legacy.syntax +19 -0
- data/vendor/snowball/tests/syntax/loops.sbl +14 -0
- data/vendor/snowball/tests/syntax/loops.stderr +9 -0
- data/vendor/snowball/tests/syntax/loops.syntax +29 -0
- data/vendor/snowball/tests/syntax/noops.sbl +76 -0
- data/vendor/snowball/tests/syntax/noops.stderr +27 -0
- data/vendor/snowball/tests/syntax/noops.syntax +135 -0
- data/vendor/snowball/tests/syntax/simplifyae.sbl +21 -0
- data/vendor/snowball/tests/syntax/simplifyae.stderr +0 -0
- data/vendor/snowball/tests/syntax/simplifyae.syntax +39 -0
- data/vendor/snowball/tests/syntax/unused.sbl +18 -0
- data/vendor/snowball/tests/syntax/unused.stderr +7 -0
- data/vendor/snowball/tests/syntax/unused.syntax +3 -0
- data/vendor/snowball/zig/env.zig +599 -0
- data/vendor/snowball/zig/generate_algorithms.pl +19 -0
- data/vendor/snowball/zig/stemwords.zig +123 -0
- metadata +102 -4
- data/vendor/snowball/compiler/syswords.h +0 -86
- data/vendor/snowball/runtime/header.h +0 -62
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: b5d5216e3e6a0e86aa92ab80ec8876e66be4d766d500d82609f166452f28fb08
|
|
4
|
+
data.tar.gz: f5db98b8a25cc9fb2183a2f0010695239a66c82faf00e1ddb34355739087c48b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 886dac65964a1b37c5156ff27b7703a71d6acde7f01c9525d1a564636b1fdfa99bf8871b98e8628ed511cdda536e4d28d7e5bebbefdb9b0a67454346e1f571c7
|
|
7
|
+
data.tar.gz: 62a9a20fca62714a28826626183dba6816d11863fb169da43c0fa7d35d2a7978704f4764be63bce5de1986b188566f15f7ec5be5a0d415d9a85fad7285b2f1ac
|
data/CHANGELOG.md
CHANGED
data/Gemfile
CHANGED
data/README.md
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Stemming for Ruby, powered by [Snowball](https://github.com/snowballstem/snowball)
|
|
4
4
|
|
|
5
|
-
:snowflake: Supports 30 languages
|
|
5
|
+
:snowflake: Supports 30+ languages
|
|
6
6
|
|
|
7
7
|
[](https://github.com/ankane/mittens/actions)
|
|
8
8
|
|
|
@@ -36,7 +36,7 @@ Specify the language
|
|
|
36
36
|
stemmer = Mittens::Stemmer.new(language: "french")
|
|
37
37
|
```
|
|
38
38
|
|
|
39
|
-
Supports `arabic`, `armenian`, `basque`, `catalan`, `danish`, `dutch`, `dutch_porter`, `english`, `esperanto`, `estonian`, `finnish`, `french`, `german`, `greek`, `hindi`, `hungarian`, `indonesian`, `irish`, `italian`, `lithuanian`, `nepali`, `norwegian`, `porter`, `portuguese`, `romanian`, `russian`, `serbian`, `spanish`, `swedish`, `tamil`, `turkish`, and `yiddish`
|
|
39
|
+
Supports `arabic`, `armenian`, `basque`, `catalan`, `czech`, `danish`, `dutch`, `dutch_porter`, `english`, `esperanto`, `estonian`, `finnish`, `french`, `german`, `greek`, `hindi`, `hungarian`, `indonesian`, `irish`, `italian`, `lithuanian`, `nepali`, `norwegian`, `persian`, `polish`, `porter`, `portuguese`, `romanian`, `russian`, `serbian`, `sesotho`, `spanish`, `swedish`, `tamil`, `turkish`, and `yiddish`
|
|
40
40
|
|
|
41
41
|
## History
|
|
42
42
|
|
data/Rakefile
CHANGED
|
@@ -1,12 +1,18 @@
|
|
|
1
1
|
require "bundler/gem_tasks"
|
|
2
2
|
require "rake/testtask"
|
|
3
3
|
require "rake/extensiontask"
|
|
4
|
+
require "ruby_memcheck"
|
|
4
5
|
|
|
5
|
-
|
|
6
|
-
Rake::TestTask.new do |t|
|
|
7
|
-
t.libs << "test"
|
|
6
|
+
test_config = lambda do |t|
|
|
8
7
|
t.pattern = "test/**/*_test.rb"
|
|
9
8
|
end
|
|
9
|
+
Rake::TestTask.new(&test_config)
|
|
10
|
+
|
|
11
|
+
namespace :test do
|
|
12
|
+
RubyMemcheck::TestTask.new(:valgrind, &test_config)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
task default: :test
|
|
10
16
|
|
|
11
17
|
Rake::ExtensionTask.new("mittens") do |ext|
|
|
12
18
|
ext.name = "ext"
|
|
@@ -14,8 +20,9 @@ Rake::ExtensionTask.new("mittens") do |ext|
|
|
|
14
20
|
end
|
|
15
21
|
|
|
16
22
|
task :remove_ext do
|
|
17
|
-
|
|
18
|
-
|
|
23
|
+
Dir["lib/mittens/ext.{bundle,so}"].each do |path|
|
|
24
|
+
File.unlink(path)
|
|
25
|
+
end
|
|
19
26
|
end
|
|
20
27
|
|
|
21
28
|
Rake::Task["build"].enhance [:remove_ext]
|
data/ext/mittens/extconf.rb
CHANGED
|
@@ -4,7 +4,9 @@ require "open3"
|
|
|
4
4
|
vendor = File.expand_path("../../vendor/snowball", __dir__)
|
|
5
5
|
# CFLAGS from vendor/snowball/GNUmakefile and -fPIC
|
|
6
6
|
cflags = "-O2 -W -Wall -Wmissing-prototypes -Wmissing-declarations -fPIC"
|
|
7
|
-
|
|
7
|
+
|
|
8
|
+
make_cmd = RbConfig::CONFIG["host_os"] =~ /bsd|dragonfly/i ? "gmake" : "make"
|
|
9
|
+
output, status = Open3.capture2(make_cmd, "CFLAGS=#{cflags}", chdir: vendor)
|
|
8
10
|
puts output
|
|
9
11
|
raise "Command failed" unless status.success?
|
|
10
12
|
|
data/lib/mittens/version.rb
CHANGED
|
@@ -11,10 +11,14 @@ on:
|
|
|
11
11
|
- '*.rst'
|
|
12
12
|
- NEWS
|
|
13
13
|
pull_request:
|
|
14
|
-
branches:
|
|
14
|
+
branches: main
|
|
15
15
|
paths-ignore:
|
|
16
16
|
- '*.rst'
|
|
17
17
|
- NEWS
|
|
18
|
+
workflow_call:
|
|
19
|
+
inputs:
|
|
20
|
+
runtime-tests: # We reuse this workflow from runtime-tests.yml.
|
|
21
|
+
type: boolean
|
|
18
22
|
|
|
19
23
|
# Allows you to run this workflow manually from the Actions tab
|
|
20
24
|
workflow_dispatch:
|
|
@@ -25,21 +29,33 @@ jobs:
|
|
|
25
29
|
matrix:
|
|
26
30
|
include:
|
|
27
31
|
- name: "C distribution build"
|
|
28
|
-
CFLAGS_DIST_BUILD: '-O2 -Wall -W -std=c90 -Wmissing-prototypes -Wmissing-declarations -Wshadow -
|
|
32
|
+
CFLAGS_DIST_BUILD: '-O2 -Wall -W -std=c90 -pedantic -Wmissing-prototypes -Wmissing-declarations -Wshadow -Werror'
|
|
29
33
|
- name: "C distribution build (clang)"
|
|
30
|
-
CFLAGS_DIST_BUILD: '-O2 -Wall -W -std=c90 -Wmissing-prototypes -Wmissing-declarations -Wshadow -
|
|
34
|
+
CFLAGS_DIST_BUILD: '-O2 -Wall -W -std=c90 -pedantic -Wmissing-prototypes -Wmissing-declarations -Wshadow -Werror'
|
|
31
35
|
CC: clang
|
|
32
36
|
- name: "C"
|
|
33
|
-
c_tests:
|
|
34
|
-
WERROR: '-std=c99 -Werror'
|
|
35
|
-
- name: "C (clang)"
|
|
36
|
-
|
|
37
|
-
|
|
37
|
+
c_tests: check
|
|
38
|
+
WERROR: '-std=c99 -pedantic -Werror'
|
|
39
|
+
- name: "C (clang+sanitisers)"
|
|
40
|
+
# Generate code for all target languages so we exercise the generator
|
|
41
|
+
# code for each target language with sanitisers on.
|
|
42
|
+
c_tests: "generate check"
|
|
43
|
+
# Overflow of unsigned types is defined behaviour, but we don't
|
|
44
|
+
# expect it to happen in the Snowball compiler or C runtime code
|
|
45
|
+
# so enable `unsigned-integer-overflow` too.
|
|
46
|
+
WERROR: '-std=c99 -pedantic -Werror -fsanitize=address,undefined,float-divide-by-zero,local-bounds,nullability,unsigned-integer-overflow -fsanitize-address-use-after-scope -fsanitize-recover=all -fno-omit-frame-pointer'
|
|
38
47
|
CC: clang
|
|
48
|
+
- name: "C++"
|
|
49
|
+
CXX: g++
|
|
50
|
+
WERROR: '-pedantic -Werror'
|
|
39
51
|
- name: java
|
|
40
52
|
JAVA: java
|
|
41
53
|
JAVAC: javac
|
|
42
54
|
JAVACFLAGS: '-Xlint:all -Werror'
|
|
55
|
+
- name: dart
|
|
56
|
+
DART: dart
|
|
57
|
+
os: 'ubuntu-24.04'
|
|
58
|
+
dart_sdk: stable
|
|
43
59
|
- name: go_old
|
|
44
60
|
os: 'ubuntu-22.04'
|
|
45
61
|
apt_packages: 'golang-1.13'
|
|
@@ -51,15 +67,32 @@ jobs:
|
|
|
51
67
|
- name: javascript_node
|
|
52
68
|
JSRUN: node
|
|
53
69
|
apt_packages: 'nodejs'
|
|
70
|
+
- name: javascript_deno
|
|
71
|
+
JSRUN: 'deno --allow-read --allow-write'
|
|
72
|
+
apt_packages: 'nodejs'
|
|
54
73
|
- name: rust
|
|
55
74
|
RUST: rust
|
|
56
75
|
apt_packages: 'rustc'
|
|
76
|
+
- name: zig
|
|
77
|
+
ZIG: zig
|
|
57
78
|
- name: csharp
|
|
58
79
|
MCS: mcs
|
|
59
80
|
apt_packages: 'mono-devel'
|
|
60
81
|
- name: Pascal
|
|
61
82
|
FPC: fpc
|
|
62
83
|
apt_packages: 'fpc'
|
|
84
|
+
- name: PHP 8.5
|
|
85
|
+
PHP: php
|
|
86
|
+
apt_extra_repo: ppa:ondrej/php
|
|
87
|
+
apt_packages: 'php8.5-cli php8.5-mbstring'
|
|
88
|
+
- name: PHP 8.4
|
|
89
|
+
PHP: php
|
|
90
|
+
apt_extra_repo: ppa:ondrej/php
|
|
91
|
+
apt_packages: 'php8.4-cli php8.4-mbstring'
|
|
92
|
+
- name: PHP 8.3
|
|
93
|
+
PHP: php
|
|
94
|
+
os: 'ubuntu-24.04'
|
|
95
|
+
apt_packages: 'php8.3-cli php8.3-mbstring'
|
|
63
96
|
- name: Python 3.8
|
|
64
97
|
PYTHON_VERSION: 3.8
|
|
65
98
|
os: 'ubuntu-22.04'
|
|
@@ -87,7 +120,7 @@ jobs:
|
|
|
87
120
|
apt_packages: 'gnat gprbuild'
|
|
88
121
|
- name: Windows (C)
|
|
89
122
|
os: windows-latest
|
|
90
|
-
c_tests:
|
|
123
|
+
c_tests: check
|
|
91
124
|
ccache: sccache
|
|
92
125
|
- name: Windows (Go)
|
|
93
126
|
os: windows-latest
|
|
@@ -102,25 +135,27 @@ jobs:
|
|
|
102
135
|
|
|
103
136
|
env:
|
|
104
137
|
CC: ${{ matrix.CC || 'gcc' }}
|
|
105
|
-
MAKE: ${{ matrix.MAKE || 'make' }}
|
|
138
|
+
MAKE: ${{ matrix.MAKE || 'make' }} --output-sync --jobs=4
|
|
106
139
|
STEMMING_DATA: 'snowball-data'
|
|
140
|
+
WERROR: ${{ matrix.WERROR }}
|
|
107
141
|
|
|
108
142
|
steps:
|
|
109
143
|
- name: Checkout
|
|
110
|
-
uses: actions/checkout@
|
|
144
|
+
uses: actions/checkout@v5
|
|
111
145
|
with:
|
|
112
146
|
show-progress: false
|
|
113
147
|
|
|
114
148
|
- name: Checkout data
|
|
149
|
+
if: ${{ ! inputs.runtime-tests }}
|
|
115
150
|
run: |
|
|
116
151
|
# Try to check out a branch of the same name from the snowball-data
|
|
117
152
|
# repo sibling of this snowball repo, so that PRs requiring changes to
|
|
118
153
|
# both can be CI tested easily.
|
|
119
154
|
#
|
|
120
|
-
# For a PR, GHA will have merged the PR branch into upstream
|
|
155
|
+
# For a PR, GHA will have merged the PR branch into upstream main so
|
|
121
156
|
# we need to similarly merge the snowball-data branch into upstream
|
|
122
|
-
#
|
|
123
|
-
# required by snowball
|
|
157
|
+
# main of the snowball-data repo as there may be changes there
|
|
158
|
+
# required by snowball main.
|
|
124
159
|
#
|
|
125
160
|
# If there's no such branch (or repo) we just use the standard
|
|
126
161
|
# snowball-data repo's default branch. If there is such a branch but
|
|
@@ -153,16 +188,40 @@ jobs:
|
|
|
153
188
|
git clone "$UPSTREAM_REPO_URL"
|
|
154
189
|
fi
|
|
155
190
|
fi
|
|
191
|
+
- name: Hook up runtime testing
|
|
192
|
+
if: ${{ inputs.runtime-tests }}
|
|
193
|
+
run: |
|
|
194
|
+
make setup_runtime_tests
|
|
156
195
|
- name: Install CCache
|
|
157
|
-
uses: hendrikmuhs/ccache-action@v1
|
|
196
|
+
uses: hendrikmuhs/ccache-action@v1.2.21
|
|
158
197
|
with:
|
|
159
198
|
key: ${{ matrix.name }}
|
|
160
199
|
variant: ${{ matrix.ccache || 'ccache' }}
|
|
200
|
+
- name: Enable extra Ubuntu package repo
|
|
201
|
+
if: matrix.apt_extra_repo
|
|
202
|
+
run: |
|
|
203
|
+
sudo apt-get -qq remove "php*-cli" "php*-dev"
|
|
204
|
+
sudo add-apt-repository -y ${{ matrix.apt_extra_repo }}
|
|
205
|
+
sudo apt-get update
|
|
161
206
|
- name: Install Ubuntu packages
|
|
162
207
|
if: matrix.apt_packages
|
|
163
208
|
run: |
|
|
164
209
|
sudo apt-get update
|
|
165
210
|
sudo apt-get install -y ${{ matrix.apt_packages }}
|
|
211
|
+
- uses: denoland/setup-deno@v2
|
|
212
|
+
if: ${{ startsWith(matrix.JSRUN, 'deno') }}
|
|
213
|
+
with:
|
|
214
|
+
deno-version: vx.x.x
|
|
215
|
+
- name: Install Dart SDK
|
|
216
|
+
if: matrix.dart_sdk
|
|
217
|
+
uses: dart-lang/setup-dart@v1.7.2
|
|
218
|
+
with:
|
|
219
|
+
sdk: ${{ matrix.dart_sdk }}
|
|
220
|
+
- name: Install Zig
|
|
221
|
+
if: matrix.ZIG
|
|
222
|
+
uses: mlugg/setup-zig@v2
|
|
223
|
+
with:
|
|
224
|
+
version: 0.16.0
|
|
166
225
|
- name: Install mingw64 packages
|
|
167
226
|
if: matrix.mingw64_packages
|
|
168
227
|
uses: msys2/setup-msys2@v2
|
|
@@ -181,10 +240,21 @@ jobs:
|
|
|
181
240
|
tar xf ../dist/libstemmer_c-*.tar.gz
|
|
182
241
|
cd libstemmer_c-*
|
|
183
242
|
$MAKE CFLAGS="${{ matrix.CFLAGS_DIST_BUILD }}"
|
|
243
|
+
cd ../..
|
|
244
|
+
rm -rf tmp
|
|
184
245
|
- name: Test C
|
|
185
246
|
if: matrix.c_tests
|
|
186
|
-
run:
|
|
187
|
-
|
|
247
|
+
run: |
|
|
248
|
+
# Kludge so "generate" target doesn't need ada installed.
|
|
249
|
+
case '${{ matrix.c_tests }}' in
|
|
250
|
+
*generate*) mkdir ada/bin && ln -s /bin/true ada/bin/generate ;;
|
|
251
|
+
esac
|
|
252
|
+
$MAKE ${{ matrix.c_tests }} CC="$CC"
|
|
253
|
+
- name: Test C++
|
|
254
|
+
if: matrix.CXX
|
|
255
|
+
run: |
|
|
256
|
+
$MAKE check_cxx CXX='${{ matrix.CXX }}'
|
|
257
|
+
- uses: actions/setup-python@v6
|
|
188
258
|
with:
|
|
189
259
|
python-version: ${{ matrix.PYTHON_VERSION }}
|
|
190
260
|
if: matrix.PYTHON_VERSION
|
|
@@ -194,12 +264,20 @@ jobs:
|
|
|
194
264
|
- name: Test Java
|
|
195
265
|
if: matrix.JAVA && matrix.JAVAC
|
|
196
266
|
run: $MAKE check_java JAVA="${{ matrix.JAVA }}" JAVAC="${{ matrix.JAVAC }}" JAVACFLAGS="${{ matrix.JAVACFLAGS }}"
|
|
267
|
+
- name: Test Dart
|
|
268
|
+
if: matrix.DART
|
|
269
|
+
run: $MAKE check_dart DART="${{ matrix.DART }}"
|
|
197
270
|
- name: Test C#
|
|
198
271
|
if: matrix.MCS
|
|
199
272
|
run: $MAKE check_csharp MCS="${{ matrix.MCS }}"
|
|
200
273
|
- name: Test Javascript
|
|
201
274
|
if: matrix.JSRUN
|
|
202
275
|
run: $MAKE check_js JSRUN="${{ matrix.JSRUN }}"
|
|
276
|
+
- name: Lint and check Javascript code
|
|
277
|
+
if: ${{ startsWith(matrix.JSRUN, 'deno') }}
|
|
278
|
+
run: |
|
|
279
|
+
deno lint javascript/*.js js_out/*.js
|
|
280
|
+
deno check javascript/*.js js_out/*.js
|
|
203
281
|
- name: Test Rust
|
|
204
282
|
if: matrix.RUST
|
|
205
283
|
run: $MAKE check_rust RUST="${{ matrix.RUST }}"
|
|
@@ -208,9 +286,58 @@ jobs:
|
|
|
208
286
|
run: |
|
|
209
287
|
go mod init github.com/snowballstem/snowball
|
|
210
288
|
$MAKE check_go GO="${{ matrix.GO }}"
|
|
289
|
+
rm -f go.mod
|
|
211
290
|
- name: Test Pascal
|
|
212
291
|
if: matrix.FPC
|
|
213
292
|
run: $MAKE check_pascal FPC="${{ matrix.FPC }}"
|
|
293
|
+
- name: Test PHP
|
|
294
|
+
if: matrix.PHP
|
|
295
|
+
run: |
|
|
296
|
+
${{ matrix.PHP }} --version
|
|
297
|
+
$MAKE check_php PHP="${{ matrix.PHP }}"
|
|
298
|
+
- name: Test Zig
|
|
299
|
+
if: matrix.ZIG
|
|
300
|
+
run: |
|
|
301
|
+
$MAKE check_zig zig="${{ matrix.ZIG }}"
|
|
302
|
+
rm -rf "$ZIG_LOCAL_CACHE_DIR"
|
|
214
303
|
- name: Test Ada
|
|
215
304
|
if: matrix.gprbuild
|
|
216
305
|
run: $MAKE check_ada gprbuild="${{ matrix.gprbuild }}"
|
|
306
|
+
- name: make clean
|
|
307
|
+
run: |
|
|
308
|
+
rm -rf '${{ env.STEMMING_DATA }}'
|
|
309
|
+
# `.ccache`/`.sccache` is specific to CI. We can't just delete it here
|
|
310
|
+
# or else ccache-action won't save the cache.
|
|
311
|
+
git status --porcelain=v1|grep -v '^?? \.s\?ccache/$'|grep '^' && { echo 'The generated files listed above are not in .gitignore' ; exit 1; }; true
|
|
312
|
+
$MAKE clean ${{ case(inputs.runtime-tests == true, 'clean_runtime_tests', '') }}
|
|
313
|
+
git status --porcelain=v1 --ignored|grep -v '^?? \.s\?ccache/$'|grep '^' && { echo 'The generated files listed above were not removed by `make clean`' ; exit 1; }; true
|
|
314
|
+
|
|
315
|
+
pypi-publish:
|
|
316
|
+
if: >
|
|
317
|
+
github.event_name == 'push' &&
|
|
318
|
+
startsWith(github.ref, 'refs/tags/v') &&
|
|
319
|
+
inputs.runtime-tests != true
|
|
320
|
+
needs:
|
|
321
|
+
- build
|
|
322
|
+
runs-on: ubuntu-latest
|
|
323
|
+
environment:
|
|
324
|
+
name: pypi
|
|
325
|
+
url: https://pypi.org/p/snowballstemmer
|
|
326
|
+
permissions:
|
|
327
|
+
id-token: write
|
|
328
|
+
steps:
|
|
329
|
+
- name: Checkout
|
|
330
|
+
uses: actions/checkout@v6
|
|
331
|
+
with:
|
|
332
|
+
show-progress: false
|
|
333
|
+
- name: Set up Python
|
|
334
|
+
uses: actions/setup-python@v6
|
|
335
|
+
with:
|
|
336
|
+
python-version: '3.14'
|
|
337
|
+
- name: Install build module
|
|
338
|
+
run: pip install build
|
|
339
|
+
- name: Generate dist
|
|
340
|
+
run: make --output-sync --jobs=4 dist_libstemmer_python
|
|
341
|
+
- name: Publish to PyPI
|
|
342
|
+
if: success()
|
|
343
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
name: Snowball code coverage
|
|
2
|
+
|
|
3
|
+
# Use bash by default on all platforms.
|
|
4
|
+
defaults:
|
|
5
|
+
run:
|
|
6
|
+
shell: bash
|
|
7
|
+
|
|
8
|
+
on:
|
|
9
|
+
push:
|
|
10
|
+
paths:
|
|
11
|
+
- 'algorithms/*.sbl'
|
|
12
|
+
pull_request:
|
|
13
|
+
branches: main
|
|
14
|
+
paths:
|
|
15
|
+
- 'algorithms/*.sbl'
|
|
16
|
+
|
|
17
|
+
# Allows you to run this workflow manually from the Actions tab
|
|
18
|
+
workflow_dispatch:
|
|
19
|
+
|
|
20
|
+
jobs:
|
|
21
|
+
coverage:
|
|
22
|
+
name: "Snowball code coverage"
|
|
23
|
+
runs-on: ubuntu-24.04
|
|
24
|
+
|
|
25
|
+
env:
|
|
26
|
+
CC: gcc
|
|
27
|
+
MAKE: make --output-sync --jobs=4
|
|
28
|
+
STEMMING_DATA: 'snowball-data'
|
|
29
|
+
|
|
30
|
+
steps:
|
|
31
|
+
- name: Checkout
|
|
32
|
+
uses: actions/checkout@v5
|
|
33
|
+
with:
|
|
34
|
+
show-progress: false
|
|
35
|
+
|
|
36
|
+
- name: Checkout data
|
|
37
|
+
run: |
|
|
38
|
+
# Try to check out a branch of the same name from the snowball-data
|
|
39
|
+
# repo sibling of this snowball repo, so that PRs requiring changes to
|
|
40
|
+
# both can be CI tested easily.
|
|
41
|
+
#
|
|
42
|
+
# For a PR, GHA will have merged the PR branch into upstream main so
|
|
43
|
+
# we need to similarly merge the snowball-data branch into upstream
|
|
44
|
+
# main of the snowball-data repo as there may be changes there
|
|
45
|
+
# required by snowball main.
|
|
46
|
+
#
|
|
47
|
+
# If there's no such branch (or repo) we just use the standard
|
|
48
|
+
# snowball-data repo's default branch. If there is such a branch but
|
|
49
|
+
# the merge fails, we treat that as a fatal error.
|
|
50
|
+
UPSTREAM_REPO_URL=https://github.com/snowballstem/snowball-data.git
|
|
51
|
+
if [ -n "$GITHUB_HEAD_REF" ] ; then
|
|
52
|
+
# Pull-request.
|
|
53
|
+
GH_BRANCH=${GITHUB_HEAD_REF}
|
|
54
|
+
GH_REPO_OWNER=${GITHUB_ACTOR}
|
|
55
|
+
GH_REPO_URL=https://github.com/$GH_REPO_OWNER/snowball-data.git
|
|
56
|
+
git clone "$UPSTREAM_REPO_URL"
|
|
57
|
+
cd snowball-data
|
|
58
|
+
git remote add pr "$GH_REPO_URL"
|
|
59
|
+
git config --global user.email "ci@example.org"
|
|
60
|
+
git config --global user.name "CI"
|
|
61
|
+
echo "Trying branch $GH_BRANCH from $GH_REPO_URL"
|
|
62
|
+
if git fetch pr && git branch --track "$GH_BRANCH" pr/"$GH_BRANCH" ; then
|
|
63
|
+
git merge "$GH_BRANCH"
|
|
64
|
+
else
|
|
65
|
+
echo "Falling back to $UPSTREAM_REPO_URL"
|
|
66
|
+
fi
|
|
67
|
+
else
|
|
68
|
+
# Push.
|
|
69
|
+
GH_BRANCH=${GITHUB_REF_NAME}
|
|
70
|
+
GH_REPO_OWNER=${GITHUB_REPOSITORY_OWNER}
|
|
71
|
+
GH_REPO_URL=https://github.com/$GH_REPO_OWNER/snowball-data.git
|
|
72
|
+
echo "Trying branch $GH_BRANCH from $GH_REPO_URL"
|
|
73
|
+
if ! git clone -b "$GH_BRANCH" "$GH_REPO_URL" ; then
|
|
74
|
+
echo "Falling back to $UPSTREAM_REPO_URL"
|
|
75
|
+
git clone "$UPSTREAM_REPO_URL"
|
|
76
|
+
fi
|
|
77
|
+
fi
|
|
78
|
+
- name: Install CCache
|
|
79
|
+
uses: hendrikmuhs/ccache-action@v1.2.21
|
|
80
|
+
with:
|
|
81
|
+
key: ${{ matrix.name }}
|
|
82
|
+
variant: ${{ matrix.ccache || 'ccache' }}
|
|
83
|
+
- name: Build
|
|
84
|
+
run: $MAKE CC="ccache $CC" SNOWBALL_FLAGS=-coverage CPPFLAGS=-DSNOWBALL_COVERAGE
|
|
85
|
+
- name: Generate coverage reports
|
|
86
|
+
run: |
|
|
87
|
+
# Find commit to compare with:
|
|
88
|
+
if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ] ; then
|
|
89
|
+
# Manually triggered - generate coverage for all algorithms.
|
|
90
|
+
ls -1 algorithms|sed 's,\.sbl$,,p;d'
|
|
91
|
+
elif [ -n "$GITHUB_HEAD_REF" ] ; then
|
|
92
|
+
# Pull-request.
|
|
93
|
+
git diff --numstat "$GITHUB_BASE_REF"... algorithms|sed 's,^[0-9].*/\([^ ]*\)\.sbl$,\1,p;d'
|
|
94
|
+
else
|
|
95
|
+
# Push.
|
|
96
|
+
#
|
|
97
|
+
# Currently we just generate coverage for any .sbl sources which
|
|
98
|
+
# changed in the most recent commit. Ideally we'd diff against the
|
|
99
|
+
# branch pre-push but github doesn't seem to provide a way to get
|
|
100
|
+
# that information.
|
|
101
|
+
git show --numstat algorithms|sed 's,^[0-9].*/\([^ ]*\)\.sbl$,\1,p;d'
|
|
102
|
+
fi | while read a ; do
|
|
103
|
+
# This will skip coverage reports for arabic, which currently has far
|
|
104
|
+
# too large a test vocabulary, such that it needs to be gzipped to get
|
|
105
|
+
# github to accept it:
|
|
106
|
+
#
|
|
107
|
+
# https://github.com/snowballstem/snowball-data/issues/31
|
|
108
|
+
#
|
|
109
|
+
# That size means generating a coverage report for it takes something
|
|
110
|
+
# like 20-25 minutes.
|
|
111
|
+
if [ -f "${STEMMING_DATA}/$a/voc.txt" ] ; then
|
|
112
|
+
$MAKE -s "check_utf8_$a" 2> coverage.log
|
|
113
|
+
echo ; echo "=== Coverage report for $a ===" ; echo
|
|
114
|
+
sort -k2,2 -k3,3n -k3,3 -k5,5g coverage.log |
|
|
115
|
+
uniq -c |awk '{$1=sprintf("% 8d",$1-1);print}'
|
|
116
|
+
fi
|
|
117
|
+
done
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: Runtime tests
|
|
2
|
+
|
|
3
|
+
# Use bash by default on all platforms.
|
|
4
|
+
defaults:
|
|
5
|
+
run:
|
|
6
|
+
shell: bash
|
|
7
|
+
|
|
8
|
+
on:
|
|
9
|
+
push:
|
|
10
|
+
paths-ignore:
|
|
11
|
+
- '*.rst'
|
|
12
|
+
- NEWS
|
|
13
|
+
pull_request:
|
|
14
|
+
branches: main
|
|
15
|
+
paths-ignore:
|
|
16
|
+
- '*.rst'
|
|
17
|
+
- NEWS
|
|
18
|
+
|
|
19
|
+
# Allows you to run this workflow manually from the Actions tab
|
|
20
|
+
workflow_dispatch:
|
|
21
|
+
|
|
22
|
+
jobs:
|
|
23
|
+
build:
|
|
24
|
+
uses: ./.github/workflows/ci.yml
|
|
25
|
+
with:
|
|
26
|
+
runtime-tests: true
|
data/vendor/snowball/.gitignore
CHANGED
|
@@ -1,7 +1,19 @@
|
|
|
1
|
+
.DS_Store
|
|
2
|
+
*.exe
|
|
1
3
|
*.o
|
|
4
|
+
# Created by `setup_runtime_tests`:
|
|
5
|
+
/overrides.mk
|
|
6
|
+
/tmp_runtime_tests_snowball_data/
|
|
7
|
+
#
|
|
8
|
+
/algorithms.mk
|
|
9
|
+
/snowball
|
|
10
|
+
/stemtest
|
|
11
|
+
/dist
|
|
12
|
+
# Ada
|
|
13
|
+
/ada/algorithms/
|
|
2
14
|
/ada/bin/
|
|
3
15
|
/ada/obj/
|
|
4
|
-
|
|
16
|
+
# C
|
|
5
17
|
/libstemmer/libstemmer.c
|
|
6
18
|
/libstemmer/libstemmer_utf8.c
|
|
7
19
|
/libstemmer/mkinc.mak
|
|
@@ -9,18 +21,32 @@
|
|
|
9
21
|
/libstemmer/modules.h
|
|
10
22
|
/libstemmer/modules_utf8.h
|
|
11
23
|
/libstemmer.a
|
|
12
|
-
/snowball
|
|
13
24
|
/src_c
|
|
14
|
-
/stemtest
|
|
15
25
|
/stemwords
|
|
16
|
-
|
|
26
|
+
# C#
|
|
27
|
+
/csharp_stemwords
|
|
28
|
+
/csharp/Snowball/Algorithms/*.generated.cs
|
|
29
|
+
# Dart
|
|
30
|
+
/dart/lib/ext/
|
|
31
|
+
/dart/lib/src/algorithms.dart
|
|
32
|
+
# Go
|
|
33
|
+
/go/algorithms/
|
|
34
|
+
/go/stemwords/algorithms.go
|
|
35
|
+
# Java
|
|
17
36
|
/java/org/tartarus/snowball/ext/
|
|
37
|
+
/java/org/tartarus/snowball/*.class
|
|
38
|
+
# JS
|
|
18
39
|
/js_out
|
|
40
|
+
# PHP
|
|
41
|
+
/php_out
|
|
42
|
+
# Python
|
|
19
43
|
/python_check
|
|
20
44
|
/python_out
|
|
21
|
-
|
|
45
|
+
# Rust
|
|
22
46
|
/rust/Cargo.lock
|
|
23
47
|
/rust/src/snowball/algorithms/*.rs
|
|
24
48
|
/rust/target/
|
|
25
|
-
|
|
26
|
-
/
|
|
49
|
+
# Zig
|
|
50
|
+
/zig/*_stemmer.zig
|
|
51
|
+
/zig/algorithms.zig
|
|
52
|
+
/zig/stemwords
|
|
@@ -223,6 +223,13 @@ Key problems to solve
|
|
|
223
223
|
backends with special handling here which may be useful to look at
|
|
224
224
|
include Javascript, Pascal and Python.
|
|
225
225
|
|
|
226
|
+
* Although all the stemmers we currently ship define a single Snowball
|
|
227
|
+
`external` named `stem`, this should not be assumed. Also externals
|
|
228
|
+
should be callable from Snowball code as well as externally (see
|
|
229
|
+
the Go and Rust generators for examples of how to handle this - if
|
|
230
|
+
an external is called from Snowball code then it is generated as a
|
|
231
|
+
routine, with a shim external which just tail calls to this routine).
|
|
232
|
+
|
|
226
233
|
Don't hardcode algorithm names
|
|
227
234
|
------------------------------
|
|
228
235
|
|
data/vendor/snowball/COPYING
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Copyright (c) 2001, Dr Martin Porter
|
|
2
2
|
Copyright (c) 2004,2005, Richard Boulton
|
|
3
3
|
Copyright (c) 2013, Yoshiki Shibukawa
|
|
4
|
-
Copyright (c) 2006
|
|
4
|
+
Copyright (c) 2006-2025, Olly Betts
|
|
5
5
|
All rights reserved.
|
|
6
6
|
|
|
7
7
|
Redistribution and use in source and binary forms, with or without
|