mittens 0.3.1 → 0.3.2

data/vendor/snowball/zig/stemwords.zig

@@ -0,0 +1,123 @@
+const std = @import("std");
+const snowball = @import("env.zig");
+const alg = @import("algorithms.zig");
+
+fn findStemmer(name: []const u8) ?alg.StemFn {
+    for (alg.algorithms) |a| {
+        if (std.mem.eql(u8, a.name, name)) return a.stem;
+    }
+    return null;
+}
+
+fn readFile(io: std.Io, allocator: std.mem.Allocator, path: []const u8) ![]u8 {
+    const file = try std.Io.Dir.cwd().openFile(io, path, .{});
+    defer file.close(io);
+    const stat = try file.stat(io);
+    const size: usize = @intCast(stat.size);
+    const buf = try allocator.alloc(u8, size);
+    errdefer allocator.free(buf);
+    const n = try file.readPositionalAll(io, buf, 0);
+    return buf[0..n];
+}
+
+fn writeFile(io: std.Io, path: []const u8, data: []const u8) !void {
+    const file = try std.Io.Dir.cwd().createFile(io, path, .{});
+    defer file.close(io);
+    try file.writePositionalAll(io, data, 0);
+}
+
+pub fn main(init: std.process.Init) !void {
+    const io = init.io;
+    const allocator = init.gpa;
+
+    var args_iter = std.process.Args.Iterator.init(init.minimal.args);
+    _ = args_iter.next(); // skip program name
+
+    var language: ?[]const u8 = null;
+    var input_path: ?[]const u8 = null;
+    var output_path: ?[]const u8 = null;
+
+    while (args_iter.next()) |arg| {
+        if (std.mem.eql(u8, arg, "-l")) {
+            language = args_iter.next();
+        } else if (std.mem.eql(u8, arg, "-i")) {
+            input_path = args_iter.next();
+        } else if (std.mem.eql(u8, arg, "-o")) {
+            output_path = args_iter.next();
+        }
+    }
+
+    const lang = language orelse std.process.fatal("error: -l language required", .{});
+    const stem_fn = findStemmer(lang) orelse
+        std.process.fatal("error: unknown language '{s}'", .{lang});
+
+    var env = snowball.Env.init(allocator);
+    defer env.deinit();
+
+    if (input_path) |p| {
+        // File input: read all, process, write all
+        const input_data = try readFile(io, allocator, p);
+        defer allocator.free(input_data);
+
+        var output = std.ArrayList(u8).initCapacity(allocator, input_data.len) catch
+            std.process.fatal("out of memory", .{});
+        defer output.deinit(allocator);
+
+        var rest = input_data;
+        while (std.mem.indexOfScalar(u8, rest, '\n')) |nl| {
+            const line = rest[0..nl];
+            if (line.len > 0) {
+                try env.setCurrent(line);
+                _ = stem_fn(&env);
+                output.appendSlice(allocator, env.getCurrent()) catch
+                    std.process.fatal("out of memory", .{});
+            }
+            output.append(allocator, '\n') catch
+                std.process.fatal("out of memory", .{});
+            rest = rest[nl + 1 ..];
+        }
+        if (rest.len > 0) {
+            try env.setCurrent(rest);
+            _ = stem_fn(&env);
+            output.appendSlice(allocator, env.getCurrent()) catch
+                std.process.fatal("out of memory", .{});
+            output.append(allocator, '\n') catch
+                std.process.fatal("out of memory", .{});
+        }
+
+        if (output_path) |out_path| {
+            try writeFile(io, out_path, output.items);
+        } else {
+            var write_buf: [8192]u8 = undefined;
+            var writer = std.Io.File.stdout().writerStreaming(io, &write_buf);
+            try writer.interface.writeAll(output.items);
+            try writer.interface.flush();
+        }
+    } else {
+        // Stdin: process line by line, stream output
+        var read_buf: [8192]u8 = undefined;
+        var reader = std.Io.File.stdin().readerStreaming(io, &read_buf);
+
+        const out_file = if (output_path) |out_path|
+            try std.Io.Dir.cwd().createFile(io, out_path, .{})
+        else
+            null;
+        defer if (out_file) |f| f.close(io);
+
+        var write_buf: [8192]u8 = undefined;
+        var writer = if (out_file) |f|
+            f.writerStreaming(io, &write_buf)
+        else
+            std.Io.File.stdout().writerStreaming(io, &write_buf);
+
+        while (try reader.interface.takeDelimiter('\n')) |line| {
+            if (line.len > 0) {
+                try env.setCurrent(line);
+                _ = stem_fn(&env);
+                try writer.interface.writeAll(env.getCurrent());
+            }
+            try writer.interface.writeByte('\n');
+        }
+        try writer.interface.flush();
+    }
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mittens
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Andrew Kane
@@ -26,6 +26,8 @@ files:
 - lib/mittens/version.rb
 - mittens.gemspec
 - vendor/snowball/.github/workflows/ci.yml
+- vendor/snowball/.github/workflows/coverage.yml
+- vendor/snowball/.github/workflows/runtime-tests.yml
 - vendor/snowball/.gitignore
 - vendor/snowball/AUTHORS
 - vendor/snowball/CONTRIBUTING.rst
@@ -45,6 +47,7 @@ files:
 - vendor/snowball/algorithms/armenian.sbl
 - vendor/snowball/algorithms/basque.sbl
 - vendor/snowball/algorithms/catalan.sbl
+- vendor/snowball/algorithms/czech.sbl
 - vendor/snowball/algorithms/danish.sbl
 - vendor/snowball/algorithms/dutch.sbl
 - vendor/snowball/algorithms/dutch_porter.sbl
@@ -64,11 +67,14 @@ files:
 - vendor/snowball/algorithms/lovins.sbl
 - vendor/snowball/algorithms/nepali.sbl
 - vendor/snowball/algorithms/norwegian.sbl
+- vendor/snowball/algorithms/persian.sbl
+- vendor/snowball/algorithms/polish.sbl
 - vendor/snowball/algorithms/porter.sbl
 - vendor/snowball/algorithms/portuguese.sbl
 - vendor/snowball/algorithms/romanian.sbl
 - vendor/snowball/algorithms/russian.sbl
 - vendor/snowball/algorithms/serbian.sbl
+- vendor/snowball/algorithms/sesotho.sbl
 - vendor/snowball/algorithms/spanish.sbl
 - vendor/snowball/algorithms/swedish.sbl
 - vendor/snowball/algorithms/tamil.sbl
@@ -81,17 +87,21 @@ files:
 - vendor/snowball/compiler/driver.c
 - vendor/snowball/compiler/generator.c
 - vendor/snowball/compiler/generator_ada.c
+- vendor/snowball/compiler/generator_c.c
 - vendor/snowball/compiler/generator_csharp.c
+- vendor/snowball/compiler/generator_dart.c
 - vendor/snowball/compiler/generator_go.c
 - vendor/snowball/compiler/generator_java.c
 - vendor/snowball/compiler/generator_js.c
 - vendor/snowball/compiler/generator_pascal.c
+- vendor/snowball/compiler/generator_php.c
 - vendor/snowball/compiler/generator_python.c
 - vendor/snowball/compiler/generator_rust.c
+- vendor/snowball/compiler/generator_zig.c
 - vendor/snowball/compiler/header.h
 - vendor/snowball/compiler/space.c
-- vendor/snowball/compiler/syswords.h
 - vendor/snowball/compiler/tokeniser.c
+- vendor/snowball/compiler/tokens.h
 - vendor/snowball/csharp/.gitignore
 - vendor/snowball/csharp/Snowball/Algorithms/.gitignore
 - vendor/snowball/csharp/Snowball/Among.cs
@@ -99,11 +109,25 @@ files:
 - vendor/snowball/csharp/Snowball/Stemmer.cs
 - vendor/snowball/csharp/Stemwords/App.config
 - vendor/snowball/csharp/Stemwords/Program.cs
+- vendor/snowball/cxx/.gitignore
+- vendor/snowball/cxx/generate_algorithms.pl
+- vendor/snowball/cxx/stemmer.h
+- vendor/snowball/cxx/stemwords.cxx
+- vendor/snowball/cxx/utilities.cxx
+- vendor/snowball/dart/.gitignore
+- vendor/snowball/dart/analysis_options.yaml
+- vendor/snowball/dart/example/test_app.dart
+- vendor/snowball/dart/generate_algorithms.pl
+- vendor/snowball/dart/lib/snowball.dart
+- vendor/snowball/dart/lib/src/snowball.dart
+- vendor/snowball/dart/pubspec.yaml
 - vendor/snowball/doc/TODO
 - vendor/snowball/doc/libstemmer_c_README
 - vendor/snowball/doc/libstemmer_csharp_README
+- vendor/snowball/doc/libstemmer_dart_README
 - vendor/snowball/doc/libstemmer_java_README
 - vendor/snowball/doc/libstemmer_js_README
+- vendor/snowball/doc/libstemmer_php_README
 - vendor/snowball/doc/libstemmer_python_README
 - vendor/snowball/examples/stemwords.c
 - vendor/snowball/go/README.md
@@ -115,6 +139,7 @@ files:
 - vendor/snowball/iconv.py
 - vendor/snowball/include/libstemmer.h
 - vendor/snowball/java/org/tartarus/snowball/Among.java
+- vendor/snowball/java/org/tartarus/snowball/CharArraySequence.java
 - vendor/snowball/java/org/tartarus/snowball/SnowballProgram.java
 - vendor/snowball/java/org/tartarus/snowball/SnowballStemmer.java
 - vendor/snowball/java/org/tartarus/snowball/TestApp.java
@@ -129,8 +154,11 @@ files:
 - vendor/snowball/pascal/SnowballProgram.pas
 - vendor/snowball/pascal/generate.pl
 - vendor/snowball/pascal/stemwords-template.dpr
+- vendor/snowball/php/base-stemmer.php
+- vendor/snowball/php/stemwords.php
 - vendor/snowball/python/MANIFEST.in
 - vendor/snowball/python/create_init.py
+- vendor/snowball/python/pyproject.toml
 - vendor/snowball/python/setup.cfg
 - vendor/snowball/python/setup.py
 - vendor/snowball/python/snowballstemmer/among.py
@@ -139,7 +167,7 @@ files:
 - vendor/snowball/python/testapp.py
 - vendor/snowball/runtime/api.c
 - vendor/snowball/runtime/api.h
-- vendor/snowball/runtime/header.h
+- vendor/snowball/runtime/snowball_runtime.h
 - vendor/snowball/runtime/utilities.c
 - vendor/snowball/rust/Cargo.toml
 - vendor/snowball/rust/build.rs
@@ -149,7 +177,77 @@ files:
 - vendor/snowball/rust/src/snowball/among.rs
 - vendor/snowball/rust/src/snowball/mod.rs
 - vendor/snowball/rust/src/snowball/snowball_env.rs
+- vendor/snowball/tests/compilertest
+- vendor/snowball/tests/errors/ae-errors.sbl
+- vendor/snowball/tests/errors/ae-errors.stderr
+- vendor/snowball/tests/errors/bad-dollar.sbl
+- vendor/snowball/tests/errors/bad-dollar.stderr
+- vendor/snowball/tests/errors/bad-grouping-definition.sbl
+- vendor/snowball/tests/errors/bad-grouping-definition.stderr
+- vendor/snowball/tests/errors/missing-bra.sbl
+- vendor/snowball/tests/errors/missing-bra.stderr
+- vendor/snowball/tests/errors/missing-command.sbl
+- vendor/snowball/tests/errors/missing-command.stderr
+- vendor/snowball/tests/errors/missing-ket-backwardmode.sbl
+- vendor/snowball/tests/errors/missing-ket-backwardmode.stderr
+- vendor/snowball/tests/errors/missing-ket.sbl
+- vendor/snowball/tests/errors/missing-ket.stderr
+- vendor/snowball/tests/errors/notdefined.sbl
+- vendor/snowball/tests/errors/notdefined.stderr
+- vendor/snowball/tests/errors/string-omitted.sbl
+- vendor/snowball/tests/errors/string-omitted.stderr
+- vendor/snowball/tests/errors/undeclared.sbl
+- vendor/snowball/tests/errors/undeclared.stderr
+- vendor/snowball/tests/errors/unexpected-token.sbl
+- vendor/snowball/tests/errors/unexpected-token.stderr
+- vendor/snowball/tests/errors/wrongdirection.sbl
+- vendor/snowball/tests/errors/wrongdirection.stderr
+- vendor/snowball/tests/runtime/among.sbl
+- vendor/snowball/tests/runtime/arithmeticexpr.sbl
+- vendor/snowball/tests/runtime/attachinsert.sbl
+- vendor/snowball/tests/runtime/booleans.sbl
+- vendor/snowball/tests/runtime/externals.sbl
+- vendor/snowball/tests/runtime/hop.sbl
+- vendor/snowball/tests/runtime/integertests.sbl
+- vendor/snowball/tests/runtime/intlimits.sbl
+- vendor/snowball/tests/runtime/naming.sbl
+- vendor/snowball/tests/runtime/not.sbl
+- vendor/snowball/tests/runtime/or.sbl
+- vendor/snowball/tests/runtime/repeat.sbl
+- vendor/snowball/tests/runtime/setlimit.sbl
+- vendor/snowball/tests/runtime/sizelen.sbl
+- vendor/snowball/tests/runtime/slice.sbl
+- vendor/snowball/tests/runtime/stringdollar.sbl
+- vendor/snowball/tests/runtime/strings.sbl
+- vendor/snowball/tests/runtime/test.sbl
 - vendor/snowball/tests/stemtest.c
+- vendor/snowball/tests/syntax/canon.sbl
+- vendor/snowball/tests/syntax/canon.stderr
+- vendor/snowball/tests/syntax/canon.syntax
+- vendor/snowball/tests/syntax/emptyprogram.sbl
+- vendor/snowball/tests/syntax/emptyprogram.syntax
+- vendor/snowball/tests/syntax/groupings.sbl
+- vendor/snowball/tests/syntax/inline.sbl
+- vendor/snowball/tests/syntax/inline.stderr
+- vendor/snowball/tests/syntax/inline.syntax
+- vendor/snowball/tests/syntax/legacy.sbl
+- vendor/snowball/tests/syntax/legacy.stderr
+- vendor/snowball/tests/syntax/legacy.syntax
+- vendor/snowball/tests/syntax/loops.sbl
+- vendor/snowball/tests/syntax/loops.stderr
+- vendor/snowball/tests/syntax/loops.syntax
+- vendor/snowball/tests/syntax/noops.sbl
+- vendor/snowball/tests/syntax/noops.stderr
+- vendor/snowball/tests/syntax/noops.syntax
+- vendor/snowball/tests/syntax/simplifyae.sbl
+- vendor/snowball/tests/syntax/simplifyae.stderr
+- vendor/snowball/tests/syntax/simplifyae.syntax
+- vendor/snowball/tests/syntax/unused.sbl
+- vendor/snowball/tests/syntax/unused.stderr
+- vendor/snowball/tests/syntax/unused.syntax
+- vendor/snowball/zig/env.zig
+- vendor/snowball/zig/generate_algorithms.pl
+- vendor/snowball/zig/stemwords.zig
 homepage: https://github.com/ankane/mittens
 licenses:
 - BSD-3-Clause
@@ -168,7 +266,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.14
 specification_version: 4
 summary: Stemming for Ruby, powered by Snowball
 test_files: []

data/vendor/snowball/compiler/syswords.h

@@ -1,86 +0,0 @@
-static const struct system_word vocab[82+1] = {
-  { 0, (const byte *)"", 82+1},
-
-  { 1, (const byte *)"$",             c_dollar },
-  { 1, (const byte *)"(",             c_bra },
-  { 1, (const byte *)")",             c_ket },
-  { 1, (const byte *)"*",             c_multiply },
-  { 1, (const byte *)"+",             c_plus },
-  { 1, (const byte *)"-",             c_minus },
-  { 1, (const byte *)"/",             c_divide },
-  { 1, (const byte *)"<",             c_lt },
-  { 1, (const byte *)"=",             c_assign },
-  { 1, (const byte *)">",             c_gt },
-  { 1, (const byte *)"?",             c_debug },
-  { 1, (const byte *)"[",             c_leftslice },
-  { 1, (const byte *)"]",             c_rightslice },
-  { 2, (const byte *)"!=",            c_ne },
-  { 2, (const byte *)"*=",            c_multiplyassign },
-  { 2, (const byte *)"+=",            c_plusassign },
-  { 2, (const byte *)"-=",            c_minusassign },
-  { 2, (const byte *)"->",            c_sliceto },
-  { 2, (const byte *)"/*",            c_comment2 },
-  { 2, (const byte *)"//",            c_comment1 },
-  { 2, (const byte *)"/=",            c_divideassign },
-  { 2, (const byte *)"<+",            c_insert },
-  { 2, (const byte *)"<-",            c_slicefrom },
-  { 2, (const byte *)"<=",            c_le },
-  { 2, (const byte *)"==",            c_eq },
-  { 2, (const byte *)"=>",            c_assignto },
-  { 2, (const byte *)">=",            c_ge },
-  { 2, (const byte *)"as",            c_as },
-  { 2, (const byte *)"do",            c_do },
-  { 2, (const byte *)"or",            c_or },
-  { 3, (const byte *)"and",           c_and },
-  { 3, (const byte *)"for",           c_for },
-  { 3, (const byte *)"get",           c_get },
-  { 3, (const byte *)"hex",           c_hex },
-  { 3, (const byte *)"hop",           c_hop },
-  { 3, (const byte *)"len",           c_len },
-  { 3, (const byte *)"non",           c_non },
-  { 3, (const byte *)"not",           c_not },
-  { 3, (const byte *)"set",           c_set },
-  { 3, (const byte *)"try",           c_try },
-  { 4, (const byte *)"fail",          c_fail },
-  { 4, (const byte *)"goto",          c_goto },
-  { 4, (const byte *)"loop",          c_loop },
-  { 4, (const byte *)"next",          c_next },
-  { 4, (const byte *)"size",          c_size },
-  { 4, (const byte *)"test",          c_test },
-  { 4, (const byte *)"true",          c_true },
-  { 5, (const byte *)"among",         c_among },
-  { 5, (const byte *)"false",         c_false },
-  { 5, (const byte *)"lenof",         c_lenof },
-  { 5, (const byte *)"limit",         c_limit },
-  { 5, (const byte *)"unset",         c_unset },
-  { 6, (const byte *)"atmark",        c_atmark },
-  { 6, (const byte *)"attach",        c_attach },
-  { 6, (const byte *)"cursor",        c_cursor },
-  { 6, (const byte *)"define",        c_define },
-  { 6, (const byte *)"delete",        c_delete },
-  { 6, (const byte *)"gopast",        c_gopast },
-  { 6, (const byte *)"insert",        c_insert },
-  { 6, (const byte *)"maxint",        c_maxint },
-  { 6, (const byte *)"minint",        c_minint },
-  { 6, (const byte *)"repeat",        c_repeat },
-  { 6, (const byte *)"sizeof",        c_sizeof },
-  { 6, (const byte *)"tomark",        c_tomark },
-  { 7, (const byte *)"atleast",       c_atleast },
-  { 7, (const byte *)"atlimit",       c_atlimit },
-  { 7, (const byte *)"decimal",       c_decimal },
-  { 7, (const byte *)"reverse",       c_reverse },
-  { 7, (const byte *)"setmark",       c_setmark },
-  { 7, (const byte *)"strings",       c_strings },
-  { 7, (const byte *)"tolimit",       c_tolimit },
-  { 8, (const byte *)"booleans",      c_booleans },
-  { 8, (const byte *)"integers",      c_integers },
-  { 8, (const byte *)"routines",      c_routines },
-  { 8, (const byte *)"setlimit",      c_setlimit },
-  { 9, (const byte *)"backwards",     c_backwards },
-  { 9, (const byte *)"externals",     c_externals },
-  { 9, (const byte *)"groupings",     c_groupings },
-  { 9, (const byte *)"stringdef",     c_stringdef },
-  { 9, (const byte *)"substring",     c_substring },
- { 12, (const byte *)"backwardmode",  c_backwardmode },
- { 13, (const byte *)"stringescapes", c_stringescapes }
-};

data/vendor/snowball/runtime/header.h

@@ -1,62 +0,0 @@
-
-#include "api.h"
-
-#define HEAD 2*sizeof(int)
-
-#define SIZE(p)        ((int *)(p))[-1]
-#define SET_SIZE(p, n) ((int *)(p))[-1] = n
-#define CAPACITY(p)    ((int *)(p))[-2]
-
-struct among
-{
-    /* Number of symbols in s. */
-    int s_size;
-    /* Search string. */
-    const symbol * s;
-    /* Delta of index to longest matching substring, or 0 if none. */
-    int substring_i;
-    /* Result of the lookup. */
-    int result;
-    /* Optional condition routine, or NULL if none. */
-    int (* function)(struct SN_env *);
-};
-
-extern symbol * create_s(void);
-extern void lose_s(symbol * p);
-
-extern int skip_utf8(const symbol * p, int c, int limit, int n);
-
-extern int skip_b_utf8(const symbol * p, int c, int limit, int n);
-
-extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
-extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
-extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
-extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
-
-extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
-extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
-extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
-extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
-
-extern int eq_s(struct SN_env * z, int s_size, const symbol * s);
-extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s);
-extern int eq_v(struct SN_env * z, const symbol * p);
-extern int eq_v_b(struct SN_env * z, const symbol * p);
-
-extern int find_among(struct SN_env * z, const struct among * v, int v_size);
-extern int find_among_b(struct SN_env * z, const struct among * v, int v_size);
-
-extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjustment);
-extern int slice_from_s(struct SN_env * z, int s_size, const symbol * s);
-extern int slice_from_v(struct SN_env * z, const symbol * p);
-extern int slice_del(struct SN_env * z);
-
-extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s);
-extern int insert_v(struct SN_env * z, int bra, int ket, const symbol * p);
-
-extern symbol * slice_to(struct SN_env * z, symbol * p);
-extern symbol * assign_to(struct SN_env * z, symbol * p);
-
-extern int len_utf8(const symbol * p);
-
-extern void debug(struct SN_env * z, int number, int line_count);