mittens 0.1.0

data/vendor/snowball/examples/stemwords.c

@@ -0,0 +1,204 @@
+/* This is a simple program which uses libstemmer to provide a command
+ * line interface for stemming using any of the algorithms provided.
+ */
+
+#include <stdio.h>
+#include <stdlib.h> /* for malloc, free */
+#include <string.h> /* for memmove */
+#include <ctype.h>  /* for isupper, tolower */
+
+#include "libstemmer.h"
+
+const char * progname;
+static int pretty = 1;
+
+static void
+stem_file(struct sb_stemmer * stemmer, FILE * f_in, FILE * f_out)
+{
+#define INC 10
+    int lim = INC;
+    sb_symbol * b = (sb_symbol *) malloc(lim * sizeof(sb_symbol));
+
+    while (1) {
+        int ch = getc(f_in);
+        if (ch == EOF) {
+            free(b); return;
+        }
+        {
+            int i = 0;
+            int inlen = 0;
+            while (ch != '\n' && ch != EOF) {
+                if (i == lim) {
+                    sb_symbol * newb;
+                    newb = (sb_symbol *)
+                            realloc(b, (lim + INC) * sizeof(sb_symbol));
+                    if (newb == 0) goto error;
+                    b = newb;
+                    lim = lim + INC;
+                }
+                /* Update count of utf-8 characters. */
+                if (ch < 0x80 || ch > 0xBF) inlen += 1;
+                /* force lower case: */
+                ch = tolower(ch);
+
+                b[i] = ch;
+                i++;
+                ch = getc(f_in);
+            }
+
+            {
+                const sb_symbol * stemmed = sb_stemmer_stem(stemmer, b, i);
+                if (stemmed == NULL)
+                {
+                    fprintf(stderr, "Out of memory");
+                    exit(1);
+                }
+
+                if (pretty == 1) {
+                    fwrite(b, i, 1, f_out);
+                    fputs(" -> ", f_out);
+                } else if (pretty == 2) {
+                    fwrite(b, i, 1, f_out);
+                    if (sb_stemmer_length(stemmer) > 0) {
+                        int j;
+                        if (inlen < 30) {
+                            for (j = 30 - inlen; j > 0; j--)
+                                fputs(" ", f_out);
+                        } else {
+                            fputs("\n", f_out);
+                            for (j = 30; j > 0; j--)
+                                fputs(" ", f_out);
+                        }
+                    }
+                }
+
+                fputs((const char *)stemmed, f_out);
+                putc('\n', f_out);
+            }
+        }
+    }
+error:
+    if (b != 0) free(b);
+    return;
+}
+
+/** Display the command line syntax, and then exit.
+ *  @param n The value to exit with.
+ */
+static void
+usage(int n)
+{
+    printf("usage: %s [-l <language>] [-i <input file>] [-o <output file>] [-c <character encoding>] [-p[2]] [-h]\n"
+          "\n"
+          "The input file consists of a list of words to be stemmed, one per\n"
+          "line. Words should be in lower case, but (for English) A-Z letters\n"
+          "are mapped to their a-z equivalents anyway. If omitted, stdin is\n"
+          "used.\n"
+          "\n"
+          "If -c is given, the argument is the character encoding of the input\n"
+          "and output files.  If it is omitted, the UTF-8 encoding is used.\n"
+          "\n"
+          "If -p is given the output file consists of each word of the input\n"
+          "file followed by \"->\" followed by its stemmed equivalent.\n"
+          "If -p2 is given the output file is a two column layout containing\n"
+          "the input words in the first column and the stemmed equivalents in\n"
+          "the second column.\n"
+          "Otherwise, the output file consists of the stemmed words, one per\n"
+          "line.\n"
+          "\n"
+          "-h displays this help\n",
+          progname);
+    exit(n);
+}
+
+int
+main(int argc, char * argv[])
+{
+    const char * in = 0;
+    const char * out = 0;
+    FILE * f_in;
+    FILE * f_out;
+    struct sb_stemmer * stemmer;
+
+    const char * language = "english";
+    const char * charenc = NULL;
+
+    int i = 1;
+    pretty = 0;
+
+    progname = argv[0];
+
+    while (i < argc) {
+        const char * s = argv[i++];
+        if (s[0] == '-') {
+            if (strcmp(s, "-o") == 0) {
+                if (i >= argc) {
+                    fprintf(stderr, "%s requires an argument\n", s);
+                    exit(1);
+                }
+                out = argv[i++];
+            } else if (strcmp(s, "-i") == 0) {
+                if (i >= argc) {
+                    fprintf(stderr, "%s requires an argument\n", s);
+                    exit(1);
+                }
+                in = argv[i++];
+            } else if (strcmp(s, "-l") == 0) {
+                if (i >= argc) {
+                    fprintf(stderr, "%s requires an argument\n", s);
+                    exit(1);
+                }
+                language = argv[i++];
+            } else if (strcmp(s, "-c") == 0) {
+                if (i >= argc) {
+                    fprintf(stderr, "%s requires an argument\n", s);
+                    exit(1);
+                }
+                charenc = argv[i++];
+            } else if (strcmp(s, "-p2") == 0) {
+                pretty = 2;
+            } else if (strcmp(s, "-p") == 0) {
+                pretty = 1;
+            } else if (strcmp(s, "-h") == 0) {
+                usage(0);
+            } else {
+                fprintf(stderr, "option %s unknown\n", s);
+                usage(1);
+            }
+        } else {
+            fprintf(stderr, "unexpected parameter %s\n", s);
+            usage(1);
+        }
+    }
+
+    /* prepare the files */
+    f_in = (in == 0) ? stdin : fopen(in, "r");
+    if (f_in == 0) {
+        fprintf(stderr, "file %s not found\n", in);
+        exit(1);
+    }
+    f_out = (out == 0) ? stdout : fopen(out, "w");
+    if (f_out == 0) {
+        fprintf(stderr, "file %s cannot be opened\n", out);
+        exit(1);
+    }
+
+    /* do the stemming process: */
+    stemmer = sb_stemmer_new(language, charenc);
+    if (stemmer == 0) {
+        if (charenc == NULL) {
+            fprintf(stderr, "language `%s' not available for stemming\n", language);
+            exit(1);
+        } else {
+            fprintf(stderr, "language `%s' not available for stemming in encoding `%s'\n", language, charenc);
+            exit(1);
+        }
+    }
+    stem_file(stemmer, f_in, f_out);
+    sb_stemmer_delete(stemmer);
+
+    if (in != 0) (void) fclose(f_in);
+    if (out != 0) (void) fclose(f_out);
+
+    return 0;
+}

data/vendor/snowball/go/README.md

@@ -0,0 +1,55 @@
+# Go Target for Snowball
+
+The initial implementation was built as a port of the Rust target.  The initial focus has been on getting it to function, and making it work correctly.  No attempt has been made to beautify the implementation, generated code, or address performance issues.
+
+## Usage
+
+To generate Go source for a Snowball algorithm:
+```
+$ snowball path/to/algorithm.sbl -go -o algorithm
+```
+
+### Go specific options
+
+`-gop[ackage]` the package name used in the generated go file (defaults to `snowball`)
+
+`-gor[untime]` the import path used for the Go Snowball runtime (defaults to `github.com/snowballstem/snowball/go`)
+
+## Code Organization
+
+`compiler/generator_go.c` has the Go code generation logic
+
+`go/` contains the default Go Snowball runtime support
+
+`go/stemwords` contains the source for a Go version of the stemwords utility
+
+`go/algorithms` location where the makefile generated code will end up
+
+## Using the Generated Stemmers
+
+Assuming you generated a stemmer, put that code in a package which is imported by this code as `english`.
+
+```
+env := snowball.NewEnv("beautiful")
+english.Stem(env)
+fmt.Printf("stemmed word is: %s", env.Current())
+```
+
+NOTE: you can use the env.SetCurrent("new_word") to reuse the env on subsequent calls to the stemmer.
+
+## Testing
+
+Only the existing Snowball algorithms have been used for testing.  This does not exercise all features of the language.
+
+Run:
+
+```
+$ make check_go
+```
+
+An initial pass of fuzz-testing has been performed on the generated stemmers for the algorithms in this repo.  Each ran for 5 minutes and used an initial corpus seeded with 10k words from the algorithm's snowballstem-data voc.txt file.
+
+## Known Limitations
+
+- Code going through generate_dollar production has not been tested
+- Code going through generate_debug production has not been tested

data/vendor/snowball/go/among.go

@@ -0,0 +1,16 @@
+package snowball
+
+import "fmt"
+
+type AmongF func(env *Env, ctx interface{}) bool
+
+type Among struct {
+	Str string
+	A   int32
+	B   int32
+	F   AmongF
+}
+
+func (a *Among) String() string {
+	return fmt.Sprintf("str: `%s`, a: %d, b: %d, f: %p", a.Str, a.A, a.B, a.F)
+}

data/vendor/snowball/go/env.go

@@ -0,0 +1,403 @@
+package snowball
+
+import (
+	"log"
+	"strings"
+	"unicode/utf8"
+)
+
+// Env represents the Snowball execution environment
+type Env struct {
+	current       string
+	Cursor        int
+	Limit         int
+	LimitBackward int
+	Bra           int
+	Ket           int
+}
+
+// NewEnv creates a new Snowball execution environment on the provided string
+func NewEnv(val string) *Env {
+	return &Env{
+		current:       val,
+		Cursor:        0,
+		Limit:         len(val),
+		LimitBackward: 0,
+		Bra:           0,
+		Ket:           len(val),
+	}
+}
+
+func (env *Env) Current() string {
+	return env.current
+}
+
+func (env *Env) SetCurrent(s string) {
+	env.current = s
+	env.Cursor = 0
+	env.Limit = len(s)
+	env.LimitBackward = 0
+	env.Bra = 0
+	env.Ket = len(s)
+}
+
+func (env *Env) ReplaceS(bra, ket int, s string) int32 {
+	adjustment := int32(len(s)) - (int32(ket) - int32(bra))
+	result, _ := splitAt(env.current, bra)
+	rsplit := ket
+	if ket < bra {
+		rsplit = bra
+	}
+	_, rhs := splitAt(env.current, rsplit)
+	result += s
+	result += rhs
+
+	newLim := int32(env.Limit) + adjustment
+	env.Limit = int(newLim)
+
+	if env.Cursor >= ket {
+		newCur := int32(env.Cursor) + adjustment
+		env.Cursor = int(newCur)
+	} else if env.Cursor > bra {
+		env.Cursor = bra
+	}
+
+	env.current = result
+	return adjustment
+}
+
+func (env *Env) EqS(s string) bool {
+	if env.Cursor >= env.Limit {
+		return false
+	}
+
+	if strings.HasPrefix(env.current[env.Cursor:], s) {
+		env.Cursor += len(s)
+		for !onCharBoundary(env.current, env.Cursor) {
+			env.Cursor++
+		}
+		return true
+	}
+	return false
+}
+
+func (env *Env) EqSB(s string) bool {
+	if int32(env.Cursor)-int32(env.LimitBackward) < int32(len(s)) {
+		return false
+	} else if !onCharBoundary(env.current, env.Cursor-len(s)) ||
+		!strings.HasPrefix(env.current[env.Cursor-len(s):], s) {
+		return false
+	} else {
+		env.Cursor -= len(s)
+		return true
+	}
+}
+
+func (env *Env) SliceFrom(s string) bool {
+	bra, ket := env.Bra, env.Ket
+	env.ReplaceS(bra, ket, s)
+	return true
+}
+
+func (env *Env) NextChar() {
+	env.Cursor++
+	for !onCharBoundary(env.current, env.Cursor) {
+		env.Cursor++
+	}
+}
+
+func (env *Env) PrevChar() {
+	env.Cursor--
+	for !onCharBoundary(env.current, env.Cursor) {
+		env.Cursor--
+	}
+}
+
+func (env *Env) Hop(delta int32) bool {
+	res := env.Cursor
+	for delta > 0 {
+		delta--
+		if res >= env.Limit {
+			return false
+		}
+		res++
+		for res < env.Limit && !onCharBoundary(env.current, res) {
+			res++
+		}
+	}
+	env.Cursor = res
+	return true
+}
+
+func (env *Env) HopChecked(delta int32) bool {
+	return delta >= 0 && env.Hop(delta)
+}
+
+func (env *Env) HopBack(delta int32) bool {
+	res := env.Cursor
+	for delta > 0 {
+		delta--
+		if res <= env.LimitBackward {
+			return false
+		}
+		res--
+		for res > env.LimitBackward && !onCharBoundary(env.current, res) {
+			res--
+		}
+	}
+	env.Cursor = res
+	return true
+}
+
+func (env *Env) HopBackChecked(delta int32) bool {
+	return delta >= 0 && env.HopBack(delta)
+}
+
+func (env *Env) InGrouping(chars []byte, min, max int32) bool {
+	if env.Cursor >= env.Limit {
+		return false
+	}
+
+	r, _ := utf8.DecodeRuneInString(env.current[env.Cursor:])
+	if r != utf8.RuneError {
+		if r > max || r < min {
+			return false
+		}
+		r -= min
+		if (chars[uint(r>>3)] & (0x1 << uint(r&0x7))) == 0 {
+			return false
+		}
+		env.NextChar()
+		return true
+	}
+	return false
+}
+
+func (env *Env) InGroupingB(chars []byte, min, max int32) bool {
+	if env.Cursor <= env.LimitBackward {
+		return false
+	}
+	env.PrevChar()
+	r, _ := utf8.DecodeRuneInString(env.current[env.Cursor:])
+	if r != utf8.RuneError {
+		env.NextChar()
+		if r > max || r < min {
+			return false
+		}
+		r -= min
+		if (chars[uint(r>>3)] & (0x1 << uint(r&0x7))) == 0 {
+			return false
+		}
+		env.PrevChar()
+		return true
+	}
+	return false
+}
+
+func (env *Env) OutGrouping(chars []byte, min, max int32) bool {
+	if env.Cursor >= env.Limit {
+		return false
+	}
+	r, _ := utf8.DecodeRuneInString(env.current[env.Cursor:])
+	if r != utf8.RuneError {
+		if r > max || r < min {
+			env.NextChar()
+			return true
+		}
+		r -= min
+		if (chars[uint(r>>3)] & (0x1 << uint(r&0x7))) == 0 {
+			env.NextChar()
+			return true
+		}
+	}
+	return false
+}
+
+func (env *Env) OutGroupingB(chars []byte, min, max int32) bool {
+	if env.Cursor <= env.LimitBackward {
+		return false
+	}
+	env.PrevChar()
+	r, _ := utf8.DecodeRuneInString(env.current[env.Cursor:])
+	if r != utf8.RuneError {
+		env.NextChar()
+		if r > max || r < min {
+			env.PrevChar()
+			return true
+		}
+		r -= min
+		if (chars[uint(r>>3)] & (0x1 << uint(r&0x7))) == 0 {
+			env.PrevChar()
+			return true
+		}
+	}
+	return false
+}
+
+func (env *Env) SliceDel() bool {
+	return env.SliceFrom("")
+}
+
+func (env *Env) Insert(bra, ket int, s string) {
+	adjustment := env.ReplaceS(bra, ket, s)
+	if bra <= env.Bra {
+		env.Bra = int(int32(env.Bra) + adjustment)
+	}
+	if bra <= env.Ket {
+		env.Ket = int(int32(env.Ket) + adjustment)
+	}
+}
+
+func (env *Env) SliceTo() string {
+	return env.current[env.Bra:env.Ket]
+}
+
+func (env *Env) FindAmong(amongs []*Among, ctx interface{}) int32 {
+	var i int32
+	j := int32(len(amongs))
+
+	c := env.Cursor
+	l := env.Limit
+
+	var commonI, commonJ int
+
+	firstKeyInspected := false
+	for {
+		k := i + ((j - i) >> 1)
+		var diff int32
+		common := min(commonI, commonJ)
+		w := amongs[k]
+		for lvar := common; lvar < len(w.Str); lvar++ {
+			if c+common == l {
+				diff--
+				break
+			}
+			diff = int32(env.current[c+common]) - int32(w.Str[lvar])
+			if diff != 0 {
+				break
+			}
+			common++
+		}
+		if diff < 0 {
+			j = k
+			commonJ = common
+		} else {
+			i = k
+			commonI = common
+		}
+		if j-i <= 1 {
+			if i > 0 {
+				break
+			}
+			if j == i {
+				break
+			}
+			if firstKeyInspected {
+				break
+			}
+			firstKeyInspected = true
+		}
+	}
+
+	for {
+		w := amongs[i]
+		if commonI >= len(w.Str) {
+			env.Cursor = c + len(w.Str)
+			if w.F != nil {
+				res := w.F(env, ctx)
+				env.Cursor = c + len(w.Str)
+				if res {
+					return w.B
+				}
+			} else {
+				return w.B
+			}
+		}
+		i = w.A
+		if i < 0 {
+			return 0
+		}
+	}
+}
+
+func (env *Env) FindAmongB(amongs []*Among, ctx interface{}) int32 {
+	var i int32
+	j := int32(len(amongs))
+
+	c := env.Cursor
+	lb := env.LimitBackward
+
+	var commonI, commonJ int
+
+	firstKeyInspected := false
+
+	for {
+		k := i + ((j - i) >> 1)
+		diff := int32(0)
+		common := min(commonI, commonJ)
+		w := amongs[k]
+		for lvar := len(w.Str) - int(common) - 1; lvar >= 0; lvar-- {
+			if c-common == lb {
+				diff--
+				break
+			}
+			diff = int32(env.current[c-common-1]) - int32(w.Str[lvar])
+			if diff != 0 {
+				break
+			}
+			// Count up commons. But not one character but the byte width of that char
+			common++
+		}
+		if diff < 0 {
+			j = k
+			commonJ = common
+		} else {
+			i = k
+			commonI = common
+		}
+		if j-i <= 1 {
+			if i > 0 {
+				break
+			}
+			if j == i {
+				break
+			}
+			if firstKeyInspected {
+				break
+			}
+			firstKeyInspected = true
+		}
+	}
+	for {
+		w := amongs[i]
+		if commonI >= len(w.Str) {
+			env.Cursor = c - len(w.Str)
+			if w.F != nil {
+				res := w.F(env, ctx)
+				env.Cursor = c - len(w.Str)
+				if res {
+					return w.B
+				}
+			} else {
+				return w.B
+			}
+		}
+		i = w.A
+		if i < 0 {
+			return 0
+		}
+	}
+}
+
+func (env *Env) Debug(count, lineNumber int) {
+	log.Printf("snowball debug, count: %d, line: %d", count, lineNumber)
+}
+
+func (env *Env) Clone() *Env {
+	clone := *env
+	return &clone
+}
+
+func (env *Env) AssignTo() string {
+	return env.Current()
+}