mittens 0.3.1 → 0.3.2

data/vendor/snowball/python/stemwords.py

@@ -1,5 +1,4 @@
 import sys
-import codecs
 import snowballstemmer
 
 def usage():
@@ -25,73 +24,83 @@ line.
 -h displays this help''' % sys.argv[0])
 
 def main():
+    pretty = 0
+    input = ''
+    output = ''
+    encoding = 'utf_8'
+    language = 'English'
+    show_help = False
     argv = sys.argv[1:]
-    if len(argv) < 5:
-        usage()
-    else:
-        pretty = 0
-        input = ''
-        output = ''
-        encoding = 'utf_8'
-        language = 'English'
-        show_help = False
-        while len(argv):
-            arg = argv.pop(0)
-            if arg == '-h':
+    while len(argv):
+        arg = argv.pop(0)
+        if arg == '-h':
+            show_help = True
+            break
+        elif arg == "-p":
+            pretty = 1
+        elif arg == "-p2":
+            pretty = 2
+        elif arg == "-l":
+            if len(argv) == 0:
+                show_help = True
+                break
+            language = argv.pop(0)
+        elif arg == "-i":
+            if len(argv) == 0:
+                show_help = True
+                break
+            input = argv.pop(0)
+        elif arg == "-o":
+            if len(argv) == 0:
                 show_help = True
                 break
-            elif arg == "-p":
-                pretty = 1
-            elif arg == "-p2":
-                pretty = 2
-            elif arg == "-l":
-                if len(argv) == 0:
-                    show_help = True
-                    break
-                language = argv.pop(0)
-            elif arg == "-i":
-                if len(argv) == 0:
-                    show_help = True
-                    break
-                input = argv.pop(0)
-            elif arg == "-o":
-                if len(argv) == 0:
-                    show_help = True
-                    break
-                output = argv.pop(0)
-            elif arg == "-c":
-                if len(argv) == 0:
-                    show_help = True
-                    break
-                encoding = argv.pop(0)
-        if show_help or input == '' or output == '':
-            usage()
+            output = argv.pop(0)
+        elif arg == "-c":
+            if len(argv) == 0:
+                show_help = True
+                break
+            encoding = argv.pop(0)
+    if show_help:
+        usage()
+    else:
+        stemmer = snowballstemmer.stemmer(language)
+        if input != '':
+            infile = open(input, "r", encoding=encoding)
+        else:
+            infile = sys.stdin
+            # reconfigure() requires Python 3.7 so check existing encoding.
+            if infile.encoding.lower() != encoding.lower():
+                infile.reconfigure(encoding = encoding)
+        if output != '':
+                outfile = open(output, "w", encoding=encoding)
         else:
-            stemming(language, input, output, encoding, pretty)
+            outfile = sys.stdout
+            if outfile.encoding.lower() != encoding.lower():
+                outfile.reconfigure(encoding = encoding)
+        stemming(stemmer, infile, outfile, pretty)
+        outfile.close()
+        infile.close()
 
 
-def stemming(lang, input, output, encoding, pretty):
-    stemmer = snowballstemmer.stemmer(lang)
-    with codecs.open(input, "r", encoding) as infile:
-        with codecs.open(output, "w", encoding) as outfile:
-            for original in infile.readlines():
-                original = original.strip()
-                # Convert only ASCII-letters to lowercase, to match C behavior
-                original = ''.join((c.lower() if 'A' <= c <= 'Z' else c for c in original))
-                stemmed = stemmer.stemWord(original)
-                if pretty == 0:
-                    if stemmed != "":
-                        outfile.write(stemmed)
-                elif pretty == 1:
-                    outfile.write(original, " -> ", stemmed)
-                elif pretty == 2:
-                    outfile.write(original)
-                    if len(original) < 30:
-                        outfile.write(" " * (30 - len(original)))
-                    else:
-                        outfile.write("\n")
-                        outfile.write(" " * 30)
-                    outfile.write(stemmed)
-                outfile.write('\n')
+def stemming(stemmer, infile, outfile, pretty):
+    for original in infile.readlines():
+        original = original.strip()
+        # Convert only ASCII-letters to lowercase, to match C behavior
+        original = ''.join(c.lower() if 'A' <= c <= 'Z' else c for c in original)
+        stemmed = stemmer.stemWord(original)
+        if pretty == 0:
+            if stemmed != "":
+                outfile.write(stemmed)
+        elif pretty == 1:
+            outfile.write(original, " -> ", stemmed)
+        elif pretty == 2:
+            outfile.write(original)
+            if len(original) < 30:
+                outfile.write(" " * (30 - len(original)))
+            else:
+                outfile.write("\n")
+                outfile.write(" " * 30)
+            outfile.write(stemmed)
+        outfile.write('\n')
 
 main()

data/vendor/snowball/runtime/api.c

@@ -1,63 +1,32 @@
 
-#include <stdlib.h> /* for malloc, calloc, free */
-#include "header.h"
+#include <stdlib.h> /* for malloc, free */
+#include "snowball_runtime.h"
 
-extern struct SN_env * SN_create_env(int S_size, int I_size)
+static const struct SN_env default_SN_env;
+
+extern struct SN_env * SN_new_env(int alloc_size)
 {
-    static const struct SN_env default_SN_env = {};
-    struct SN_env * z = (struct SN_env *) malloc(sizeof(struct SN_env));
+    struct SN_env * z = (struct SN_env *) malloc(alloc_size);
     if (z == NULL) return NULL;
     *z = default_SN_env;
     z->p = create_s();
-    if (z->p == NULL) goto error;
-    if (S_size)
-    {
-        int i;
-        z->S = (symbol * *) malloc(S_size * sizeof(symbol *));
-        if (z->S == NULL) goto error;
-
-        for (i = 0; i < S_size; i++)
-        {
-            z->S[i] = create_s();
-            if (z->S[i] == NULL) {
-                S_size = i;
-                goto error;
-            }
-        }
-    }
-
-    if (I_size)
-    {
-        z->I = (int *) calloc(I_size, sizeof(int));
-        if (z->I == NULL) goto error;
+    if (z->p == NULL) {
+        SN_delete_env(z);
+        return NULL;
     }
-
     return z;
-error:
-    SN_close_env(z, S_size);
-    return NULL;
 }
 
-extern void SN_close_env(struct SN_env * z, int S_size)
+extern void SN_delete_env(struct SN_env * z)
 {
     if (z == NULL) return;
-    if (z->S)
-    {
-        int i;
-        for (i = 0; i < S_size; i++)
-        {
-            lose_s(z->S[i]);
-        }
-        free(z->S);
-    }
-    free(z->I);
     if (z->p) lose_s(z->p);
     free(z);
 }
 
 extern int SN_set_current(struct SN_env * z, int size, const symbol * s)
 {
-    int err = replace_s(z, 0, z->l, size, s, NULL);
+    int err = replace_s(z, 0, z->l, size, s);
     z->c = 0;
     return err;
 }

data/vendor/snowball/runtime/api.h

@@ -1,32 +1,34 @@
+#ifndef SNOWBALL_API_H_INCLUDED
+#define SNOWBALL_API_H_INCLUDED
 
 typedef unsigned char symbol;
 
 /* Or replace 'char' above with 'short' for 16 bit characters.
 
    More precisely, replace 'char' with whatever type guarantees the
-   character width you need. Note however that sizeof(symbol) should divide
-   HEAD, defined in header.h as 2*sizeof(int), without remainder, otherwise
-   there is an alignment problem. In the unlikely event of a problem here,
-   consult Martin Porter.
-
+   character width you need.
 */
 
 struct SN_env {
     symbol * p;
     int c; int l; int lb; int bra; int ket;
-    symbol * * S;
-    int * I;
+    int af;
+#ifdef __cplusplus
+    SN_env() : p(), c(), l(), lb(), bra(), ket(), af() { }
+#endif
 };
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-extern struct SN_env * SN_create_env(int S_size, int I_size);
-extern void SN_close_env(struct SN_env * z, int S_size);
+extern struct SN_env * SN_new_env(int alloc_size);
+extern void SN_delete_env(struct SN_env * z);
 
 extern int SN_set_current(struct SN_env * z, int size, const symbol * s);
 
 #ifdef __cplusplus
 }
 #endif
+
+#endif

data/vendor/snowball/runtime/snowball_runtime.h

@@ -0,0 +1,110 @@
+#ifndef SNOWBALL_INCLUDED_SNOWBALL_RUNTIME_H
+#define SNOWBALL_INCLUDED_SNOWBALL_RUNTIME_H
+
+#include "api.h"
+
+#include <limits.h>
+
+#ifdef __cplusplus
+/* Use reinterpret_cast<> to avoid -Wcast-align warnings from clang++. */
+# define SIZE(p)        (reinterpret_cast<const int *>(p))[-1]
+# define SET_SIZE(p, n) (reinterpret_cast<int *>(p))[-1] = n
+# define CAPACITY(p)    (reinterpret_cast<int *>(p))[-2]
+#else
+# define SIZE(p)        ((const int *)(p))[-1]
+# define SET_SIZE(p, n) ((int *)(p))[-1] = n
+# define CAPACITY(p)    ((int *)(p))[-2]
+#endif
+
+#ifdef SNOWBALL_RUNTIME_THROW_EXCEPTIONS
+# define SNOWBALL_ERR void
+#else
+# define SNOWBALL_ERR int
+#endif
+
+#ifdef SNOWBALL_DEBUG_COMMAND_USED
+# include <stdio.h>
+static void debug(struct SN_env * z, int n, int line) {
+    int i;
+    int len = SIZE(z->p);
+    printf("%3d (line %4d): [%d]'", n, line, len);
+    for (i = 0; i <= len; i++) {
+        if (z->lb == i) printf("{");
+        if (z->bra == i) printf("[");
+        if (z->c == i) printf("|");
+        if (z->ket == i) printf("]");
+        if (z->l == i) printf("}");
+        if (i < len) {
+            int ch = z->p[i];
+            if (ch == 0) ch = '#';
+            printf("%c", ch);
+        }
+    }
+    printf("'\n");
+}
+#endif
+
+struct among
+{
+    /* Number of symbols in s. */
+    int s_size;
+    /* Search string. */
+    const symbol * s;
+    /* Delta of index to longest matching substring, or 0 if none. */
+    int substring_i;
+    /* Result of the lookup. */
+    int result;
+    /* Optional condition routine index, or 0 if none. */
+    int function;
+};
+
+/* MSVC doesn't like functions declared `extern "C"` throwing exceptions. */
+#if defined __cplusplus && !defined SNOWBALL_RUNTIME_THROW_EXCEPTIONS
+extern "C" {
+#endif
+
+extern symbol * create_s(void);
+extern void lose_s(symbol * p);
+
+extern int skip_utf8(const symbol * p, int c, int limit, int n);
+
+extern int skip_b_utf8(const symbol * p, int c, int limit, int n);
+
+extern int in_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int in_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_b_U(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+
+extern int in_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int in_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+extern int out_grouping_b(struct SN_env * z, const unsigned char * s, int min, int max, int repeat);
+
+extern int eq_s(struct SN_env * z, int s_size, const symbol * s);
+extern int eq_s_b(struct SN_env * z, int s_size, const symbol * s);
+extern int eq_v(struct SN_env * z, const symbol * p);
+extern int eq_v_b(struct SN_env * z, const symbol * p);
+
+extern int find_among(struct SN_env * z, const struct among * v, int v_size,
+                      int (*)(struct SN_env *));
+extern int find_among_b(struct SN_env * z, const struct among * v, int v_size,
+                        int (*)(struct SN_env *));
+
+extern SNOWBALL_ERR replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s);
+extern SNOWBALL_ERR slice_from_s(struct SN_env * z, int s_size, const symbol * s);
+extern SNOWBALL_ERR slice_from_v(struct SN_env * z, const symbol * p);
+extern SNOWBALL_ERR slice_del(struct SN_env * z);
+
+extern SNOWBALL_ERR insert_s(struct SN_env * z, int bra, int ket, int s_size, const symbol * s);
+extern SNOWBALL_ERR insert_v(struct SN_env * z, int bra, int ket, const symbol * p);
+
+extern SNOWBALL_ERR slice_to(struct SN_env * z, symbol ** p);
+extern SNOWBALL_ERR assign_to(struct SN_env * z, symbol ** p);
+
+extern int len_utf8(const symbol * p);
+
+#if defined __cplusplus && !defined SNOWBALL_RUNTIME_THROW_EXCEPTIONS
+}
+#endif
+
+#endif