isomorfeus-ferret 0.12.0

data/ext/isomorfeus_ferret_ext/scanner.h

@@ -0,0 +1,28 @@
+#ifndef FRT_SCANNER_H
+#define FRT_SCANNER_H
+
+/*
+ * Scan +in+ and copy the token into +out+, up until +out_size+ bytes.
+ * The +start+ and +end+ are pointers to the original untouched token
+ * somewhere inside +in+.  This token may not always be copied
+ * verbatim into +out+.  For example, the http://google.com token will
+ * be truncated down to just google.com during the copy.
+ * +token_length+ is the size of the resulting token.
+ */
+void frt_std_scan(const char *in,
+                  char *out, size_t out_size,
+                  const char **start, const char **end,
+                  int *token_length);
+
+
+void frt_std_scan_mb(const char *in,
+                     char *out, size_t out_size,
+                     const char **start, const char **end,
+                     int *token_length);
+
+void frt_std_scan_utf8(const char *in,
+                       char *out, size_t out_size,
+                       const char **start, const char **end,
+                       int *token_length);
+
+#endif /* FRT_SCANNER */

data/ext/isomorfeus_ferret_ext/scanner.in

@@ -0,0 +1,43 @@
+#// scanner.in -*-C-*-
+
+%%{
+    machine StdTok;
+    include URL "url.rl";
+    include Email "email.rl";
+
+    token = frt_alpha frt_alnum*;
+
+    frt_tokenizer := |*
+        #// Email
+        email { RET; };
+
+        #// Token, or token with possessive
+        token          { RET; };
+        token [\']     { trunc = 1; RET; };
+        token [\'][sS] { trunc = 2; RET; };
+
+        #// contractions
+        frt_alpha+ [\'] frt_alpha+ { RET; };
+
+        #// Token with hyphens
+        frt_alnum+ ([\-_] frt_alnum+)* { RET; };
+
+        #// Company name
+        token [\&\@] token* { RET; };
+
+        #// URL
+        url { RET; };
+
+        #// Acronym
+        (frt_alpha '.')+ frt_alpha { STRIP('.'); };
+
+        #// Int+float
+        [\-\+]?frt_digit+                { RET; };
+        [\-\+]?frt_digit+ '.' frt_digit+ { RET; };
+
+        #// Ignore whitespace and other crap
+        0 { return; };
+        (any - frt_alnum) {};
+
+        *|;
+}%%

data/ext/isomorfeus_ferret_ext/scanner.rl

@@ -0,0 +1,84 @@
+/* scanner.rl -*-C-*- */
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include "frt_global.h"
+
+#define RET goto ret;
+
+#define STRIP(c) do { \
+    strip_char = c;   \
+    goto ret;         \
+} while(0)
+
+%%{
+    machine StdTok;
+    alphtype unsigned char;
+
+    frt_alpha = alpha;
+    frt_alnum = alnum;
+    frt_digit = digit;
+
+    include StdTok "scanner.in";
+
+    main := any @{ fhold; fcall frt_tokenizer; };
+}%%
+
+%% write data nofinal;
+
+void frt_std_scan(const char *in,
+                  char *out, size_t out_size,
+                  const char **start,
+                  const char **end,
+                  int *token_size)
+{
+    int cs, act, top;
+    int stack[32];
+    char *ts = 0, *te = 0;
+
+    %% write init;
+
+    char *p = (char *)in, *pe = 0, *eof = pe;
+    int skip = 0;
+    int trunc = 0;
+    char strip_char = 0;
+
+    *end = 0;
+    *start = 0;
+    *token_size = 0;
+
+    %% write exec;
+
+    if ( cs == StdTok_error )
+                   fprintf(stderr, "PARSE ERROR\n" );
+    else if ( ts ) fprintf(stderr, "STUFF LEFT: '%s'\n", ts);
+    return;
+
+ ret:
+    {
+        size_t __len = te - ts - skip - trunc;
+        if (__len > out_size)
+            __len = out_size;
+
+        *start = ts;
+        *end   = te;
+
+        if (strip_char) {
+            char *__p = ts + skip;
+            char *__o = out;
+            for (; __p < (ts + skip + __len); ++__p) {
+                if (*__p != strip_char)
+                    *__o++ = *__p;
+            }
+            *token_size = __o - out;
+        }
+        else {
+            memcpy(out, ts + skip, __len);
+            *token_size = __len;
+        }
+
+        out[*token_size] = 0;
+    }
+}

data/ext/isomorfeus_ferret_ext/scanner_mb.rl

@@ -0,0 +1,200 @@
+/* scanner.rl -*-C-*- */
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <wchar.h>
+#include <wctype.h>
+#include "frt_global.h"
+
+#define RET goto ret;
+
+#define STRIP(c) do { \
+    strip_char = c;   \
+    goto ret;         \
+} while(0)
+
+%%{
+    machine StdTok;
+    alphtype unsigned int;
+    include WChar "uchar-ucs4.rl";
+
+    frt_alpha = ualpha;
+    frt_alnum = ualnum;
+    frt_digit = udigit;
+
+    include StdTok "scanner.in";
+
+    main := any @{ fhold; fcall frt_tokenizer; };
+}%%
+
+%% write data nofinal;
+
+static const char *position_in_mb( const unsigned int *orig_wc,
+                                   const char *orig_mb,
+                                   const unsigned int *curr_wc )
+{
+    const char         *mb = orig_mb;
+    const unsigned int *wc = orig_wc;
+
+    while (wc < curr_wc)
+    {
+        char buf[MB_LEN_MAX];
+        mb += wctomb(buf, *wc);
+        ++wc;
+    }
+
+    return mb;
+}
+
+static int mb_next_char(unsigned int *wchr, const char *s, mbstate_t *state)
+{
+    int num_bytes;
+    if ((num_bytes = (int)mbrtowc((wchar_t*)wchr, s, MB_CUR_MAX, state)) < 0) {
+        const char *t = s;
+        do {
+            t++;
+            FRT_ZEROSET(state, mbstate_t);
+            num_bytes = (int)mbrtowc((wchar_t*)wchr, t, MB_CUR_MAX, state);
+        } while ((num_bytes < 0) && (*t != 0));
+        num_bytes = t - s;
+        if (*t == 0) *wchr = 0;
+    }
+    return num_bytes;
+}
+
+static int wc_next_char(char *s, const unsigned int *wchr, mbstate_t *state)
+{
+    return (int)wcrtomb(s, *wchr, state);
+}
+
+/*
+ * All input to Ragel must be in a buffer of 32bit unicode codepoints.
+ * To that end, we require that the input to the scanner must be in a
+ * codepage that mbtowc will convert to unicode codepoints.  The easy
+ * way to do this is to supply the scanner with UTF8.  It will call
+ * mbtowc on the buffer, pass it to the tokenizer which will extract
+ * one token out.  This token will then be converted back with wctomb.
+ *
+ * frt_scan_mb takes in a pointer to the mb buffer +inmb+ that has max
+ * size +in_size+.  The resulting token will be stored in +outmb+,
+ * with at most +out_size+ bytes written.
+ *
+ * While tokenizing, part of the token may be stripped out. Eg,
+ * 'foo!!' will become 'foo', and 'http://www.bar.com' will become
+ * 'www.bar.com'.  So that the caller can track what exactly has been
+ * tokenized in +inmb+, +startmb+ and +endmb+ are set to point to the
+ * extremeties of the original unmodified token in +inmb+.
+ *
+ * The size of the token written out to +outmb+ is stored in
+ * +token_size+.
+ */
+
+static void mb_to_wc(const char *in,
+                     unsigned int *out, size_t out_size)
+{
+    mbstate_t state;
+    const char *in_p     = in;
+    unsigned int *out_p = out;
+    FRT_ZEROSET(&state, mbstate_t);
+
+    while (*in_p && out_p < (out + out_size/sizeof(*out)))
+    {
+        /* We can break out early here on, say, a space XXX */
+
+        int n = mb_next_char(out_p, in_p, &state);
+        if (n < 0)
+        {
+            ++in_p;
+            continue;
+        }
+
+        in_p += n;
+        ++out_p;
+    }
+}
+
+static void wc_to_mb(char *out, size_t out_size, int *token_size,
+                     const unsigned int *in_wc, size_t in_wc_size)
+{
+    mbstate_t state;
+    char *out_p = out;
+    const unsigned int *in_wc_p = in_wc;
+    FRT_ZEROSET(&state, mbstate_t);
+    *token_size = 0;
+
+    while (out_p < (out + out_size) && in_wc_p < (in_wc + in_wc_size))
+    {
+        if (!*in_wc_p)
+            break;
+
+        int n = wc_next_char(out_p, in_wc_p, &state);
+        if (n < 0)
+        {
+            ++in_wc_p;
+            continue;
+        }
+
+        out_p += n;
+        ++in_wc_p;
+    }
+
+    *token_size = out_p - out;
+}
+
+void frt_std_scan_mb(const char *in_mb,
+                     char *out_mb, size_t out_mb_size,
+                     const char **start_mb,
+                     const char **end_mb,
+                     int *token_size)
+{
+    int cs, act, top;
+    int stack[32];
+    unsigned int *ts = 0, *te = 0;
+
+    %% write init;
+
+    unsigned int in_wc[4096] = {0};
+    mb_to_wc(in_mb, in_wc, sizeof(in_wc));
+
+    unsigned int *p = in_wc, *pe = 0, *eof = pe;
+    int skip = 0;
+    int trunc = 0;
+    unsigned int strip_char = 0;
+
+    *end_mb = 0;
+    *start_mb = 0;
+    *token_size = 0;
+
+    %% write exec;
+
+    if ( cs == StdTok_error )
+                   fwprintf(stderr, L"PARSE ERROR\n");
+    else if ( ts ) fwprintf(stderr, L"STUFF LEFT: '%ls'\n", ts);
+    return;
+
+ ret:
+    {
+        unsigned int out_wc[4096] = {0};
+        size_t __len = te - ts - skip - trunc;
+
+        *start_mb = position_in_mb(in_wc, in_mb, ts);
+        *end_mb   = position_in_mb(in_wc, in_mb, te);
+
+        if (strip_char) {
+            unsigned int *__p = ts + skip;
+            unsigned int *__o = out_wc;
+            for (; __p < (ts + skip + __len); ++__p) {
+                if (*__p != strip_char)
+                    *__o++ = *__p;
+            }
+        }
+        else {
+            memcpy(out_wc, ts + skip, __len*sizeof(unsigned int));
+        }
+
+        wc_to_mb(out_mb, out_mb_size, token_size, out_wc, sizeof(out_wc));
+        out_mb[*token_size] = 0;
+    }
+}

data/ext/isomorfeus_ferret_ext/scanner_utf8.rl

@@ -0,0 +1,85 @@
+/* scanner.rl -*-C-*- */
+#include <ctype.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include "frt_global.h"
+
+#define RET goto ret;
+
+#define STRIP(c) do { \
+    strip_char = c;   \
+    goto ret;         \
+} while(0)
+
+%%{
+    machine StdTok;
+    alphtype unsigned char;
+    include UTF8 "uchar-utf8.rl";
+
+    frt_alpha = ualpha;
+    frt_alnum = ualnum;
+    frt_digit = udigit;
+
+    include StdTok "scanner.in";
+
+    main := any @{ fhold; fcall frt_tokenizer; };
+}%%
+
+%% write data nofinal;
+
+void frt_std_scan_utf8(const char *in,
+                       char *out, size_t out_size,
+                       const char **start,
+                       const char **end,
+                       int *token_size)
+{
+    int cs, act, top;
+    int stack[32];
+    char *ts = 0, *te = 0;
+
+    %% write init;
+
+    char *p = (char *)in, *pe = 0, *eof = pe;
+    int skip = 0;
+    int trunc = 0;
+    char strip_char = 0;
+
+    *end = 0;
+    *start = 0;
+    *token_size = 0;
+
+    %% write exec;
+
+    if ( cs == StdTok_error )
+                   fprintf(stderr, "PARSE ERROR\n" );
+    else if ( ts ) fprintf(stderr, "STUFF LEFT: '%s'\n", ts);
+    return;
+
+ ret:
+    {
+        size_t __len = te - ts - skip - trunc;
+        if (__len > out_size)
+            __len = out_size;
+
+        *start = ts;
+        *end   = te;
+
+        if (strip_char) {
+            char *__p = ts + skip;
+            char *__o = out;
+            for (; __p < (ts + skip + __len); ++__p) {
+                if (*__p != strip_char)
+                    *__o++ = *__p;
+            }
+            *token_size = __o - out;
+        }
+        else {
+            memcpy(out, ts + skip, __len);
+            *token_size = __len;
+        }
+
+        out[*token_size] = 0;
+    }
+}