isomorfeus-ferret 0.12.7 → 0.13.0

data/ext/isomorfeus_ferret_ext/scanner.h

@@ -1,28 +0,0 @@
-#ifndef FRT_SCANNER_H
-#define FRT_SCANNER_H
-
-/*
- * Scan +in+ and copy the token into +out+, up until +out_size+ bytes.
- * The +start+ and +end+ are pointers to the original untouched token
- * somewhere inside +in+.  This token may not always be copied
- * verbatim into +out+.  For example, the http://google.com token will
- * be truncated down to just google.com during the copy.
- * +token_length+ is the size of the resulting token.
- */
-void frt_std_scan(const char *in,
-                  char *out, size_t out_size,
-                  const char **start, const char **end,
-                  int *token_length);
-
-
-void frt_std_scan_mb(const char *in,
-                     char *out, size_t out_size,
-                     const char **start, const char **end,
-                     int *token_length);
-
-void frt_std_scan_utf8(const char *in,
-                       char *out, size_t out_size,
-                       const char **start, const char **end,
-                       int *token_length);
-
-#endif /* FRT_SCANNER */

data/ext/isomorfeus_ferret_ext/scanner.in

@@ -1,43 +0,0 @@
-#// scanner.in -*-C-*-
-
-%%{
-    machine StdTok;
-    include URL "url.rl";
-    include Email "email.rl";
-
-    token = frt_alpha frt_alnum*;
-
-    frt_tokenizer := |*
-        #// Email
-        email { RET; };
-
-        #// Token, or token with possessive
-        token          { RET; };
-        token [\']     { trunc = 1; RET; };
-        token [\'][sS] { trunc = 2; RET; };
-
-        #// contractions
-        frt_alpha+ [\'] frt_alpha+ { RET; };
-
-        #// Token with hyphens
-        frt_alnum+ ([\-_] frt_alnum+)* { RET; };
-
-        #// Company name
-        token [\&\@] token* { RET; };
-
-        #// URL
-        url { RET; };
-
-        #// Acronym
-        (frt_alpha '.')+ frt_alpha { STRIP('.'); };
-
-        #// Int+float
-        [\-\+]?frt_digit+                { RET; };
-        [\-\+]?frt_digit+ '.' frt_digit+ { RET; };
-
-        #// Ignore whitespace and other crap
-        0 { return; };
-        (any - frt_alnum) {};
-
-        *|;
-}%%

data/ext/isomorfeus_ferret_ext/scanner.rl

@@ -1,84 +0,0 @@
-/* scanner.rl -*-C-*- */
-#include <ctype.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include "frt_global.h"
-
-#define RET goto ret;
-
-#define STRIP(c) do { \
-    strip_char = c;   \
-    goto ret;         \
-} while(0)
-
-%%{
-    machine StdTok;
-    alphtype unsigned char;
-
-    frt_alpha = alpha;
-    frt_alnum = alnum;
-    frt_digit = digit;
-
-    include StdTok "scanner.in";
-
-    main := any @{ fhold; fcall frt_tokenizer; };
-}%%
-
-%% write data nofinal;
-
-void frt_std_scan(const char *in,
-                  char *out, size_t out_size,
-                  const char **start,
-                  const char **end,
-                  int *token_size)
-{
-    int cs, act, top;
-    int stack[32];
-    char *ts = 0, *te = 0;
-
-    %% write init;
-
-    char *p = (char *)in, *pe = 0, *eof = pe;
-    int skip = 0;
-    int trunc = 0;
-    char strip_char = 0;
-
-    *end = 0;
-    *start = 0;
-    *token_size = 0;
-
-    %% write exec;
-
-    if ( cs == StdTok_error )
-                   fprintf(stderr, "PARSE ERROR\n" );
-    else if ( ts ) fprintf(stderr, "STUFF LEFT: '%s'\n", ts);
-    return;
-
- ret:
-    {
-        size_t __len = te - ts - skip - trunc;
-        if (__len > out_size)
-            __len = out_size;
-
-        *start = ts;
-        *end   = te;
-
-        if (strip_char) {
-            char *__p = ts + skip;
-            char *__o = out;
-            for (; __p < (ts + skip + __len); ++__p) {
-                if (*__p != strip_char)
-                    *__o++ = *__p;
-            }
-            *token_size = __o - out;
-        }
-        else {
-            memcpy(out, ts + skip, __len);
-            *token_size = __len;
-        }
-
-        out[*token_size] = 0;
-    }
-}

data/ext/isomorfeus_ferret_ext/scanner_mb.rl

@@ -1,200 +0,0 @@
-/* scanner.rl -*-C-*- */
-#include <ctype.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <wchar.h>
-#include <wctype.h>
-#include "frt_global.h"
-
-#define RET goto ret;
-
-#define STRIP(c) do { \
-    strip_char = c;   \
-    goto ret;         \
-} while(0)
-
-%%{
-    machine StdTok;
-    alphtype unsigned int;
-    include WChar "uchar-ucs4.rl";
-
-    frt_alpha = ualpha;
-    frt_alnum = ualnum;
-    frt_digit = udigit;
-
-    include StdTok "scanner.in";
-
-    main := any @{ fhold; fcall frt_tokenizer; };
-}%%
-
-%% write data nofinal;
-
-static const char *position_in_mb( const unsigned int *orig_wc,
-                                   const char *orig_mb,
-                                   const unsigned int *curr_wc )
-{
-    const char         *mb = orig_mb;
-    const unsigned int *wc = orig_wc;
-
-    while (wc < curr_wc)
-    {
-        char buf[MB_LEN_MAX];
-        mb += wctomb(buf, *wc);
-        ++wc;
-    }
-
-    return mb;
-}
-
-static int mb_next_char(unsigned int *wchr, const char *s, mbstate_t *state)
-{
-    int num_bytes;
-    if ((num_bytes = (int)mbrtowc((wchar_t*)wchr, s, MB_CUR_MAX, state)) < 0) {
-        const char *t = s;
-        do {
-            t++;
-            FRT_ZEROSET(state, mbstate_t);
-            num_bytes = (int)mbrtowc((wchar_t*)wchr, t, MB_CUR_MAX, state);
-        } while ((num_bytes < 0) && (*t != 0));
-        num_bytes = t - s;
-        if (*t == 0) *wchr = 0;
-    }
-    return num_bytes;
-}
-
-static int wc_next_char(char *s, const unsigned int *wchr, mbstate_t *state)
-{
-    return (int)wcrtomb(s, *wchr, state);
-}
-
-/*
- * All input to Ragel must be in a buffer of 32bit unicode codepoints.
- * To that end, we require that the input to the scanner must be in a
- * codepage that mbtowc will convert to unicode codepoints.  The easy
- * way to do this is to supply the scanner with UTF8.  It will call
- * mbtowc on the buffer, pass it to the tokenizer which will extract
- * one token out.  This token will then be converted back with wctomb.
- *
- * frt_scan_mb takes in a pointer to the mb buffer +inmb+ that has max
- * size +in_size+.  The resulting token will be stored in +outmb+,
- * with at most +out_size+ bytes written.
- *
- * While tokenizing, part of the token may be stripped out. Eg,
- * 'foo!!' will become 'foo', and 'http://www.bar.com' will become
- * 'www.bar.com'.  So that the caller can track what exactly has been
- * tokenized in +inmb+, +startmb+ and +endmb+ are set to point to the
- * extremeties of the original unmodified token in +inmb+.
- *
- * The size of the token written out to +outmb+ is stored in
- * +token_size+.
- */
-
-static void mb_to_wc(const char *in,
-                     unsigned int *out, size_t out_size)
-{
-    mbstate_t state;
-    const char *in_p     = in;
-    unsigned int *out_p = out;
-    FRT_ZEROSET(&state, mbstate_t);
-
-    while (*in_p && out_p < (out + out_size/sizeof(*out)))
-    {
-        /* We can break out early here on, say, a space XXX */
-
-        int n = mb_next_char(out_p, in_p, &state);
-        if (n < 0)
-        {
-            ++in_p;
-            continue;
-        }
-
-        in_p += n;
-        ++out_p;
-    }
-}
-
-static void wc_to_mb(char *out, size_t out_size, int *token_size,
-                     const unsigned int *in_wc, size_t in_wc_size)
-{
-    mbstate_t state;
-    char *out_p = out;
-    const unsigned int *in_wc_p = in_wc;
-    FRT_ZEROSET(&state, mbstate_t);
-    *token_size = 0;
-
-    while (out_p < (out + out_size) && in_wc_p < (in_wc + in_wc_size))
-    {
-        if (!*in_wc_p)
-            break;
-
-        int n = wc_next_char(out_p, in_wc_p, &state);
-        if (n < 0)
-        {
-            ++in_wc_p;
-            continue;
-        }
-
-        out_p += n;
-        ++in_wc_p;
-    }
-
-    *token_size = out_p - out;
-}
-
-void frt_std_scan_mb(const char *in_mb,
-                     char *out_mb, size_t out_mb_size,
-                     const char **start_mb,
-                     const char **end_mb,
-                     int *token_size)
-{
-    int cs, act, top;
-    int stack[32];
-    unsigned int *ts = 0, *te = 0;
-
-    %% write init;
-
-    unsigned int in_wc[4096] = {0};
-    mb_to_wc(in_mb, in_wc, sizeof(in_wc));
-
-    unsigned int *p = in_wc, *pe = 0, *eof = pe;
-    int skip = 0;
-    int trunc = 0;
-    unsigned int strip_char = 0;
-
-    *end_mb = 0;
-    *start_mb = 0;
-    *token_size = 0;
-
-    %% write exec;
-
-    if ( cs == StdTok_error )
-                   fwprintf(stderr, L"PARSE ERROR\n");
-    else if ( ts ) fwprintf(stderr, L"STUFF LEFT: '%ls'\n", ts);
-    return;
-
- ret:
-    {
-        unsigned int out_wc[4096] = {0};
-        size_t __len = te - ts - skip - trunc;
-
-        *start_mb = position_in_mb(in_wc, in_mb, ts);
-        *end_mb   = position_in_mb(in_wc, in_mb, te);
-
-        if (strip_char) {
-            unsigned int *__p = ts + skip;
-            unsigned int *__o = out_wc;
-            for (; __p < (ts + skip + __len); ++__p) {
-                if (*__p != strip_char)
-                    *__o++ = *__p;
-            }
-        }
-        else {
-            memcpy(out_wc, ts + skip, __len*sizeof(unsigned int));
-        }
-
-        wc_to_mb(out_mb, out_mb_size, token_size, out_wc, sizeof(out_wc));
-        out_mb[*token_size] = 0;
-    }
-}

data/ext/isomorfeus_ferret_ext/scanner_utf8.rl

@@ -1,85 +0,0 @@
-/* scanner.rl -*-C-*- */
-#include <ctype.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include "frt_global.h"
-
-#define RET goto ret;
-
-#define STRIP(c) do { \
-    strip_char = c;   \
-    goto ret;         \
-} while(0)
-
-%%{
-    machine StdTok;
-    alphtype unsigned char;
-    include UTF8 "uchar-utf8.rl";
-
-    frt_alpha = ualpha;
-    frt_alnum = ualnum;
-    frt_digit = udigit;
-
-    include StdTok "scanner.in";
-
-    main := any @{ fhold; fcall frt_tokenizer; };
-}%%
-
-%% write data nofinal;
-
-void frt_std_scan_utf8(const char *in,
-                       char *out, size_t out_size,
-                       const char **start,
-                       const char **end,
-                       int *token_size)
-{
-    int cs, act, top;
-    int stack[32];
-    char *ts = 0, *te = 0;
-
-    %% write init;
-
-    char *p = (char *)in, *pe = 0, *eof = pe;
-    int skip = 0;
-    int trunc = 0;
-    char strip_char = 0;
-
-    *end = 0;
-    *start = 0;
-    *token_size = 0;
-
-    %% write exec;
-
-    if ( cs == StdTok_error )
-                   fprintf(stderr, "PARSE ERROR\n" );
-    else if ( ts ) fprintf(stderr, "STUFF LEFT: '%s'\n", ts);
-    return;
-
- ret:
-    {
-        size_t __len = te - ts - skip - trunc;
-        if (__len > out_size)
-            __len = out_size;
-
-        *start = ts;
-        *end   = te;
-
-        if (strip_char) {
-            char *__p = ts + skip;
-            char *__o = out;
-            for (; __p < (ts + skip + __len); ++__p) {
-                if (*__p != strip_char)
-                    *__o++ = *__p;
-            }
-            *token_size = __o - out;
-        }
-        else {
-            memcpy(out, ts + skip, __len);
-            *token_size = __len;
-        }
-
-        out[*token_size] = 0;
-    }
-}