sdsykes-ferret 0.11.6.19

data/ext/analysis.h

@@ -0,0 +1,219 @@
+#ifndef FRT_ANALYSIS_H
+#define FRT_ANALYSIS_H
+
+#include "global.h"
+#include "hash.h"
+#include "multimapper.h"
+#include <wchar.h>
+
+/****************************************************************************
+ *
+ * Token
+ *
+ ****************************************************************************/
+
+typedef struct Token
+{
+    char text[MAX_WORD_SIZE];
+    int len;
+    off_t start;
+    off_t end;
+    int pos_inc;
+} Token;
+
+extern Token *tk_new();
+extern void tk_destroy(void *p);
+extern Token *tk_set(Token *tk, char *text, int tlen, off_t start, off_t end,
+                     int pos_inc);
+extern Token *tk_set_no_len(Token *tk, char *text, off_t start, off_t end,
+                            int pos_inc);
+extern int tk_eq(Token *tk1, Token *tk2);
+extern int tk_cmp(Token *tk1, Token *tk2);
+
+/****************************************************************************
+ *
+ * TokenStream
+ *
+ ****************************************************************************/
+
+
+typedef struct TokenStream TokenStream;
+struct TokenStream
+{
+    char        *t;             /* ptr used to scan text */
+    char        *text;
+    Token       *(*next)(TokenStream *ts);
+    TokenStream *(*reset)(TokenStream *ts, char *text);
+    TokenStream *(*clone_i)(TokenStream *ts);
+    void         (*destroy_i)(TokenStream *ts);
+    int          ref_cnt;
+};
+
+#define ts_new(type) ts_new_i(sizeof(type))
+extern TokenStream *ts_new_i(size_t size);
+extern TokenStream *ts_clone_size(TokenStream *orig_ts, size_t size);
+
+typedef struct CachedTokenStream
+{
+    TokenStream super;
+    Token       token;
+} CachedTokenStream;
+
+typedef struct MultiByteTokenStream
+{
+    CachedTokenStream super;
+    mbstate_t         state;
+} MultiByteTokenStream;
+
+typedef struct StandardTokenizer
+{
+    CachedTokenStream super;
+    bool        (*advance_to_start)(TokenStream *ts);
+    bool        (*is_tok_char)(char *c);
+    int         (*get_alpha)(TokenStream *ts, char *token);
+    int         (*get_apostrophe)(char *input);
+} StandardTokenizer;
+
+typedef struct TokenFilter
+{
+    TokenStream super;
+    TokenStream *sub_ts;
+} TokenFilter;
+
+extern TokenStream *filter_clone_size(TokenStream *ts, size_t size);
+#define tf_new(type, sub) tf_new_i(sizeof(type), sub)
+extern TokenStream *tf_new_i(size_t size, TokenStream *sub_ts);
+
+typedef struct StopFilter 
+{
+    TokenFilter super;
+    HashTable  *words;
+} StopFilter;
+
+typedef struct MappingFilter 
+{
+    TokenFilter  super;
+    MultiMapper *mapper;
+} MappingFilter;
+
+typedef struct HyphenFilter 
+{
+    TokenFilter super;
+    char text[MAX_WORD_SIZE];
+    int start;
+    int pos;
+    int len;
+    Token *tk;
+} HyphenFilter;
+
+typedef struct StemFilter
+{
+    TokenFilter        super;
+    struct sb_stemmer  *stemmer;
+    char               *algorithm;
+    char               *charenc;
+} StemFilter;
+
+#define ts_next(mts) mts->next(mts)
+#define ts_clone(mts) mts->clone_i(mts)
+
+extern void ts_deref(TokenStream *ts);
+
+extern TokenStream *non_tokenizer_new();
+
+extern TokenStream *whitespace_tokenizer_new();
+extern TokenStream *mb_whitespace_tokenizer_new(bool lowercase);
+
+extern TokenStream *letter_tokenizer_new();
+extern TokenStream *mb_letter_tokenizer_new(bool lowercase);
+
+extern TokenStream *standard_tokenizer_new();
+extern TokenStream *mb_standard_tokenizer_new();
+
+extern TokenStream *hyphen_filter_new(TokenStream *ts);
+extern TokenStream *lowercase_filter_new(TokenStream *ts);
+extern TokenStream *mb_lowercase_filter_new(TokenStream *ts);
+
+extern const char *ENGLISH_STOP_WORDS[];
+extern const char *FULL_ENGLISH_STOP_WORDS[];
+extern const char *EXTENDED_ENGLISH_STOP_WORDS[];
+extern const char *FULL_FRENCH_STOP_WORDS[];
+extern const char *FULL_SPANISH_STOP_WORDS[];
+extern const char *FULL_PORTUGUESE_STOP_WORDS[];
+extern const char *FULL_ITALIAN_STOP_WORDS[];
+extern const char *FULL_GERMAN_STOP_WORDS[];
+extern const char *FULL_DUTCH_STOP_WORDS[];
+extern const char *FULL_SWEDISH_STOP_WORDS[];
+extern const char *FULL_NORWEGIAN_STOP_WORDS[];
+extern const char *FULL_DANISH_STOP_WORDS[];
+extern const char *FULL_RUSSIAN_STOP_WORDS[];
+extern const char *FULL_FINNISH_STOP_WORDS[];
+
+extern TokenStream *stop_filter_new_with_words_len(TokenStream *ts,
+                                                   const char **words, int len);
+extern TokenStream *stop_filter_new_with_words(TokenStream *ts,
+                                               const char **words);
+extern TokenStream *stop_filter_new(TokenStream *ts);
+extern TokenStream *stem_filter_new(TokenStream *ts, const char *algorithm,
+                                    const char *charenc);
+
+extern TokenStream *mapping_filter_new(TokenStream *ts);
+extern TokenStream *mapping_filter_add(TokenStream *ts, const char *pattern,
+                                       const char *replacement);
+
+/****************************************************************************
+ *
+ * Analyzer
+ *
+ ****************************************************************************/
+
+typedef struct Analyzer
+{
+    TokenStream *current_ts;
+    TokenStream *(*get_ts)(struct Analyzer *a, char *field, char *text);
+    void (*destroy_i)(struct Analyzer *a);
+    int ref_cnt;
+} Analyzer;
+
+extern void a_deref(Analyzer *a);
+
+#define a_get_ts(ma, field, text) ma->get_ts(ma, field, text)
+
+extern Analyzer *analyzer_new(TokenStream *ts,
+                              void (*destroy)(Analyzer *a),
+                              TokenStream *(*get_ts)(Analyzer *a,
+                                                     char *field,
+                                                     char *text));
+extern void a_standard_destroy(Analyzer *a);
+extern Analyzer *non_analyzer_new();
+
+extern Analyzer *whitespace_analyzer_new(bool lowercase);
+extern Analyzer *mb_whitespace_analyzer_new(bool lowercase);
+
+extern Analyzer *letter_analyzer_new(bool lowercase);
+extern Analyzer *mb_letter_analyzer_new(bool lowercase);
+
+extern Analyzer *standard_analyzer_new(bool lowercase);
+extern Analyzer *mb_standard_analyzer_new(bool lowercase);
+
+extern Analyzer *standard_analyzer_new_with_words(const char **words,
+                                                  bool lowercase);
+extern Analyzer *standard_analyzer_new_with_words_len(const char **words, int len,
+                                                      bool lowercase);
+extern Analyzer *mb_standard_analyzer_new_with_words(const char **words,
+                                                     bool lowercase);
+extern Analyzer *mb_standard_analyzer_new_with_words_len(const char **words,
+                                                  int len, bool lowercase);
+
+#define PFA(analyzer) ((PerFieldAnalyzer *)(analyzer))
+typedef struct PerFieldAnalyzer
+{
+    Analyzer    super;
+    HashTable  *dict;
+    Analyzer   *default_a;
+} PerFieldAnalyzer;
+
+extern Analyzer *per_field_analyzer_new(Analyzer *a);
+extern void pfa_add_field(Analyzer *self, char *field, Analyzer *analyzer);
+
+#endif

data/ext/api.c

@@ -0,0 +1,69 @@
+
+#include <stdlib.h> /* for calloc, free */
+#include "header.h"
+
+extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size)
+{
+    struct SN_env * z = (struct SN_env *) calloc(1, sizeof(struct SN_env));
+    if (z == NULL) return NULL;
+    z->p = create_s();
+    if (z->p == NULL) goto error;
+    if (S_size)
+    {
+        int i;
+        z->S = (symbol * *) calloc(S_size, sizeof(symbol *));
+        if (z->S == NULL) goto error;
+
+        for (i = 0; i < S_size; i++)
+        {
+            z->S[i] = create_s();
+            if (z->S[i] == NULL) goto error;
+        }
+        z->S_size = S_size;
+    }
+
+    if (I_size)
+    {
+        z->I = (int *) calloc(I_size, sizeof(int));
+        if (z->I == NULL) goto error;
+        z->I_size = I_size;
+    }
+
+    if (B_size)
+    {
+        z->B = (symbol *) calloc(B_size, sizeof(symbol));
+        if (z->B == NULL) goto error;
+        z->B_size = B_size;
+    }
+
+    return z;
+error:
+    SN_close_env(z);
+    return NULL;
+}
+
+extern void SN_close_env(struct SN_env * z)
+{
+    if (z == NULL) return;
+    if (z->S_size)
+    {
+        int i;
+        for (i = 0; i < z->S_size; i++)
+        {
+            lose_s(z->S[i]);
+        }
+        free(z->S);
+    }
+    if (z->I_size) free(z->I);
+    if (z->B_size) free(z->B);
+    if (z->p) lose_s(z->p);
+    free(z);
+}
+
+extern int SN_set_current(struct SN_env * z, int size, const symbol * s)
+{
+    int err = replace_s(z, 0, z->l, size, s, NULL);
+    z->c = 0;
+    return err;
+}
+

data/ext/api.h

@@ -0,0 +1,27 @@
+
+typedef unsigned char symbol;
+
+/* Or replace 'char' above with 'short' for 16 bit characters.
+
+   More precisely, replace 'char' with whatever type guarantees the
+   character width you need. Note however that sizeof(symbol) should divide
+   HEAD, defined in header.h as 2*sizeof(int), without remainder, otherwise
+   there is an alignment problem. In the unlikely event of a problem here,
+   consult Martin Porter.
+
+*/
+
+struct SN_env {
+    symbol * p;
+    int c; int a; int l; int lb; int bra; int ket;
+    int S_size; int I_size; int B_size;
+    symbol * * S;
+    int * I;
+    symbol * B;
+};
+
+extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size);
+extern void SN_close_env(struct SN_env * z);
+
+extern int SN_set_current(struct SN_env * z, int size, const symbol * s);
+

data/ext/array.c

@@ -0,0 +1,123 @@
+#include "array.h"
+#include <string.h>
+
+#define META_CNT ARY_META_CNT
+#define DATA_SZ sizeof(int) * META_CNT
+
+void **ary_new_i(int type_size, int init_capa)
+{
+    void **ary;
+    if (init_capa <= 0) {
+        init_capa = ARY_INIT_CAPA;
+    }
+    ary = (void **)&(((int *)ecalloc(DATA_SZ + init_capa * type_size))[META_CNT]);
+    ary_type_size(ary) = type_size;
+    ary_capa(ary) = init_capa;
+    return ary;
+}
+
+INLINE void ary_resize_i(void ***ary, int size)
+{
+    size++;
+    if (size >= ary_sz(*ary)) {
+        int capa = ary_capa(*ary);
+        if (size >= capa) {
+            int *ary_start = &((int *)*ary)[-META_CNT];
+            while (size >= capa) {
+                capa <<= 1;
+            }
+
+            ary_start = (int *)erealloc(ary_start,
+                                        DATA_SZ + capa * ary_type_size(*ary));
+            *ary = (void **)&(ary_start[META_CNT]);
+            memset(((char *)*ary) + ary_type_size(*ary) * ary_sz(*ary), 0,
+                   (capa - ary_sz(*ary)) * ary_type_size(*ary));
+            ary_capa(*ary) = capa;
+        }
+        ary_sz(*ary) = size;
+    }
+}
+
+void ary_set_i(void ***ary, int index, void *value)
+{
+    if (index < 0) {
+        index += ary_sz(*ary);
+        if (index < 0) {
+            RAISE(INDEX_ERROR, "index %d out array", index);
+        }
+    }
+    ary_resize_i(ary, index);
+    (*ary)[index] = value;
+}
+
+void *ary_get_i(void **ary, int index)
+{
+    if (index < 0) {
+        index += ary_sz(ary);
+    }
+    if (index >= 0 && index < ary_sz(ary)) {
+        return ary[index];
+    }
+    else {
+        return NULL;
+    }
+}
+
+void ary_push_i(void ***ary, void *value)
+{
+    int size = ary_sz(*ary);
+    ary_resize_i(ary, size);
+    (*ary)[size] = value;
+}
+
+void *ary_pop_i(void **ary)
+{
+    void *val = ary[--ary_sz(ary)];
+    ary[ary_sz(ary)] = NULL;
+    return val;
+}
+
+void ary_unshift_i(void ***ary, void *value)
+{
+    int size = ary_sz(*ary);
+    ary_resize_i(ary, size);
+    memmove(*ary, *ary + 1, size * sizeof(void *));
+    (*ary)[0] = value;
+}
+
+void *ary_shift_i(void **ary)
+{
+    void *val = ary[0];
+    int size = --ary_sz(ary);
+    memmove(ary, ary + 1, size * sizeof(void *));
+    ary[size] = NULL;
+    return val;
+}
+
+void *ary_remove_i(void **ary, int index)
+{
+    if (index >= 0 && index < ary_sz(ary)) {
+        void *val = ary[index];
+        memmove(ary + index, ary + index + 1,
+                (ary_sz(ary) - index + 1) * sizeof(void *));
+        ary_sz(ary)--;
+        return val;
+    }
+    else {
+        return NULL;
+    }
+}
+
+void ary_delete_i(void **ary, int index, void (*free_elem)(void *p))
+{
+    free_elem(ary_remove(ary, index));
+}
+
+void ary_destroy_i(void **ary, void (*free_elem)(void *p))
+{
+    int i;
+    for (i = ary_sz(ary) - 1; i >= 0; i--) {
+        free_elem(ary[i]);
+    }
+    ary_free(ary);
+}

data/ext/array.h

@@ -0,0 +1,53 @@
+#ifndef FRT_ARRAY_H
+#define FRT_ARRAY_H
+#include "global.h"
+
+#if defined POSH_OS_SOLARIS || defined POSH_OS_SUNOS
+# define ARY_META_CNT 4
+#else
+# define ARY_META_CNT 3
+#endif
+
+#define ARY_INIT_CAPA 8
+#define ary_size(ary)      ary_sz(ary)
+#define ary_sz(ary)        (((int *)ary)[-1])
+#define ary_capa(ary)      (((int *)ary)[-2])
+#define ary_type_size(ary) (((int *)ary)[-3])
+#define ary_start(ary)     ((void **)&(((int *)ary)[-ARY_META_CNT]))
+#define ary_free(ary)      free(ary_start(ary))
+
+#define ary_new_type_capa(type, init_capa)\
+                                (type *)ary_new_i(sizeof(type), init_capa)
+#define ary_new_type(type)      (type *)ary_new_i(sizeof(type), 0)
+#define ary_new_capa(init_capa) ary_new_i(sizeof(void *), init_capa)
+#define ary_new()               ary_new_i(sizeof(void *), 0)
+#define ary_resize(ary, size)   ary_resize_i(((void ***)(void *)&ary), size)
+#define ary_set(ary, i, val)    ary_set_i(((void ***)(void *)&ary), i, val)
+#define ary_get(ary, i)         ary_get_i(((void **)ary), i)
+#define ary_push(ary, val)      ary_push_i(((void ***)(void *)&ary), val)
+#define ary_pop(ary)            ary_pop_i(((void **)ary))
+#define ary_unshift(ary, val)   ary_unshift_i(((void ***)(void *)&ary), val)
+#define ary_shift(ary)          ary_shift_i(((void **)ary))
+#define ary_remove(ary, i)      ary_remove_i(((void **)ary), i)
+#define ary_delete(ary, i, f)   ary_delete_i(((void **)ary), i, (free_ft)f)
+#define ary_destroy(ary, f)     ary_destroy_i(((void **)ary), (free_ft)f)
+#define ary_rsz(ary, size)      ary_resize(ary, size)
+#define ary_grow(ary)           ary_resize(ary, ary_sz(ary))
+#define ary_last(ary)           ary[ary_sz(ary) - 1]
+#define ary_sort(ary, cmp)      qsort(ary, ary_size(ary), ary_type_size(ary), cmp)
+#define ary_each_rev(ary, i)    for (i = ary_size(ary) - 1; i >= 0; i--)
+#define ary_each(ary, i)        for (i = 0; i < ary_size(ary); i++)
+
+extern void   ary_resize_i(void ***ary, int size);
+extern void **ary_new_i(int type_size, int init_capa);
+extern void   ary_set_i(void ***ary, int index, void *value);
+extern void  *ary_get_i(void **ary, int index);
+extern void   ary_push_i(void ***ary, void *value);
+extern void  *ary_pop_i(void **ary);
+extern void   ary_unshift_i(void ***ary, void *value);
+extern void  *ary_shift_i(void **ary);
+extern void  *ary_remove_i(void **ary, int index);
+extern void   ary_delete_i(void **ary, int index, void (*free_elem)(void *p));
+extern void   ary_destroy_i(void **ary, void (*free_elem)(void *p));
+
+#endif