isomorfeus-ferret 0.12.0

data/ext/isomorfeus_ferret_ext/frt_analysis.h

@@ -0,0 +1,247 @@
+#ifndef FRT_ANALYSIS_H
+#define FRT_ANALYSIS_H
+
+#include <wchar.h>
+#include "frt_global.h"
+#include "frt_hash.h"
+#include "frt_multimapper.h"
+
+/****************************************************************************
+ *
+ * FrtToken
+ *
+ ****************************************************************************/
+
+typedef struct FrtToken
+{
+    char text[FRT_MAX_WORD_SIZE];
+    int len;
+    off_t start;
+    off_t end;
+    int pos_inc;
+} FrtToken;
+
+extern FrtToken *frt_tk_new();
+extern void frt_tk_destroy(void *p);
+extern FrtToken *frt_tk_set(FrtToken *tk, char *text, int tlen, off_t start, off_t end, int pos_inc);
+extern FrtToken *frt_tk_set_no_len(FrtToken *tk, char *text, off_t start, off_t end, int pos_inc);
+extern int frt_tk_eq(FrtToken *tk1, FrtToken *tk2);
+extern int frt_tk_cmp(FrtToken *tk1, FrtToken *tk2);
+
+/****************************************************************************
+ *
+ * FrtTokenStream
+ *
+ ****************************************************************************/
+
+
+typedef struct FrtTokenStream FrtTokenStream;
+struct FrtTokenStream
+{
+    char            *t;             /* ptr used to scan text */
+    char            *text;
+    FrtToken        *(*next)(FrtTokenStream *ts);
+    FrtTokenStream  *(*reset)(FrtTokenStream *ts, char *text);
+    FrtTokenStream  *(*clone_i)(FrtTokenStream *ts);
+    void            (*destroy_i)(FrtTokenStream *ts);
+    int             ref_cnt;
+};
+
+#define frt_ts_new(type) frt_ts_new_i(sizeof(type))
+extern FrtTokenStream *frt_ts_new_i(size_t size);
+extern FrtTokenStream *frt_ts_clone_size(FrtTokenStream *orig_ts, size_t size);
+
+typedef struct FrtCachedTokenStream
+{
+    FrtTokenStream super;
+    FrtToken       token;
+} FrtCachedTokenStream;
+
+typedef struct FrtMultiByteTokenStream
+{
+    FrtCachedTokenStream super;
+    mbstate_t            state;
+} FrtMultiByteTokenStream;
+
+typedef enum
+{
+    FRT_STT_ASCII,
+    FRT_STT_MB,
+    FRT_STT_UTF8
+} FrtStandardTokenizerType;
+
+typedef struct FrtStandardTokenizer
+{
+    FrtCachedTokenStream     super;
+    FrtStandardTokenizerType type;
+} FrtStandardTokenizer;
+
+typedef struct FrtLegacyStandardTokenizer
+{
+    FrtCachedTokenStream super;
+    bool        (*advance_to_start)(FrtTokenStream *ts);
+    bool        (*is_tok_char)(char *c);
+    int         (*get_alpha)(FrtTokenStream *ts, char *token);
+    int         (*get_apostrophe)(char *input);
+} FrtLegacyStandardTokenizer;
+
+typedef struct FrtTokenFilter
+{
+    FrtTokenStream super;
+    FrtTokenStream *sub_ts;
+} FrtTokenFilter;
+
+extern FrtTokenStream *frt_filter_clone_size(FrtTokenStream *ts, size_t size);
+#define tf_new(type, sub) frt_tf_new_i(sizeof(type), sub)
+extern FrtTokenStream *frt_tf_new_i(size_t size, FrtTokenStream *sub_ts);
+
+typedef struct FrtStopFilter
+{
+    FrtTokenFilter super;
+    FrtHash  *words;
+} FrtStopFilter;
+
+typedef struct FrtMappingFilter
+{
+    FrtTokenFilter  super;
+    FrtMultiMapper *mapper;
+} FrtMappingFilter;
+
+typedef struct FrtHyphenFilter
+{
+    FrtTokenFilter super;
+    char text[FRT_MAX_WORD_SIZE];
+    int start;
+    int pos;
+    int len;
+    FrtToken *tk;
+} FrtHyphenFilter;
+
+typedef struct FrtStemFilter
+{
+    FrtTokenFilter        super;
+    struct sb_stemmer  *stemmer;
+    char               *algorithm;
+    char               *charenc;
+} FrtStemFilter;
+
+#define frt_ts_next(mts) mts->next(mts)
+#define frt_ts_clone(mts) mts->clone_i(mts)
+
+extern void frt_ts_deref(FrtTokenStream *ts);
+
+extern FrtTokenStream *frt_non_tokenizer_new();
+
+extern FrtTokenStream *frt_whitespace_tokenizer_new();
+extern FrtTokenStream *frt_mb_whitespace_tokenizer_new(bool lowercase);
+
+extern FrtTokenStream *frt_letter_tokenizer_new();
+extern FrtTokenStream *frt_mb_letter_tokenizer_new(bool lowercase);
+
+extern FrtTokenStream *frt_standard_tokenizer_new();
+extern FrtTokenStream *frt_mb_standard_tokenizer_new();
+extern FrtTokenStream *frt_utf8_standard_tokenizer_new();
+
+extern FrtTokenStream *frt_legacy_standard_tokenizer_new();
+extern FrtTokenStream *frt_mb_legacy_standard_tokenizer_new();
+
+extern FrtTokenStream *frt_hyphen_filter_new(FrtTokenStream *ts);
+extern FrtTokenStream *frt_lowercase_filter_new(FrtTokenStream *ts);
+extern FrtTokenStream *frt_mb_lowercase_filter_new(FrtTokenStream *ts);
+
+extern const char *FRT_ENGLISH_STOP_WORDS[];
+extern const char *FRT_FULL_ENGLISH_STOP_WORDS[];
+extern const char *FRT_EXTENDED_ENGLISH_STOP_WORDS[];
+extern const char *FRT_FULL_FRENCH_STOP_WORDS[];
+extern const char *FRT_FULL_SPANISH_STOP_WORDS[];
+extern const char *FRT_FULL_PORTUGUESE_STOP_WORDS[];
+extern const char *FRT_FULL_ITALIAN_STOP_WORDS[];
+extern const char *FRT_FULL_GERMAN_STOP_WORDS[];
+extern const char *FRT_FULL_DUTCH_STOP_WORDS[];
+extern const char *FRT_FULL_SWEDISH_STOP_WORDS[];
+extern const char *FRT_FULL_NORWEGIAN_STOP_WORDS[];
+extern const char *FRT_FULL_DANISH_STOP_WORDS[];
+extern const char *FRT_FULL_RUSSIAN_STOP_WORDS[];
+extern const char *FRT_FULL_FINNISH_STOP_WORDS[];
+extern const char *FRT_FULL_HUNGARIAN_STOP_WORDS[];
+
+extern FrtTokenStream *frt_stop_filter_new_with_words_len(FrtTokenStream *ts,
+                                                   const char **words, int len);
+extern FrtTokenStream *frt_stop_filter_new_with_words(FrtTokenStream *ts,
+                                               const char **words);
+extern FrtTokenStream *frt_stop_filter_new(FrtTokenStream *ts);
+extern FrtTokenStream *frt_stem_filter_new(FrtTokenStream *ts, const char *algorithm,
+                                    const char *charenc);
+
+extern FrtTokenStream *frt_mapping_filter_new(FrtTokenStream *ts);
+extern FrtTokenStream *frt_mapping_filter_add(FrtTokenStream *ts, const char *pattern,
+                                       const char *replacement);
+
+/****************************************************************************
+ *
+ * FrtAnalyzer
+ *
+ ****************************************************************************/
+
+typedef struct FrtAnalyzer
+{
+    FrtTokenStream *current_ts;
+    FrtTokenStream *(*get_ts)(struct FrtAnalyzer *a, FrtSymbol field, char *text);
+    void (*destroy_i)(struct FrtAnalyzer *a);
+    int ref_cnt;
+} FrtAnalyzer;
+
+extern void frt_a_deref(FrtAnalyzer *a);
+
+#define frt_a_get_ts(ma, field, text) ma->get_ts(ma, field, text)
+
+extern FrtAnalyzer *frt_analyzer_new(FrtTokenStream *ts,
+                              void (*destroy)(FrtAnalyzer *a),
+                              FrtTokenStream *(*get_ts)(FrtAnalyzer *a,
+                                                     FrtSymbol field,
+                                                     char *text));
+extern FrtAnalyzer *frt_non_analyzer_new();
+
+extern void frt_a_standard_destroy(FrtAnalyzer *a);
+
+extern FrtAnalyzer *frt_whitespace_analyzer_new(bool lowercase);
+extern FrtAnalyzer *frt_mb_whitespace_analyzer_new(bool lowercase);
+
+extern FrtAnalyzer *frt_letter_analyzer_new(bool lowercase);
+extern FrtAnalyzer *frt_mb_letter_analyzer_new(bool lowercase);
+
+extern FrtAnalyzer *frt_standard_analyzer_new(bool lowercase);
+extern FrtAnalyzer *frt_mb_standard_analyzer_new(bool lowercase);
+extern FrtAnalyzer *frt_utf8_standard_analyzer_new(bool lowercase);
+
+extern FrtAnalyzer *frt_standard_analyzer_new_with_words(
+    const char **words, bool lowercase);
+extern FrtAnalyzer *frt_standard_analyzer_new_with_words_len(
+    const char **words, int len, bool lowercase);
+extern FrtAnalyzer *frt_mb_standard_analyzer_new_with_words(
+    const char **words, bool lowercase);
+extern FrtAnalyzer *frt_utf8_standard_analyzer_new_with_words(
+    const char **words, bool lowercase);
+
+extern FrtAnalyzer *frt_legacy_standard_analyzer_new(bool lowercase);
+extern FrtAnalyzer *frt_mb_legacy_standard_analyzer_new(bool lowercase);
+
+extern FrtAnalyzer *frt_legacy_standard_analyzer_new_with_words(
+    const char **words, bool lowercase);
+extern FrtAnalyzer *frt_mb_legacy_standard_analyzer_new_with_words(
+    const char **words, bool lowercase);
+
+#define PFA(analyzer) ((FrtPerFieldAnalyzer *)(analyzer))
+typedef struct FrtPerFieldAnalyzer
+{
+    FrtAnalyzer    super;
+    FrtHash  *dict;
+    FrtAnalyzer   *default_a;
+} FrtPerFieldAnalyzer;
+
+extern FrtAnalyzer *frt_per_field_analyzer_new(FrtAnalyzer *a);
+extern void frt_pfa_add_field(FrtAnalyzer *self,
+                              FrtSymbol field,
+                              FrtAnalyzer *analyzer);
+
+#endif

data/ext/isomorfeus_ferret_ext/frt_array.c

@@ -0,0 +1,124 @@
+#include "frt_array.h"
+#include <string.h>
+
+#define META_CNT FRT_ARY_META_CNT
+#define DATA_SZ sizeof(int) * META_CNT
+
+void **frt_ary_new_i(int type_size, int init_capa)
+{
+    void **ary;
+    if (init_capa <= 0) {
+        init_capa = FRT_ARY_INIT_CAPA;
+    }
+    ary = (void **)&(((int *)frt_ecalloc(DATA_SZ +
+                                        init_capa * type_size))[META_CNT]);
+    frt_ary_type_size(ary) = type_size;
+    frt_ary_capa(ary) = init_capa;
+    return ary;
+}
+
+void frt_ary_resize_i(void ***ary, int size)
+{
+    size++;
+    if (size > frt_ary_sz(*ary)) {
+        int capa = frt_ary_capa(*ary);
+        if (size >= capa) {
+            int *frt_ary_start = &((int *)*ary)[-META_CNT];
+            while (size >= capa) {
+                capa <<= 1;
+            }
+
+            frt_ary_start = (int *)frt_erealloc(frt_ary_start,
+                                        DATA_SZ + capa * frt_ary_type_size(*ary));
+            *ary = (void **)&(frt_ary_start[META_CNT]);
+            memset(((char *)*ary) + frt_ary_type_size(*ary) * frt_ary_sz(*ary), 0,
+                   (capa - frt_ary_sz(*ary)) * frt_ary_type_size(*ary));
+            frt_ary_capa(*ary) = capa;
+        }
+        frt_ary_sz(*ary) = size;
+    }
+}
+
+void frt_ary_set_i(void ***ary, int index, void *value)
+{
+    if (index < 0) {
+        index += frt_ary_sz(*ary);
+        if (index < 0) {
+            FRT_RAISE(FRT_INDEX_ERROR, "index %d out array", index);
+        }
+    }
+    frt_ary_resize_i(ary, index);
+    (*ary)[index] = value;
+}
+
+void *frt_ary_get_i(void **ary, int index)
+{
+    if (index < 0) {
+        index += frt_ary_sz(ary);
+    }
+    if (index >= 0 && index < frt_ary_sz(ary)) {
+        return ary[index];
+    }
+    else {
+        return NULL;
+    }
+}
+
+void frt_ary_push_i(void ***ary, void *value)
+{
+    int size = frt_ary_sz(*ary);
+    frt_ary_resize_i(ary, size);
+    (*ary)[size] = value;
+}
+
+void *frt_ary_pop_i(void **ary)
+{
+    void *val = ary[--frt_ary_sz(ary)];
+    ary[frt_ary_sz(ary)] = NULL;
+    return val;
+}
+
+void frt_ary_unshift_i(void ***ary, void *value)
+{
+    int size = frt_ary_sz(*ary);
+    frt_ary_resize_i(ary, size);
+    memmove(*ary + 1, *ary, size * sizeof(void *));
+    (*ary)[0] = value;
+}
+
+void *frt_ary_shift_i(void **ary)
+{
+    void *val = ary[0];
+    int size = --frt_ary_sz(ary);
+    memmove(ary, ary + 1, size * sizeof(void *));
+    ary[size] = NULL;
+    return val;
+}
+
+void *frt_ary_remove_i(void **ary, int index)
+{
+    if (index >= 0 && index < frt_ary_sz(ary)) {
+        void *val = ary[index];
+        memmove(ary + index, ary + index + 1,
+                (frt_ary_sz(ary) - index + 1) * sizeof(void *));
+        frt_ary_sz(ary)--;
+        return val;
+    }
+    else {
+        return NULL;
+    }
+}
+
+void frt_ary_delete_i(void **ary, int index, void (*free_elem)(void *p))
+{
+    free_elem(frt_ary_remove(ary, index));
+}
+
+void frt_ary_destroy_i(void **ary, void (*free_elem)(void *p))
+{
+    int i;
+    for (i = frt_ary_sz(ary) - 1; i >= 0; i--) {
+        free_elem(ary[i]);
+    }
+    frt_ary_free(ary);
+}

data/ext/isomorfeus_ferret_ext/frt_array.h

@@ -0,0 +1,54 @@
+#ifndef FRT_ARRAY_H
+#define FRT_ARRAY_H
+
+#include "frt_global.h"
+
+#if defined POSH_OS_SOLARIS || defined POSH_OS_SUNOS
+# define FRT_ARY_META_CNT 4
+#else
+# define FRT_ARY_META_CNT 3
+#endif
+
+#define FRT_ARY_INIT_CAPA 8
+#define frt_ary_size(ary)      frt_ary_sz(ary)
+#define frt_ary_sz(ary)        (((int *)ary)[-1])
+#define frt_ary_capa(ary)      (((int *)ary)[-2])
+#define frt_ary_type_size(ary) (((int *)ary)[-3])
+#define frt_ary_start(ary)     ((void **)&(((int *)ary)[-FRT_ARY_META_CNT]))
+#define frt_ary_free(ary)      free(frt_ary_start(ary))
+
+#define frt_ary_new_type_capa(type, init_capa)\
+                                (type *)frt_ary_new_i(sizeof(type), init_capa)
+#define frt_ary_new_type(type)      (type *)frt_ary_new_i(sizeof(type), 0)
+#define frt_ary_new_capa(init_capa) frt_ary_new_i(sizeof(void *), init_capa)
+#define frt_ary_new()               frt_ary_new_i(sizeof(void *), 0)
+#define frt_ary_resize(ary, size)   frt_ary_resize_i(((void ***)(void *)&ary), size)
+#define frt_ary_set(ary, i, val)    frt_ary_set_i(((void ***)(void *)&ary), i, val)
+#define frt_ary_get(ary, i)         frt_ary_get_i(((void **)ary), i)
+#define frt_ary_push(ary, val)      frt_ary_push_i(((void ***)(void *)&ary), val)
+#define frt_ary_pop(ary)            frt_ary_pop_i(((void **)ary))
+#define frt_ary_unshift(ary, val)   frt_ary_unshift_i(((void ***)(void *)&ary), val)
+#define frt_ary_shift(ary)          frt_ary_shift_i(((void **)ary))
+#define frt_ary_remove(ary, i)      frt_ary_remove_i(((void **)ary), i)
+#define frt_ary_delete(ary, i, f)   frt_ary_delete_i(((void **)ary), i, (frt_free_ft)f)
+#define frt_ary_destroy(ary, f)     frt_ary_destroy_i(((void **)ary), (frt_free_ft)f)
+#define frt_ary_rsz(ary, size)      frt_ary_resize(ary, size)
+#define frt_ary_grow(ary)           frt_ary_resize(ary, frt_ary_sz(ary))
+#define frt_ary_last(ary)           ary[frt_ary_sz(ary) - 1]
+#define frt_ary_sort(ary, cmp)      qsort(ary, frt_ary_size(ary), frt_ary_type_size(ary), cmp)
+#define frt_ary_each_rev(ary, i)    for (i = frt_ary_size(ary) - 1; i >= 0; i--)
+#define frt_ary_each(ary, i)        for (i = 0; i < frt_ary_size(ary); i++)
+
+extern void   frt_ary_resize_i(void ***ary, int size);
+extern void **frt_ary_new_i(int type_size, int init_capa);
+extern void   frt_ary_set_i(void ***ary, int index, void *value);
+extern void  *frt_ary_get_i(void **ary, int index);
+extern void   frt_ary_push_i(void ***ary, void *value);
+extern void  *frt_ary_pop_i(void **ary);
+extern void   frt_ary_unshift_i(void ***ary, void *value);
+extern void  *frt_ary_shift_i(void **ary);
+extern void  *frt_ary_remove_i(void **ary, int index);
+extern void   frt_ary_delete_i(void **ary, int index, frt_free_ft p);
+extern void   frt_ary_destroy_i(void **ary, frt_free_ft p);
+
+#endif

data/ext/isomorfeus_ferret_ext/frt_bitvector.c

@@ -0,0 +1,95 @@
+#include "frt_bitvector.h"
+#include <string.h>
+
+FrtBitVector *frt_bv_new_capa(int capa)
+{
+    FrtBitVector *bv = FRT_ALLOC_AND_ZERO(FrtBitVector);
+
+    /* The capacity passed by the user is number of bits allowed, however we
+     * store capacity as the number of words (U32) allocated. */
+    bv->capa = FRT_MAX(FRT_TO_WORD(capa), 4);
+    bv->bits = FRT_ALLOC_AND_ZERO_N(frt_u32, bv->capa);
+    bv->curr_bit = -1;
+    bv->ref_cnt = 1;
+    return bv;
+}
+
+FrtBitVector *frt_bv_new()
+{
+    return frt_bv_new_capa(FRT_BV_INIT_CAPA);
+}
+
+void frt_bv_destroy(FrtBitVector *bv)
+{
+    if (--(bv->ref_cnt) == 0) {
+        free(bv->bits);
+        free(bv);
+    }
+}
+
+void frt_bv_clear(FrtBitVector *bv)
+{
+    memset(bv->bits, 0, bv->capa * sizeof(frt_u32));
+    bv->extends_as_ones = 0;
+    bv->count = 0;
+    bv->size = 0;
+}
+
+void frt_bv_scan_reset(FrtBitVector *bv)
+{
+    bv->curr_bit = -1;
+}
+
+int frt_bv_eq(FrtBitVector *bv1, FrtBitVector *bv2)
+{
+    frt_u32 *bits, *bits2;
+    int min_size, word_size, ext_word_size = 0, i;
+    if (bv1 == bv2) {
+        return true;
+    }
+
+    if (bv1->extends_as_ones != bv2->extends_as_ones) {
+        return false;
+    }
+
+    bits = bv1->bits;
+    bits2 = bv2->bits;
+    min_size = FRT_MIN(bv1->size, bv2->size);
+    word_size = FRT_TO_WORD(min_size);
+
+    for (i = 0; i < word_size; i++) {
+        if (bits[i] != bits2[i]) {
+            return false;
+        }
+    }
+    if (bv1->size > min_size) {
+        bits = bv1->bits;
+        ext_word_size = FRT_TO_WORD(bv1->size);
+    }
+    else if (bv2->size > min_size) {
+        bits = bv2->bits;
+        ext_word_size = FRT_TO_WORD(bv2->size);
+    }
+    if (ext_word_size) {
+        const frt_u32 expected = (bv1->extends_as_ones ? 0xFFFFFFFF : 0);
+        for (i = word_size; i < ext_word_size; i++) {
+            if (bits[i] != expected) {
+                return false;
+            }
+        }
+    }
+    return true;
+}
+
+unsigned long long frt_bv_hash(FrtBitVector *bv)
+{
+    unsigned long long hash = 0;
+    const frt_u32 empty_word = bv->extends_as_ones ? 0xFFFFFFFF : 0;
+    int i;
+    for (i = FRT_TO_WORD(bv->size) - 1; i >= 0; i--) {
+        const frt_u32 word = bv->bits[i];
+        if (word != empty_word)
+            hash = (hash << 1) ^ word;
+    }
+    return (hash << 1) | bv->extends_as_ones;
+}