isomorfeus-ferret 0.12.5 → 0.13.0

data/ext/isomorfeus_ferret_ext/frt_analysis.h

@@ -1,130 +1,63 @@
 #ifndef FRT_ANALYSIS_H
 #define FRT_ANALYSIS_H
 
-#include <wchar.h>
 #include "frt_global.h"
 #include "frt_hash.h"
 #include "frt_multimapper.h"
+#include <ruby/encoding.h>
 
-/****************************************************************************
- *
- * FrtToken
- *
- ****************************************************************************/
+/*****************************************************************************/
+/*** FrtToken ****************************************************************/
+/*****************************************************************************/
 
-typedef struct FrtToken
-{
-    char text[FRT_MAX_WORD_SIZE];
-    int len;
+typedef struct FrtToken {
+    char  text[FRT_MAX_WORD_SIZE];
+    int   len;
     off_t start;
     off_t end;
-    int pos_inc;
+    int   pos_inc;
 } FrtToken;
 
 extern FrtToken *frt_tk_new();
-extern void frt_tk_destroy(void *p);
-extern FrtToken *frt_tk_set(FrtToken *tk, char *text, int tlen, off_t start, off_t end, int pos_inc);
-extern FrtToken *frt_tk_set_no_len(FrtToken *tk, char *text, off_t start, off_t end, int pos_inc);
-extern int frt_tk_eq(FrtToken *tk1, FrtToken *tk2);
-extern int frt_tk_cmp(FrtToken *tk1, FrtToken *tk2);
-
-/****************************************************************************
- *
- * FrtTokenStream
- *
- ****************************************************************************/
+extern void      frt_tk_destroy(void *p);
+extern FrtToken *frt_tk_set(FrtToken *tk, char *text, int tlen, off_t start, off_t end, int pos_inc, rb_encoding *encoding);
+extern FrtToken *frt_tk_set_no_len(FrtToken *tk, char *text, off_t start, off_t end, int pos_inc, rb_encoding *encoding);
+extern int       frt_tk_eq(FrtToken *tk1, FrtToken *tk2);
+extern int       frt_tk_cmp(FrtToken *tk1, FrtToken *tk2);
 
+/*****************************************************************************/
+/*** FrtTokenStream **********************************************************/
+/*****************************************************************************/
 
 typedef struct FrtTokenStream FrtTokenStream;
-struct FrtTokenStream
-{
-    char            *t;             /* ptr used to scan text */
-    char            *text;
-    FrtToken        *(*next)(FrtTokenStream *ts);
-    FrtTokenStream  *(*reset)(FrtTokenStream *ts, char *text);
-    FrtTokenStream  *(*clone_i)(FrtTokenStream *ts);
-    void            (*destroy_i)(FrtTokenStream *ts);
+struct FrtTokenStream {
+    char           *t;             /* ptr used to scan text */
+    char           *text;
+    int             length;
+    rb_encoding    *encoding;
+    FrtToken       *(*next)(FrtTokenStream *ts);
+    FrtTokenStream *(*reset)(FrtTokenStream *ts, char *text, rb_encoding *encoding);
+    FrtTokenStream *(*clone_i)(FrtTokenStream *ts);
+    void           (*destroy_i)(FrtTokenStream *ts);
     int             ref_cnt;
+    VALUE           rts;
+    FrtToken        token;
 };
 
-#define frt_ts_new(type) frt_ts_new_i(sizeof(type))
 extern FrtTokenStream *frt_ts_new_i(size_t size);
+extern FrtTokenStream *frt_ts_init(FrtTokenStream *ts);
+extern FrtTokenStream *frt_ts_reset(FrtTokenStream *ts, char *text, rb_encoding *encoding);
 extern FrtTokenStream *frt_ts_clone_size(FrtTokenStream *orig_ts, size_t size);
 
-typedef struct FrtCachedTokenStream
-{
-    FrtTokenStream super;
-    FrtToken       token;
-} FrtCachedTokenStream;
-
-typedef struct FrtMultiByteTokenStream
-{
-    FrtCachedTokenStream super;
-    mbstate_t            state;
-} FrtMultiByteTokenStream;
-
-typedef enum
-{
-    FRT_STT_ASCII,
-    FRT_STT_MB,
-    FRT_STT_UTF8
-} FrtStandardTokenizerType;
-
-typedef struct FrtStandardTokenizer
-{
-    FrtCachedTokenStream     super;
-    FrtStandardTokenizerType type;
-} FrtStandardTokenizer;
-
-typedef struct FrtLegacyStandardTokenizer
-{
-    FrtCachedTokenStream super;
-    bool        (*advance_to_start)(FrtTokenStream *ts);
-    bool        (*is_tok_char)(char *c);
-    int         (*get_alpha)(FrtTokenStream *ts, char *token);
-    int         (*get_apostrophe)(char *input);
-} FrtLegacyStandardTokenizer;
-
-typedef struct FrtTokenFilter
-{
+typedef struct FrtTokenFilter {
     FrtTokenStream super;
     FrtTokenStream *sub_ts;
 } FrtTokenFilter;
 
 extern FrtTokenStream *frt_filter_clone_size(FrtTokenStream *ts, size_t size);
-#define tf_new(type, sub) frt_tf_new_i(sizeof(type), sub)
+#define frt_tf_new(type, sub) frt_tf_new_i(sizeof(type), sub)
 extern FrtTokenStream *frt_tf_new_i(size_t size, FrtTokenStream *sub_ts);
 
-typedef struct FrtStopFilter
-{
-    FrtTokenFilter super;
-    FrtHash  *words;
-} FrtStopFilter;
-
-typedef struct FrtMappingFilter
-{
-    FrtTokenFilter  super;
-    FrtMultiMapper *mapper;
-} FrtMappingFilter;
-
-typedef struct FrtHyphenFilter
-{
-    FrtTokenFilter super;
-    char text[FRT_MAX_WORD_SIZE];
-    int start;
-    int pos;
-    int len;
-    FrtToken *tk;
-} FrtHyphenFilter;
-
-typedef struct FrtStemFilter
-{
-    FrtTokenFilter        super;
-    struct sb_stemmer  *stemmer;
-    char               *algorithm;
-    char               *charenc;
-} FrtStemFilter;
-
 #define frt_ts_next(mts) mts->next(mts)
 #define frt_ts_clone(mts) mts->clone_i(mts)
 
@@ -132,22 +65,58 @@ extern void frt_ts_deref(FrtTokenStream *ts);
 
 extern FrtTokenStream *frt_non_tokenizer_new();
 
-extern FrtTokenStream *frt_whitespace_tokenizer_new();
-extern FrtTokenStream *frt_mb_whitespace_tokenizer_new(bool lowercase);
+/*****************************************************************************/
+/*** FrtWhiteSpaceTokenizer **************************************************/
+/*****************************************************************************/
+
+extern FrtTokenStream *frt_whitespace_tokenizer_alloc(void);
+extern FrtTokenStream *frt_whitespace_tokenizer_init(FrtTokenStream *ts);
+extern FrtTokenStream *frt_whitespace_tokenizer_new(void);
+
+/*****************************************************************************/
+/*** FrtLetterTokenizer ******************************************************/
+/*****************************************************************************/
+
+extern FrtTokenStream *frt_letter_tokenizer_alloc(void);
+extern FrtTokenStream *frt_letter_tokenizer_init(FrtTokenStream *ts);
+extern FrtTokenStream *frt_letter_tokenizer_new(void);
+
+/*****************************************************************************/
+/*** FrtStandardTokenizer ****************************************************/
+/*****************************************************************************/
 
-extern FrtTokenStream *frt_letter_tokenizer_new();
-extern FrtTokenStream *frt_mb_letter_tokenizer_new(bool lowercase);
+extern FrtTokenStream *frt_standard_tokenizer_alloc(void);
+extern FrtTokenStream *frt_standard_tokenizer_init(FrtTokenStream *ts);
+extern FrtTokenStream *frt_standard_tokenizer_new(void);
 
-extern FrtTokenStream *frt_standard_tokenizer_new();
-extern FrtTokenStream *frt_mb_standard_tokenizer_new();
-extern FrtTokenStream *frt_utf8_standard_tokenizer_new();
+/*****************************************************************************/
+/*** FrtHyphenFilter *********************************************************/
+/*****************************************************************************/
 
-extern FrtTokenStream *frt_legacy_standard_tokenizer_new();
-extern FrtTokenStream *frt_mb_legacy_standard_tokenizer_new();
+typedef struct FrtHyphenFilter {
+    FrtTokenFilter  super;
+    char            text[FRT_MAX_WORD_SIZE];
+    int             start;
+    int             pos;
+    int             len;
+    FrtToken       *tk;
+} FrtHyphenFilter;
+
+extern FrtTokenStream *frt_hyphen_filter_alloc(void);
+extern FrtTokenStream *frt_hyphen_filter_init(FrtTokenStream *ts, FrtTokenStream *sub_ts);
+extern FrtTokenStream *frt_hyphen_filter_new(FrtTokenStream *sub_ts);
 
-extern FrtTokenStream *frt_hyphen_filter_new(FrtTokenStream *ts);
-extern FrtTokenStream *frt_lowercase_filter_new(FrtTokenStream *ts);
-extern FrtTokenStream *frt_mb_lowercase_filter_new(FrtTokenStream *ts);
+/*****************************************************************************/
+/*** FrtLowercaseFilter ******************************************************/
+/*****************************************************************************/
+
+extern FrtTokenStream *frt_lowercase_filter_alloc(void);
+extern void            frt_lowercase_filter_init(FrtTokenStream *ts, FrtTokenStream *sub_ts);
+extern FrtTokenStream *frt_lowercase_filter_new(FrtTokenStream *sub_ts);
+
+/*****************************************************************************/
+/*** FrtStopFilter ***********************************************************/
+/*****************************************************************************/
 
 extern const char *FRT_ENGLISH_STOP_WORDS[];
 extern const char *FRT_FULL_ENGLISH_STOP_WORDS[];
@@ -165,83 +134,118 @@ extern const char *FRT_FULL_RUSSIAN_STOP_WORDS[];
 extern const char *FRT_FULL_FINNISH_STOP_WORDS[];
 extern const char *FRT_FULL_HUNGARIAN_STOP_WORDS[];
 
-extern FrtTokenStream *frt_stop_filter_new_with_words_len(FrtTokenStream *ts,
-                                                   const char **words, int len);
-extern FrtTokenStream *frt_stop_filter_new_with_words(FrtTokenStream *ts,
-                                               const char **words);
-extern FrtTokenStream *frt_stop_filter_new(FrtTokenStream *ts);
-extern FrtTokenStream *frt_stem_filter_new(FrtTokenStream *ts, const char *algorithm,
-                                    const char *charenc);
-
-extern FrtTokenStream *frt_mapping_filter_new(FrtTokenStream *ts);
-extern FrtTokenStream *frt_mapping_filter_add(FrtTokenStream *ts, const char *pattern,
-                                       const char *replacement);
-
-/****************************************************************************
- *
- * FrtAnalyzer
- *
- ****************************************************************************/
-
-typedef struct FrtAnalyzer
-{
+typedef struct FrtStopFilter {
+    FrtTokenFilter  super;
+    FrtHash        *words;
+} FrtStopFilter;
+
+extern FrtTokenStream *frt_stop_filter_alloc(void);
+extern FrtTokenStream *frt_stop_filter_init(FrtTokenStream *ts, FrtTokenStream *sub_ts);
+extern void            frt_stop_filter_set_words(FrtTokenStream *ts, const char **words);
+extern void            frt_stop_filter_set_words_len(FrtTokenStream *ts, const char **words, int len);
+extern FrtTokenStream *frt_stop_filter_new(FrtTokenStream *sub_ts);
+extern FrtTokenStream *frt_stop_filter_new_with_words(FrtTokenStream *sub_ts, const char **words);
+extern FrtTokenStream *frt_stop_filter_new_with_words_len(FrtTokenStream *sub_ts, const char **words, int len);
+
+/*****************************************************************************/
+/*** FrtStemFilter ***********************************************************/
+/*****************************************************************************/
+
+typedef struct FrtStemFilter {
+    FrtTokenFilter      super;
+    struct sb_stemmer  *stemmer;
+    char               *algorithm;
+    char               *charenc;
+} FrtStemFilter;
+
+extern FrtTokenStream *frt_stem_filter_alloc(void);
+extern void            frt_stem_filter_init(FrtTokenStream *ts, FrtTokenStream *sub_ts, const char *algorithm);
+extern FrtTokenStream *frt_stem_filter_new(FrtTokenStream *sub_ts, const char *algorithm);
+
+/*****************************************************************************/
+/*** FrtMappingFilter ********************************************************/
+/*****************************************************************************/
+
+typedef struct FrtMappingFilter {
+    FrtTokenFilter  super;
+    FrtMultiMapper *mapper;
+} FrtMappingFilter;
+
+extern FrtTokenStream *frt_mapping_filter_alloc(void);
+extern void            frt_mapping_filter_init(FrtTokenStream *ts, FrtTokenStream *sub_ts);
+extern FrtTokenStream *frt_mapping_filter_new(FrtTokenStream *sub_ts);
+extern FrtTokenStream *frt_mapping_filter_add(FrtTokenStream *ts, const char *pattern, const char *replacement);
+
+/*****************************************************************************/
+/*** FrtAnalyzer *************************************************************/
+/*****************************************************************************/
+
+typedef struct FrtAnalyzer {
     FrtTokenStream *current_ts;
-    FrtTokenStream *(*get_ts)(struct FrtAnalyzer *a, FrtSymbol field, char *text);
-    void (*destroy_i)(struct FrtAnalyzer *a);
-    int ref_cnt;
+    FrtTokenStream *(*get_ts)(struct FrtAnalyzer *a, ID field, char *text, rb_encoding *encoding);
+    void           (*destroy_i)(struct FrtAnalyzer *a);
+    int            ref_cnt;
+    VALUE          ranalyzer;
 } FrtAnalyzer;
 
 extern void frt_a_deref(FrtAnalyzer *a);
 
-#define frt_a_get_ts(ma, field, text) ma->get_ts(ma, field, text)
+#define frt_a_get_ts(ma, field, text, encoding) ma->get_ts(ma, field, text, encoding)
+
+extern FrtAnalyzer *frt_analyzer_alloc(void);
+extern void         frt_analyzer_init(FrtAnalyzer *a, FrtTokenStream *ts, void (*destroy)(FrtAnalyzer *a),
+                                    FrtTokenStream *(*get_ts)(FrtAnalyzer *a, ID field, char *text, rb_encoding *encoding));
+extern FrtAnalyzer *frt_analyzer_new(FrtTokenStream *ts, void (*destroy)(FrtAnalyzer *a),
+                                    FrtTokenStream *(*get_ts)(FrtAnalyzer *a, ID field, char *text, rb_encoding *encoding));
 
-extern FrtAnalyzer *frt_analyzer_new(FrtTokenStream *ts,
-                              void (*destroy)(FrtAnalyzer *a),
-                              FrtTokenStream *(*get_ts)(FrtAnalyzer *a,
-                                                     FrtSymbol field,
-                                                     char *text));
-extern FrtAnalyzer *frt_non_analyzer_new();
+/*****************************************************************************/
+/*** FrtNonAnalyzer **********************************************************/
+/*****************************************************************************/
+
+extern FrtAnalyzer *frt_non_analyzer_new(void);
 
 extern void frt_a_standard_destroy(FrtAnalyzer *a);
 
+/*****************************************************************************/
+/*** FrtWhiteSpaceAnalyzer ***************************************************/
+/*****************************************************************************/
+
+extern FrtAnalyzer *frt_whitespace_analyzer_alloc(void);
+extern void         frt_whitespace_analyzer_init(FrtAnalyzer *a, bool lowercase);
 extern FrtAnalyzer *frt_whitespace_analyzer_new(bool lowercase);
-extern FrtAnalyzer *frt_mb_whitespace_analyzer_new(bool lowercase);
 
+/*****************************************************************************/
+/*** FrtLetterAnalyzer *******************************************************/
+/*****************************************************************************/
+
+extern FrtAnalyzer *frt_letter_analyzer_alloc(void);
+extern void         frt_letter_analyzer_init(FrtAnalyzer *a, bool lowercase);
 extern FrtAnalyzer *frt_letter_analyzer_new(bool lowercase);
-extern FrtAnalyzer *frt_mb_letter_analyzer_new(bool lowercase);
 
+/*****************************************************************************/
+/*** FrtStandardAnalyzer *****************************************************/
+/*****************************************************************************/
+
+extern FrtAnalyzer *frt_standard_analyzer_alloc(void);
+extern void         frt_standard_analyzer_init(FrtAnalyzer *a, bool lowercase, const char **words);
 extern FrtAnalyzer *frt_standard_analyzer_new(bool lowercase);
-extern FrtAnalyzer *frt_mb_standard_analyzer_new(bool lowercase);
-extern FrtAnalyzer *frt_utf8_standard_analyzer_new(bool lowercase);
-
-extern FrtAnalyzer *frt_standard_analyzer_new_with_words(
-    const char **words, bool lowercase);
-extern FrtAnalyzer *frt_standard_analyzer_new_with_words_len(
-    const char **words, int len, bool lowercase);
-extern FrtAnalyzer *frt_mb_standard_analyzer_new_with_words(
-    const char **words, bool lowercase);
-extern FrtAnalyzer *frt_utf8_standard_analyzer_new_with_words(
-    const char **words, bool lowercase);
-
-extern FrtAnalyzer *frt_legacy_standard_analyzer_new(bool lowercase);
-extern FrtAnalyzer *frt_mb_legacy_standard_analyzer_new(bool lowercase);
-
-extern FrtAnalyzer *frt_legacy_standard_analyzer_new_with_words(
-    const char **words, bool lowercase);
-extern FrtAnalyzer *frt_mb_legacy_standard_analyzer_new_with_words(
-    const char **words, bool lowercase);
+extern FrtAnalyzer *frt_standard_analyzer_new_with_words(bool lowercase, const char **words);
+
+/*****************************************************************************/
+/*** FrtPerFieldAnalyzer *****************************************************/
+/*****************************************************************************/
 
 #define PFA(analyzer) ((FrtPerFieldAnalyzer *)(analyzer))
-typedef struct FrtPerFieldAnalyzer
-{
-    FrtAnalyzer    super;
-    FrtHash  *dict;
-    FrtAnalyzer   *default_a;
+
+typedef struct FrtPerFieldAnalyzer {
+    FrtAnalyzer  super;
+    FrtHash     *dict;
+    FrtAnalyzer *default_a;
 } FrtPerFieldAnalyzer;
 
-extern FrtAnalyzer *frt_per_field_analyzer_new(FrtAnalyzer *a);
-extern void frt_pfa_add_field(FrtAnalyzer *self,
-                              FrtSymbol field,
-                              FrtAnalyzer *analyzer);
+extern FrtAnalyzer *frt_per_field_analyzer_alloc(void);
+extern void         frt_per_field_analyzer_init(FrtAnalyzer *a, FrtAnalyzer *default_a);
+extern FrtAnalyzer *frt_per_field_analyzer_new(FrtAnalyzer *default_a);
+extern void         frt_pfa_add_field(FrtAnalyzer *self, ID field, FrtAnalyzer *analyzer);
 
 #endif

data/ext/isomorfeus_ferret_ext/frt_array.c

@@ -109,13 +109,11 @@ void *frt_ary_remove_i(void **ary, int index)
     }
 }
 
-void frt_ary_delete_i(void **ary, int index, void (*free_elem)(void *p))
-{
+void frt_ary_delete_i(void **ary, int index, void (*free_elem)(void *p)) {
     free_elem(frt_ary_remove(ary, index));
 }
 
-void frt_ary_destroy_i(void **ary, void (*free_elem)(void *p))
-{
+void frt_ary_destroy_i(void **ary, void (*free_elem)(void *p)) {
     int i;
     for (i = frt_ary_sz(ary) - 1; i >= 0; i--) {
         free_elem(ary[i]);

data/ext/isomorfeus_ferret_ext/frt_bitvector.c

@@ -1,8 +1,7 @@
 #include "frt_bitvector.h"
 #include <string.h>
 
-FrtBitVector *frt_bv_new_capa(int capa)
-{
+FrtBitVector *frt_bv_new_capa(int capa) {
     FrtBitVector *bv = FRT_ALLOC_AND_ZERO(FrtBitVector);
 
     /* The capacity passed by the user is number of bits allowed, however we
@@ -11,37 +10,33 @@ FrtBitVector *frt_bv_new_capa(int capa)
     bv->bits = FRT_ALLOC_AND_ZERO_N(frt_u32, bv->capa);
     bv->curr_bit = -1;
     bv->ref_cnt = 1;
+    bv->rbv = Qnil;
     return bv;
 }
 
-FrtBitVector *frt_bv_new()
-{
+FrtBitVector *frt_bv_new(void) {
     return frt_bv_new_capa(FRT_BV_INIT_CAPA);
 }
 
-void frt_bv_destroy(FrtBitVector *bv)
-{
+void frt_bv_destroy(FrtBitVector *bv) {
     if (--(bv->ref_cnt) == 0) {
         free(bv->bits);
         free(bv);
     }
 }
 
-void frt_bv_clear(FrtBitVector *bv)
-{
+void frt_bv_clear(FrtBitVector *bv) {
     memset(bv->bits, 0, bv->capa * sizeof(frt_u32));
     bv->extends_as_ones = 0;
     bv->count = 0;
     bv->size = 0;
 }
 
-void frt_bv_scan_reset(FrtBitVector *bv)
-{
+void frt_bv_scan_reset(FrtBitVector *bv) {
     bv->curr_bit = -1;
 }
 
-int frt_bv_eq(FrtBitVector *bv1, FrtBitVector *bv2)
-{
+int frt_bv_eq(FrtBitVector *bv1, FrtBitVector *bv2) {
     frt_u32 *bits, *bits2;
     int min_size, word_size, ext_word_size = 0, i;
     if (bv1 == bv2) {
@@ -65,8 +60,7 @@ int frt_bv_eq(FrtBitVector *bv1, FrtBitVector *bv2)
     if (bv1->size > min_size) {
         bits = bv1->bits;
         ext_word_size = FRT_TO_WORD(bv1->size);
-    }
-    else if (bv2->size > min_size) {
+    } else if (bv2->size > min_size) {
         bits = bv2->bits;
         ext_word_size = FRT_TO_WORD(bv2->size);
     }
@@ -81,8 +75,7 @@ int frt_bv_eq(FrtBitVector *bv1, FrtBitVector *bv2)
     return true;
 }
 
-unsigned long long frt_bv_hash(FrtBitVector *bv)
-{
+unsigned long long frt_bv_hash(FrtBitVector *bv) {
     unsigned long long hash = 0;
     const frt_u32 empty_word = bv->extends_as_ones ? 0xFFFFFFFF : 0;
     int i;