ferret 0.11.6 → 0.11.8.4

data/ext/analysis.h

@@ -1,219 +1,266 @@
 #ifndef FRT_ANALYSIS_H
 #define FRT_ANALYSIS_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #include "global.h"
 #include "hash.h"
+#include "symbol.h"
 #include "multimapper.h"
 #include <wchar.h>
 
 /****************************************************************************
  *
- * Token
+ * FrtToken
  *
  ****************************************************************************/
 
-typedef struct Token
+typedef struct FrtToken
 {
-    char text[MAX_WORD_SIZE];
+    char text[FRT_MAX_WORD_SIZE];
     int len;
     off_t start;
     off_t end;
     int pos_inc;
-} Token;
+} FrtToken;
 
-extern Token *tk_new();
-extern void tk_destroy(void *p);
-extern Token *tk_set(Token *tk, char *text, int tlen, off_t start, off_t end,
+extern FrtToken *frt_tk_new();
+extern void frt_tk_destroy(void *p);
+extern FrtToken *frt_tk_set(FrtToken *tk, char *text, int tlen, off_t start, off_t end,
                      int pos_inc);
-extern Token *tk_set_no_len(Token *tk, char *text, off_t start, off_t end,
+extern FrtToken *frt_tk_set_no_len(FrtToken *tk, char *text, off_t start, off_t end,
                             int pos_inc);
-extern int tk_eq(Token *tk1, Token *tk2);
-extern int tk_cmp(Token *tk1, Token *tk2);
+extern int frt_tk_eq(FrtToken *tk1, FrtToken *tk2);
+extern int frt_tk_cmp(FrtToken *tk1, FrtToken *tk2);
 
 /****************************************************************************
  *
- * TokenStream
+ * FrtTokenStream
  *
  ****************************************************************************/
 
 
-typedef struct TokenStream TokenStream;
-struct TokenStream
+typedef struct FrtTokenStream FrtTokenStream;
+struct FrtTokenStream
 {
-    char        *t;             /* ptr used to scan text */
-    char        *text;
-    Token       *(*next)(TokenStream *ts);
-    TokenStream *(*reset)(TokenStream *ts, char *text);
-    TokenStream *(*clone_i)(TokenStream *ts);
-    void         (*destroy_i)(TokenStream *ts);
-    int          ref_cnt;
+    char            *t;             /* ptr used to scan text */
+    char            *text;
+    FrtToken        *(*next)(FrtTokenStream *ts);
+    FrtTokenStream  *(*reset)(FrtTokenStream *ts, char *text);
+    FrtTokenStream  *(*clone_i)(FrtTokenStream *ts);
+    void            (*destroy_i)(FrtTokenStream *ts);
+    int             ref_cnt;
 };
 
-#define ts_new(type) ts_new_i(sizeof(type))
-extern TokenStream *ts_new_i(size_t size);
-extern TokenStream *ts_clone_size(TokenStream *orig_ts, size_t size);
+#define frt_ts_new(type) frt_ts_new_i(sizeof(type))
+extern FrtTokenStream *frt_ts_new_i(size_t size);
+extern FrtTokenStream *frt_ts_clone_size(FrtTokenStream *orig_ts, size_t size);
+
+typedef struct FrtCachedTokenStream
+{
+    FrtTokenStream super;
+    FrtToken       token;
+} FrtCachedTokenStream;
+
+typedef struct FrtMultiByteTokenStream
+{
+    FrtCachedTokenStream super;
+    mbstate_t            state;
+} FrtMultiByteTokenStream;
 
-typedef struct CachedTokenStream
+typedef enum 
 {
-    TokenStream super;
-    Token       token;
-} CachedTokenStream;
+    FRT_STT_ASCII,
+    FRT_STT_MB,
+    FRT_STT_UTF8
+} FrtStandardTokenizerType;
 
-typedef struct MultiByteTokenStream
+typedef struct FrtStandardTokenizer
 {
-    CachedTokenStream super;
-    mbstate_t         state;
-} MultiByteTokenStream;
+    FrtCachedTokenStream     super;
+    FrtStandardTokenizerType type;
+} FrtStandardTokenizer;
 
-typedef struct StandardTokenizer
+typedef struct FrtLegacyStandardTokenizer
 {
-    CachedTokenStream super;
-    bool        (*advance_to_start)(TokenStream *ts);
+    FrtCachedTokenStream super;
+    bool        (*advance_to_start)(FrtTokenStream *ts);
     bool        (*is_tok_char)(char *c);
-    int         (*get_alpha)(TokenStream *ts, char *token);
+    int         (*get_alpha)(FrtTokenStream *ts, char *token);
     int         (*get_apostrophe)(char *input);
-} StandardTokenizer;
+} FrtLegacyStandardTokenizer;
 
-typedef struct TokenFilter
+typedef struct FrtTokenFilter
 {
-    TokenStream super;
-    TokenStream *sub_ts;
-} TokenFilter;
+    FrtTokenStream super;
+    FrtTokenStream *sub_ts;
+} FrtTokenFilter;
 
-extern TokenStream *filter_clone_size(TokenStream *ts, size_t size);
-#define tf_new(type, sub) tf_new_i(sizeof(type), sub)
-extern TokenStream *tf_new_i(size_t size, TokenStream *sub_ts);
+extern FrtTokenStream *frt_filter_clone_size(FrtTokenStream *ts, size_t size);
+#define tf_new(type, sub) frt_tf_new_i(sizeof(type), sub)
+extern FrtTokenStream *frt_tf_new_i(size_t size, FrtTokenStream *sub_ts);
 
-typedef struct StopFilter 
+typedef struct FrtStopFilter
 {
-    TokenFilter super;
-    HashTable  *words;
-} StopFilter;
+    FrtTokenFilter super;
+    FrtHash  *words;
+} FrtStopFilter;
 
-typedef struct MappingFilter 
+typedef struct FrtMappingFilter
 {
-    TokenFilter  super;
-    MultiMapper *mapper;
-} MappingFilter;
+    FrtTokenFilter  super;
+    FrtMultiMapper *mapper;
+} FrtMappingFilter;
 
-typedef struct HyphenFilter 
+typedef struct FrtHyphenFilter
 {
-    TokenFilter super;
-    char text[MAX_WORD_SIZE];
+    FrtTokenFilter super;
+    char text[FRT_MAX_WORD_SIZE];
     int start;
     int pos;
     int len;
-    Token *tk;
-} HyphenFilter;
+    FrtToken *tk;
+} FrtHyphenFilter;
 
-typedef struct StemFilter
+typedef struct FrtStemFilter
 {
-    TokenFilter        super;
+    FrtTokenFilter        super;
     struct sb_stemmer  *stemmer;
     char               *algorithm;
     char               *charenc;
-} StemFilter;
-
-#define ts_next(mts) mts->next(mts)
-#define ts_clone(mts) mts->clone_i(mts)
-
-extern void ts_deref(TokenStream *ts);
-
-extern TokenStream *non_tokenizer_new();
-
-extern TokenStream *whitespace_tokenizer_new();
-extern TokenStream *mb_whitespace_tokenizer_new(bool lowercase);
-
-extern TokenStream *letter_tokenizer_new();
-extern TokenStream *mb_letter_tokenizer_new(bool lowercase);
-
-extern TokenStream *standard_tokenizer_new();
-extern TokenStream *mb_standard_tokenizer_new();
-
-extern TokenStream *hyphen_filter_new(TokenStream *ts);
-extern TokenStream *lowercase_filter_new(TokenStream *ts);
-extern TokenStream *mb_lowercase_filter_new(TokenStream *ts);
-
-extern const char *ENGLISH_STOP_WORDS[];
-extern const char *FULL_ENGLISH_STOP_WORDS[];
-extern const char *EXTENDED_ENGLISH_STOP_WORDS[];
-extern const char *FULL_FRENCH_STOP_WORDS[];
-extern const char *FULL_SPANISH_STOP_WORDS[];
-extern const char *FULL_PORTUGUESE_STOP_WORDS[];
-extern const char *FULL_ITALIAN_STOP_WORDS[];
-extern const char *FULL_GERMAN_STOP_WORDS[];
-extern const char *FULL_DUTCH_STOP_WORDS[];
-extern const char *FULL_SWEDISH_STOP_WORDS[];
-extern const char *FULL_NORWEGIAN_STOP_WORDS[];
-extern const char *FULL_DANISH_STOP_WORDS[];
-extern const char *FULL_RUSSIAN_STOP_WORDS[];
-extern const char *FULL_FINNISH_STOP_WORDS[];
-
-extern TokenStream *stop_filter_new_with_words_len(TokenStream *ts,
+} FrtStemFilter;
+
+#define frt_ts_next(mts) mts->next(mts)
+#define frt_ts_clone(mts) mts->clone_i(mts)
+
+extern void frt_ts_deref(FrtTokenStream *ts);
+
+extern FrtTokenStream *frt_non_tokenizer_new();
+
+extern FrtTokenStream *frt_whitespace_tokenizer_new();
+extern FrtTokenStream *frt_mb_whitespace_tokenizer_new(bool lowercase);
+
+extern FrtTokenStream *frt_letter_tokenizer_new();
+extern FrtTokenStream *frt_mb_letter_tokenizer_new(bool lowercase);
+
+extern FrtTokenStream *frt_standard_tokenizer_new();
+extern FrtTokenStream *frt_mb_standard_tokenizer_new();
+extern FrtTokenStream *frt_utf8_standard_tokenizer_new();
+
+extern FrtTokenStream *frt_legacy_standard_tokenizer_new();
+extern FrtTokenStream *frt_mb_legacy_standard_tokenizer_new();
+
+extern FrtTokenStream *frt_hyphen_filter_new(FrtTokenStream *ts);
+extern FrtTokenStream *frt_lowercase_filter_new(FrtTokenStream *ts);
+extern FrtTokenStream *frt_mb_lowercase_filter_new(FrtTokenStream *ts);
+
+extern const char *FRT_ENGLISH_STOP_WORDS[];
+extern const char *FRT_FULL_ENGLISH_STOP_WORDS[];
+extern const char *FRT_EXTENDED_ENGLISH_STOP_WORDS[];
+extern const char *FRT_FULL_FRENCH_STOP_WORDS[];
+extern const char *FRT_FULL_SPANISH_STOP_WORDS[];
+extern const char *FRT_FULL_PORTUGUESE_STOP_WORDS[];
+extern const char *FRT_FULL_ITALIAN_STOP_WORDS[];
+extern const char *FRT_FULL_GERMAN_STOP_WORDS[];
+extern const char *FRT_FULL_DUTCH_STOP_WORDS[];
+extern const char *FRT_FULL_SWEDISH_STOP_WORDS[];
+extern const char *FRT_FULL_NORWEGIAN_STOP_WORDS[];
+extern const char *FRT_FULL_DANISH_STOP_WORDS[];
+extern const char *FRT_FULL_RUSSIAN_STOP_WORDS[];
+extern const char *FRT_FULL_FINNISH_STOP_WORDS[];
+extern const char *FRT_FULL_HUNGARIAN_STOP_WORDS[];
+extern const char *FRT_FULL_RUSSIAN_STOP_WORDS_KOI8_R[];
+
+extern FrtTokenStream *frt_stop_filter_new_with_words_len(FrtTokenStream *ts,
                                                    const char **words, int len);
-extern TokenStream *stop_filter_new_with_words(TokenStream *ts,
+extern FrtTokenStream *frt_stop_filter_new_with_words(FrtTokenStream *ts,
                                                const char **words);
-extern TokenStream *stop_filter_new(TokenStream *ts);
-extern TokenStream *stem_filter_new(TokenStream *ts, const char *algorithm,
+extern FrtTokenStream *frt_stop_filter_new(FrtTokenStream *ts);
+extern FrtTokenStream *frt_stem_filter_new(FrtTokenStream *ts, const char *algorithm,
                                     const char *charenc);
 
-extern TokenStream *mapping_filter_new(TokenStream *ts);
-extern TokenStream *mapping_filter_add(TokenStream *ts, const char *pattern,
+extern FrtTokenStream *frt_mapping_filter_new(FrtTokenStream *ts);
+extern FrtTokenStream *frt_mapping_filter_add(FrtTokenStream *ts, const char *pattern,
                                        const char *replacement);
 
 /****************************************************************************
  *
- * Analyzer
+ * FrtAnalyzer
  *
  ****************************************************************************/
 
-typedef struct Analyzer
+typedef struct FrtAnalyzer
 {
-    TokenStream *current_ts;
-    TokenStream *(*get_ts)(struct Analyzer *a, char *field, char *text);
-    void (*destroy_i)(struct Analyzer *a);
+    FrtTokenStream *current_ts;
+    FrtTokenStream *(*get_ts)(struct FrtAnalyzer *a, FrtSymbol field, char *text);
+    void (*destroy_i)(struct FrtAnalyzer *a);
     int ref_cnt;
-} Analyzer;
+} FrtAnalyzer;
 
-extern void a_deref(Analyzer *a);
+extern void frt_a_deref(FrtAnalyzer *a);
 
-#define a_get_ts(ma, field, text) ma->get_ts(ma, field, text)
+#define frt_a_get_ts(ma, field, text) ma->get_ts(ma, field, text)
 
-extern Analyzer *analyzer_new(TokenStream *ts,
-                              void (*destroy)(Analyzer *a),
-                              TokenStream *(*get_ts)(Analyzer *a,
-                                                     char *field,
+extern FrtAnalyzer *frt_analyzer_new(FrtTokenStream *ts,
+                              void (*destroy)(FrtAnalyzer *a),
+                              FrtTokenStream *(*get_ts)(FrtAnalyzer *a,
+                                                     FrtSymbol field,
                                                      char *text));
-extern void a_standard_destroy(Analyzer *a);
-extern Analyzer *non_analyzer_new();
-
-extern Analyzer *whitespace_analyzer_new(bool lowercase);
-extern Analyzer *mb_whitespace_analyzer_new(bool lowercase);
-
-extern Analyzer *letter_analyzer_new(bool lowercase);
-extern Analyzer *mb_letter_analyzer_new(bool lowercase);
-
-extern Analyzer *standard_analyzer_new(bool lowercase);
-extern Analyzer *mb_standard_analyzer_new(bool lowercase);
-
-extern Analyzer *standard_analyzer_new_with_words(const char **words,
-                                                  bool lowercase);
-extern Analyzer *standard_analyzer_new_with_words_len(const char **words, int len,
-                                                      bool lowercase);
-extern Analyzer *mb_standard_analyzer_new_with_words(const char **words,
-                                                     bool lowercase);
-extern Analyzer *mb_standard_analyzer_new_with_words_len(const char **words,
-                                                  int len, bool lowercase);
-
-#define PFA(analyzer) ((PerFieldAnalyzer *)(analyzer))
-typedef struct PerFieldAnalyzer
+extern void frt_a_standard_destroy(FrtAnalyzer *a);
+extern FrtAnalyzer *frt_non_analyzer_new();
+
+extern FrtAnalyzer *frt_whitespace_analyzer_new(bool lowercase);
+extern FrtAnalyzer *frt_mb_whitespace_analyzer_new(bool lowercase);
+
+extern FrtAnalyzer *frt_letter_analyzer_new(bool lowercase);
+extern FrtAnalyzer *frt_mb_letter_analyzer_new(bool lowercase);
+
+extern FrtAnalyzer *frt_standard_analyzer_new(bool lowercase);
+extern FrtAnalyzer *frt_mb_standard_analyzer_new(bool lowercase);
+extern FrtAnalyzer *frt_utf8_standard_analyzer_new(bool lowercase);
+
+extern FrtAnalyzer *frt_standard_analyzer_new_with_words(
+    const char **words, bool lowercase);
+extern FrtAnalyzer *frt_standard_analyzer_new_with_words_len(
+    const char **words, int len, bool lowercase);
+extern FrtAnalyzer *frt_mb_standard_analyzer_new_with_words(
+    const char **words, bool lowercase);
+extern FrtAnalyzer *frt_mb_standard_analyzer_new_with_words_len(
+    const char **words, int len, bool lowercase);
+extern FrtAnalyzer *frt_utf8_standard_analyzer_new_with_words(
+    const char **words, bool lowercase);
+extern FrtAnalyzer *frt_utf8_standard_analyzer_new_with_words_len(
+    const char **words, int len, bool lowercase);
+
+extern FrtAnalyzer *frt_legacy_standard_analyzer_new(bool lowercase);
+extern FrtAnalyzer *frt_mb_legacy_standard_analyzer_new(bool lowercase);
+
+extern FrtAnalyzer *frt_legacy_standard_analyzer_new_with_words(
+    const char **words, bool lowercase);
+extern FrtAnalyzer *frt_legacy_standard_analyzer_new_with_words_len(
+    const char **words, int len, bool lowercase);
+extern FrtAnalyzer *frt_mb_legacy_standard_analyzer_new_with_words(
+    const char **words, bool lowercase);
+extern FrtAnalyzer *frt_mb_legacy_standard_analyzer_new_with_words_len(
+    const char **words, int len, bool lowercase);
+
+#define PFA(analyzer) ((FrtPerFieldAnalyzer *)(analyzer))
+typedef struct FrtPerFieldAnalyzer
 {
-    Analyzer    super;
-    HashTable  *dict;
-    Analyzer   *default_a;
-} PerFieldAnalyzer;
-
-extern Analyzer *per_field_analyzer_new(Analyzer *a);
-extern void pfa_add_field(Analyzer *self, char *field, Analyzer *analyzer);
+    FrtAnalyzer    super;
+    FrtHash  *dict;
+    FrtAnalyzer   *default_a;
+} FrtPerFieldAnalyzer;
+
+extern FrtAnalyzer *frt_per_field_analyzer_new(FrtAnalyzer *a);
+extern void frt_pfa_add_field(FrtAnalyzer *self,
+                              FrtSymbol field,
+                              FrtAnalyzer *analyzer);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
 
 #endif

data/ext/api.h

@@ -13,15 +13,14 @@ typedef unsigned char symbol;
 
 struct SN_env {
     symbol * p;
-    int c; int a; int l; int lb; int bra; int ket;
-    int S_size; int I_size; int B_size;
+    int c; int l; int lb; int bra; int ket;
     symbol * * S;
     int * I;
-    symbol * B;
+    unsigned char * B;
 };
 
 extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size);
-extern void SN_close_env(struct SN_env * z);
+extern void SN_close_env(struct SN_env * z, int S_size);
 
 extern int SN_set_current(struct SN_env * z, int size, const symbol * s);
 

data/ext/array.c

@@ -1,5 +1,6 @@
 #include "array.h"
 #include <string.h>
+#include "internal.h"
 
 #define META_CNT ARY_META_CNT
 #define DATA_SZ sizeof(int) * META_CNT
@@ -10,7 +11,8 @@ void **ary_new_i(int type_size, int init_capa)
     if (init_capa <= 0) {
         init_capa = ARY_INIT_CAPA;
     }
-    ary = (void **)&(((int *)ecalloc(DATA_SZ + init_capa * type_size))[META_CNT]);
+    ary = (void **)&(((int *)ecalloc(DATA_SZ +
+                                        init_capa * type_size))[META_CNT]);
     ary_type_size(ary) = type_size;
     ary_capa(ary) = init_capa;
     return ary;
@@ -19,7 +21,7 @@ void **ary_new_i(int type_size, int init_capa)
 INLINE void ary_resize_i(void ***ary, int size)
 {
     size++;
-    if (size >= ary_sz(*ary)) {
+    if (size > ary_sz(*ary)) {
         int capa = ary_capa(*ary);
         if (size >= capa) {
             int *ary_start = &((int *)*ary)[-META_CNT];
@@ -81,7 +83,7 @@ void ary_unshift_i(void ***ary, void *value)
 {
     int size = ary_sz(*ary);
     ary_resize_i(ary, size);
-    memmove(*ary, *ary + 1, size * sizeof(void *));
+    memmove(*ary + 1, *ary, size * sizeof(void *));
     (*ary)[0] = value;
 }