ferret 0.9.6 → 0.10.0

data/ext/analysis.h

@@ -1,8 +1,9 @@
 #ifndef FRT_ANALYSIS_H
 #define FRT_ANALYSIS_H
 
-#include <global.h>
-#include <hash.h>
+#include "global.h"
+#include "hash.h"
+#include <wchar.h>
 
 /****************************************************************************
  *
@@ -10,19 +11,23 @@
  *
  ****************************************************************************/
 
-typedef struct Token {
-  char text[MAX_WORD_SIZE];
-  int start;
-  int end;
-  int pos_inc;
+typedef struct Token
+{
+    char text[MAX_WORD_SIZE];
+    int len;
+    int start;
+    int end;
+    int pos_inc;
 } Token;
 
-Token *tk_create();
-void tk_destroy(void *p);
-Token *tk_set(Token *tk, char *text, int tlen, int start, int end, int pos_inc);
-Token *tk_set_no_len(Token *tk, char *text, int start, int end, int pos_inc);
-int tk_eq(Token *tk1, Token *tk2);
-int tk_cmp(Token *tk1, Token *tk2);
+extern Token *tk_new();
+extern void tk_destroy(void *p);
+extern Token *tk_set(Token *tk, char *text, int tlen, int start, int end,
+                     int pos_inc);
+extern Token *tk_set_no_len(Token *tk, char *text, int start, int end,
+                            int pos_inc);
+extern int tk_eq(Token *tk1, Token *tk2);
+extern int tk_cmp(Token *tk1, Token *tk2);
 
 /****************************************************************************
  *
@@ -32,34 +37,82 @@ int tk_cmp(Token *tk1, Token *tk2);
 
 
 typedef struct TokenStream TokenStream;
-struct TokenStream {
-  void *data;
-  char *text;
-  char *t; /* ptr used to scan text */
-  Token *token;
-  Token *(*next)(TokenStream *ts);
-  void (*reset)(TokenStream *ts, char *text);
-  void (*clone_i)(TokenStream *orig_ts, TokenStream *new_ts);
-  void (*destroy)(TokenStream *ts);
-  TokenStream *sub_ts; /* used by filters */
-  int ref_cnt;
+struct TokenStream
+{
+    char        *t;             /* ptr used to scan text */
+    char        *text;
+    Token       *(*next)(TokenStream *ts);
+    TokenStream *(*reset)(TokenStream *ts, char *text);
+    TokenStream *(*clone_i)(TokenStream *ts);
+    void         (*destroy_i)(TokenStream *ts);
+    int          ref_cnt;
 };
 
+#define ts_new(type) ts_new_i(sizeof(type))
+extern TokenStream *ts_new_i(size_t size);
+extern TokenStream *ts_clone_size(TokenStream *orig_ts, size_t size);
+
+typedef struct CachedTokenStream
+{
+    TokenStream super;
+    Token       token;
+} CachedTokenStream;
+
+typedef struct MultiByteTokenStream
+{
+    CachedTokenStream super;
+    mbstate_t         state;
+} MultiByteTokenStream;
+
+typedef struct StandardTokenizer
+{
+    CachedTokenStream super;
+    bool        (*advance_to_start)(TokenStream *ts);
+    bool        (*is_tok_char)(char *c);
+    int         (*get_alpha)(TokenStream *ts, char *token);
+    int         (*get_apostrophe)(char *input);
+} StandardTokenizer;
+
+typedef struct TokenFilter
+{
+    TokenStream super;
+    TokenStream *sub_ts;
+} TokenFilter;
+
+extern TokenStream *filter_clone_size(TokenStream *ts, size_t size);
+#define tf_new(type, sub) tf_new_i(sizeof(type), sub)
+extern TokenStream *tf_new_i(size_t size, TokenStream *sub_ts);
+
+typedef struct StopFilter 
+{
+    TokenFilter super;
+    HashTable  *words;
+} StopFilter;
+
+typedef struct StemFilter
+{
+    TokenFilter        super;
+    struct sb_stemmer  *stemmer;
+    char               *algorithm;
+    char               *charenc;
+} StemFilter;
+
 #define ts_next(mts) mts->next(mts)
+#define ts_clone(mts) mts->clone_i(mts)
 
-void ts_deref(void *p);
+extern void ts_deref(TokenStream *ts);
 
-TokenStream *whitespace_tokenizer_create();
-TokenStream *mb_whitespace_tokenizer_create(bool lowercase);
+extern TokenStream *whitespace_tokenizer_new();
+extern TokenStream *mb_whitespace_tokenizer_new(bool lowercase);
 
-TokenStream *letter_tokenizer_create();
-TokenStream *mb_letter_tokenizer_create(bool lowercase);
+extern TokenStream *letter_tokenizer_new();
+extern TokenStream *mb_letter_tokenizer_new(bool lowercase);
 
-TokenStream *standard_tokenizer_create();
-TokenStream *mb_standard_tokenizer_create();
+extern TokenStream *standard_tokenizer_new();
+extern TokenStream *mb_standard_tokenizer_new();
 
-TokenStream *lowercase_filter_create(TokenStream *ts);
-TokenStream *mb_lowercase_filter_create(TokenStream *ts);
+extern TokenStream *lowercase_filter_new(TokenStream *ts);
+extern TokenStream *mb_lowercase_filter_new(TokenStream *ts);
 
 extern const char *ENGLISH_STOP_WORDS[];
 extern const char *FULL_ENGLISH_STOP_WORDS[];
@@ -76,13 +129,13 @@ extern const char *FULL_DANISH_STOP_WORDS[];
 extern const char *FULL_RUSSIAN_STOP_WORDS[];
 extern const char *FULL_FINNISH_STOP_WORDS[];
 
-TokenStream *stop_filter_create_with_words_len(TokenStream *ts,
-    const char **words, int len);
-TokenStream *stop_filter_create_with_words(TokenStream *ts, const char **words);
-TokenStream *stop_filter_create(TokenStream *ts);
-TokenStream *stem_filter_create(TokenStream *ts, const char * algorithm,
-    const char * charenc);
-TokenStream *ts_clone(TokenStream *orig_ts);
+extern TokenStream *stop_filter_new_with_words_len(TokenStream *ts,
+                                                   const char **words, int len);
+extern TokenStream *stop_filter_new_with_words(TokenStream *ts,
+                                               const char **words);
+extern TokenStream *stop_filter_new(TokenStream *ts);
+extern TokenStream *stem_filter_new(TokenStream *ts, const char *algorithm,
+                                    const char *charenc);
 
 /****************************************************************************
  *
@@ -90,47 +143,51 @@ TokenStream *ts_clone(TokenStream *orig_ts);
  *
  ****************************************************************************/
 
-typedef struct Analyzer {
-  void *data;
-  TokenStream *current_ts;
-  TokenStream *(*get_ts)(struct Analyzer *a, char *field, char *text);
-  void (*destroy)(struct Analyzer *a);
-  int ref_cnt;
+typedef struct Analyzer
+{
+    TokenStream *current_ts;
+    TokenStream *(*get_ts)(struct Analyzer *a, char *field, char *text);
+    void (*destroy_i)(struct Analyzer *a);
+    int ref_cnt;
 } Analyzer;
 
-void a_deref(void *p);
+extern void a_deref(Analyzer *a);
 
 #define a_get_ts(ma, field, text) ma->get_ts(ma, field, text)
-#define a_get_new_ts(ma, field, text) ts_clone(ma->get_ts(ma, field, text))
-
-Analyzer *analyzer_create(void *data, TokenStream *ts,
-    void (*destroy)(Analyzer *),
-    TokenStream *(*get_ts)(Analyzer *a, char *field, char *text));
-void a_standard_destroy(Analyzer *a);
-Analyzer *whitespace_analyzer_create(bool lowercase);
-Analyzer *mb_whitespace_analyzer_create(bool lowercase);
-
-Analyzer *letter_analyzer_create(bool lowercase);
-Analyzer *mb_letter_analyzer_create(bool lowercase);
-
-Analyzer *standard_analyzer_create(bool lowercase);
-Analyzer *mb_standard_analyzer_create(bool lowercase);
-
-Analyzer *standard_analyzer_create_with_words(
-    const char **words, bool lowercase);
-Analyzer *standard_analyzer_create_with_words_len(
-    const char **words, int len, bool lowercase);
-Analyzer *mb_standard_analyzer_create_with_words(
-    const char **words, bool lowercase);
-Analyzer *mb_standard_analyzer_create_with_words_len(
-    const char **words, int len, bool lowercase);
-
-typedef struct PerFieldAnalyzer {
-  HshTable *dict;
-  Analyzer *def;
+
+extern Analyzer *analyzer_new(TokenStream *ts,
+                              void (*destroy)(Analyzer *a),
+                              TokenStream *(*get_ts)(Analyzer *a,
+                                                     char *field,
+                                                     char *text));
+extern void a_standard_destroy(Analyzer *a);
+extern Analyzer *whitespace_analyzer_new(bool lowercase);
+extern Analyzer *mb_whitespace_analyzer_new(bool lowercase);
+
+extern Analyzer *letter_analyzer_new(bool lowercase);
+extern Analyzer *mb_letter_analyzer_new(bool lowercase);
+
+extern Analyzer *standard_analyzer_new(bool lowercase);
+extern Analyzer *mb_standard_analyzer_new(bool lowercase);
+
+extern Analyzer *standard_analyzer_new_with_words(const char **words,
+                                                  bool lowercase);
+extern Analyzer *standard_analyzer_new_with_words_len(const char **words, int len,
+                                                      bool lowercase);
+extern Analyzer *mb_standard_analyzer_new_with_words(const char **words,
+                                                     bool lowercase);
+extern Analyzer *mb_standard_analyzer_new_with_words_len(const char **words,
+                                                  int len, bool lowercase);
+
+#define PFA(analyzer) ((PerFieldAnalyzer *)(analyzer))
+typedef struct PerFieldAnalyzer
+{
+    Analyzer    super;
+    HashTable  *dict;
+    Analyzer   *default_a;
 } PerFieldAnalyzer;
 
-Analyzer *per_field_analyzer_create(Analyzer *def);
-void pfa_add_field(Analyzer *self, char *field, Analyzer *analyzer);
+extern Analyzer *per_field_analyzer_new(Analyzer *a);
+extern void pfa_add_field(Analyzer *self, char *field, Analyzer *analyzer);
 
 #endif

data/ext/array.c

@@ -1,85 +1,123 @@
-#include <global.h>
-#include <array.h>
+#include "array.h"
+#include "global.h"
 #include <string.h>
 
-Array *ary_create(int allocate, void (*free_elem)(void *p))
+#define DATA_SZ sizeof(int) * 3
+
+void **ary_new_i(int type_size, int init_capa)
+{
+    int *ary;
+    if (init_capa <= 0) {
+        init_capa = ARY_INIT_CAPA;
+    }
+    ary = ((int *)ecalloc(DATA_SZ + init_capa * type_size));
+    ary[0] = type_size;
+    ary[1] = init_capa;
+    return (void **)&ary[3];
+}
+
+inline void ary_resize_i(void ***ary, int size)
 {
-  Array *ary = ALLOC(Array);
-  if (allocate == 0) {
-    ary->elems = NULL;
-  } else {
-    ary->elems = ALLOC_N(void *, allocate);
-    memset(ary->elems, 0, sizeof(void *) * allocate);
-  }
-  ary->size = 0;
-  ary->allocated = allocate;
-  ary->free_elem = free_elem;
+    size++;
+    if (size >= ary_sz(*ary)) {
+        int capa = ary_capa(*ary);
+        if (size >= capa) {
+            int *ary_start = &((int *)*ary)[-3];
+            while (size >= capa) {
+                capa <<= 1;
+            }
 
-  return ary;
+            ary_start = (int *)erealloc(ary_start,
+                                        DATA_SZ + capa * ary_type_size(*ary));
+            *ary = (void **)&(ary_start[3]);
+            memset(((char *)*ary) + ary_type_size(*ary) * ary_sz(*ary), 0,
+                   (capa - ary_sz(*ary)) * ary_type_size(*ary));
+            ary_capa(*ary) = capa;
+        }
+        ary_sz(*ary) = size;
+    }
 }
 
-void ary_destroy(Array *ary)
+void ary_set_i(void ***ary, int index, void *value)
 {
-  int i;
-  if (ary->free_elem) {
-    for (i = 0; i < ary->size; i++) {
-      if (ary->elems[i])
-        ary->free_elem(ary->elems[i]);
+    if (index < 0) {
+        index += ary_sz(*ary);
+        if (index < 0) {
+            RAISE(INDEX_ERROR, "index %d out array", index);
+        }
     }
-  }
-  free(ary->elems);
-  free(ary);
+    ary_resize_i(ary, index);
+    (*ary)[index] = value;
 }
 
-void ary_set(Array *ary, int index, void *value)
+void *ary_get_i(void **ary, int index)
 {
-  if (index >= ary->allocated) {
-    ary->allocated = (index + 1)*2;
-    REALLOC_N(ary->elems, void *, (ary->allocated));
-    memset((&ary->elems[ary->size]), 0, sizeof(void *) * (ary->allocated - ary->size));
-  }
+    if (index < 0) {
+        index += ary_sz(ary);
+    }
+    if (index >= 0 && index < ary_sz(ary)) {
+        return ary[index];
+    }
+    else {
+        return NULL;
+    }
+}
 
-  if (index >= ary->size)
-    ary->size = index + 1;
+void ary_push_i(void ***ary, void *value)
+{
+    int size = ary_sz(*ary);
+    ary_resize_i(ary, size);
+    (*ary)[size] = value;
+}
 
-  if (ary->free_elem && ary->elems[index])
-    ary->free_elem(ary->elems[index]);
+void *ary_pop_i(void **ary)
+{
+    void *val = ary[--ary_sz(ary)];
+    ary[ary_sz(ary)] = NULL;
+    return val;
+}
 
-  ary->elems[index] = value;
+void ary_unshift_i(void ***ary, void *value)
+{
+    int size = ary_sz(*ary);
+    ary_resize_i(ary, size);
+    memmove(*ary, *ary + 1, size * sizeof(void *));
+    (*ary)[0] = value;
 }
 
-void ary_append(Array *ary, void *value)
+void *ary_shift_i(void **ary)
 {
-  ary_set(ary, ary->size, value);
+    void *val = ary[0];
+    int size = --ary_sz(ary);
+    memmove(ary, ary + 1, size * sizeof(void *));
+    ary[size] = NULL;
+    return val;
 }
 
-void *ary_get(Array *ary, int index)
+void *ary_remove_i(void **ary, int index)
 {
-  if (index >= ary->size)
-    return NULL;
-  return ary->elems[index];
+    if (index >= 0 && index < ary_sz(ary)) {
+        void *val = ary[index];
+        memmove(ary + index, ary + index + 1,
+                (ary_sz(ary) - index + 1) * sizeof(void *));
+        ary_sz(ary)--;
+        return val;
+    }
+    else {
+        return NULL;
+    }
 }
 
-void ary_delete(Array *ary, int index)
+void ary_delete_i(void **ary, int index, void (*free_elem)(void *p))
 {
-  if (index >= ary->size)
-    return;
-  if (ary->free_elem && ary->elems[index])
-    ary->free_elem(ary->elems[index]);
-  ary->elems[index] = NULL;
-  if (index == ary->size - 1)
-    ary->size--;
+    free_elem(ary_remove(ary, index));
 }
 
-void *ary_remove(Array *ary, int index)
+void ary_destroy_i(void **ary, void (*free_elem)(void *p))
 {
-  void *p;
-  if (index >= ary->size)
-    return NULL;
-  p = ary->elems[index];
-  ary->elems[index] = NULL;
-  ary->size--;
-  memmove(&ary->elems[index], &ary->elems[index + 1],
-      sizeof(void *) *(ary->size - index));
-  return p;
+    int i;
+    for (i = ary_sz(ary) - 1; i >= 0; i--) {
+        free_elem(ary[i]);
+    }
+    ary_free(ary);
 }

data/ext/array.h

@@ -1,19 +1,46 @@
 #ifndef FRT_ARRAY_H
 #define FRT_ARRAY_H
 
-typedef struct Array {
-  void **elems;
-  int size;
-  int allocated;
-  void (*free_elem)(void *p);
-} Array;
+#define ARY_INIT_CAPA 8
+#define ary_size(ary)      ary_sz(ary)
+#define ary_sz(ary)        (((int *)ary)[-1])
+#define ary_capa(ary)      (((int *)ary)[-2])
+#define ary_type_size(ary) (((int *)ary)[-3])
+#define ary_start(ary)     ((void **)&(((int *)ary)[-3]))
+#define ary_free(ary)      free(ary_start(ary))
 
-Array *ary_create(int size, void (*free_elem)(void *p));
-void ary_destroy(Array *ary);
-void ary_set(Array *ary, int index, void *value);
-void ary_append(Array *ary, void *value);
-void *ary_get(Array *ary, int index);
-void ary_delete(Array *ary, int index);
-void *ary_remove(Array *ary, int index);
+#define ary_new_type_capa(type, init_capa)\
+                                (type *)ary_new_i(sizeof(type), init_capa)
+#define ary_new_type(type)      (type *)ary_new_i(sizeof(type), 0)
+#define ary_new_capa(init_capa) ary_new_i(sizeof(void *), init_capa)
+#define ary_new()               ary_new_i(sizeof(void *), 0)
+#define ary_resize(ary, size)   ary_resize_i(((void ***)(void *)&ary), size)
+#define ary_set(ary, i, val)    ary_set_i(((void ***)(void *)&ary), i, val)
+#define ary_get(ary, i)         ary_get_i(((void **)ary), i)
+#define ary_push(ary, val)      ary_push_i(((void ***)(void *)&ary), val)
+#define ary_pop(ary)            ary_pop_i(((void **)ary))
+#define ary_unshift(ary, val)   ary_unshift_i(((void ***)(void *)&ary), val)
+#define ary_shift(ary)          ary_shift_i(((void **)ary))
+#define ary_remove(ary, i)      ary_remove_i(((void **)ary), i)
+#define ary_delete(ary, i, f)   ary_delete_i(((void **)ary), i, (free_ft)f)
+#define ary_destroy(ary, f)     ary_destroy_i(((void **)ary), (free_ft)f)
+#define ary_rsz(ary, size)      ary_resize(ary, size)
+#define ary_grow(ary)           ary_resize(ary, ary_sz(ary))
+#define ary_last(ary)           ary[ary_sz(ary) - 1]
+#define ary_sort(ary, cmp)      qsort(ary, ary_size(ary), ary_type_size(ary), cmp)
+#define ary_each_rev(ary, i)    for (i = ary_size(ary) - 1; i >= 0; i--)
+#define ary_each(ary, i)        for (i = 0; i < ary_size(ary); i++)
+
+extern void   ary_resize_i(void ***ary, int size);
+extern void **ary_new_i(int type_size, int init_capa);
+extern void   ary_set_i(void ***ary, int index, void *value);
+extern void  *ary_get_i(void **ary, int index);
+extern void   ary_push_i(void ***ary, void *value);
+extern void  *ary_pop_i(void **ary);
+extern void   ary_unshift_i(void ***ary, void *value);
+extern void  *ary_shift_i(void **ary);
+extern void  *ary_remove_i(void **ary, int index);
+extern void   ary_delete_i(void **ary, int index, void (*free_elem)(void *p));
+extern void   ary_destroy_i(void **ary, void (*free_elem)(void *p));
 
 #endif