ferret 0.11.6 → 0.11.8.4

data/ext/lang.c

@@ -0,0 +1,10 @@
+#include "lang.h"
+#include "internal.h"
+
+
+struct timeval rb_time_interval _((VALUE));
+extern void micro_sleep(const int micro_seconds)
+{
+    rb_thread_wait_for(rb_time_interval(rb_float_new((double)micro_seconds/1000000.0)));
+}
+

data/ext/lang.h

@@ -10,39 +10,59 @@
 #undef rename
 #undef read
 
-#define frt_malloc xmalloc
-#define frt_calloc(n) xcalloc(n, 1)
-#define frt_realloc xrealloc
+#define frt_emalloc xmalloc
+#define frt_ecalloc(n) xcalloc(n, 1)
+#define frt_erealloc xrealloc
+/* FIXME: should eventually delete this */
+#define FRT_REALLOC_N REALLOC_N
 
 
 #ifdef FRT_HAS_ISO_VARARGS
 /* C99-compliant compiler */
 
-# define FRT_EXIT(...) frt_rb_raise(__FILE__, __LINE__, __func__, __VA_ARGS__)
-extern void frt_rb_raise(const char *file, int line_num, const char *func,
+# define FRT_XEXIT(...) frb_rb_raise(__FILE__, __LINE__, __func__, __VA_ARGS__)
+extern void frb_rb_raise(const char *file, int line_num, const char *func,
                          const char *err_type, const char *fmt, ...);
 
-# define V_FRT_EXIT(err_type, fmt, args) \
-    vfrt_rb_raise(__FILE__, __LINE__, __func__, err_type, fmt, args)
-extern void vfrt_rb_raise(const char *file, int line_num, const char *func,
+# define FRT_VEXIT(err_type, fmt, args) \
+    frb_vrb_raise(__FILE__, __LINE__, __func__, err_type, fmt, args)
+extern void frb_vrb_raise(const char *file, int line_num, const char *func,
                           const char *err_type, const char *fmt, va_list args);
 
 #elif defined(FRT_HAS_GNUC_VARARGS)
 /* gcc has an extension */
 
-# define FRT_EXIT(args...) frt_rb_raise(__FILE__, __LINE__, __func__, ##args)
-extern void frt_rb_raise(const char *file, int line_num, const char *func,
+# define FRT_XEXIT(args...) frb_rb_raise(__FILE__, __LINE__, __func__, ##args)
+extern void frb_rb_raise(const char *file, int line_num, const char *func,
                          const char *err_type, const char *fmt, ...);
 
-# define V_FRT_EXIT(err_type, fmt, args) \
-    vfrt_rb_raise(__FILE__, __LINE__, __func__, err_type, fmt, args)
-extern void vfrt_rb_raise(const char *file, int line_num, const char *func,
+# define FRT_VEXIT(err_type, fmt, args) \
+    frb_vrb_raise(__FILE__, __LINE__, __func__, err_type, fmt, args)
+extern void frb_vrb_raise(const char *file, int line_num, const char *func,
                           const char *err_type, const char *fmt, va_list args);
 #else
 /* Can't do VARARGS */
 
-extern void FRT_EXIT(const char *err_type, const char *fmt, ...);
-extern void V_FRT_EXIT(const char *err_type, const char *fmt, va_list args);
+extern void FRT_XEXIT(const char *err_type, const char *fmt, ...);
+extern void FRT_VEXIT(const char *err_type, const char *fmt, va_list args);
+#endif
+
+#ifdef RUBY_RUBY_H
+#  define FRT_RUBY_VERSION_1_9
+#endif
+
+// ruby 1.8 compat with 1.9 to avoid ifdefs
+#if !defined RSTRING_LEN
+#define RSTRING_LEN(a) RSTRING(a)->len
+#endif
+#if !defined RSTRING_PTR
+#define RSTRING_PTR(a) RSTRING(a)->ptr
+#endif
+#if !defined RARRAY_LEN
+#define RARRAY_LEN(a) RARRAY(a)->len
+#endif
+#if !defined RARRAY_PTR
+#define RARRAY_PTR(a) RARRAY(a)->ptr
 #endif
 
 #endif

data/ext/mempool.c

@@ -1,6 +1,7 @@
 #include "global.h"
 #include "mempool.h"
 #include <string.h>
+#include "internal.h"
 
 MemoryPool *mp_new_capa(int chuck_size, int init_buf_capa)
 {
@@ -9,7 +10,7 @@ MemoryPool *mp_new_capa(int chuck_size, int init_buf_capa)
     mp->buf_capa = init_buf_capa;
     mp->buffers = ALLOC_N(char *, init_buf_capa);
 
-    mp->buffers[0] = mp->curr_buffer = emalloc(mp->chunk_size);
+    mp->buffers[0] = mp->curr_buffer = (char *)emalloc(mp->chunk_size);
     mp->buf_alloc = 1;
     mp->buf_pointer = 0;
     mp->pointer = 0;
@@ -38,7 +39,7 @@ INLINE void *mp_alloc(MemoryPool *mp, int size)
                 mp->buf_capa <<= 1;
                 REALLOC_N(mp->buffers, char *, mp->buf_capa);
             }
-            mp->buffers[mp->buf_pointer] = emalloc(mp->chunk_size);
+            mp->buffers[mp->buf_pointer] = (char *)emalloc(mp->chunk_size);
         }
         p = mp->curr_buffer = mp->buffers[mp->buf_pointer];
         mp->pointer = size;
@@ -49,12 +50,12 @@ INLINE void *mp_alloc(MemoryPool *mp, int size)
 char *mp_strdup(MemoryPool *mp, const char *str)
 {
     int len = strlen(str) + 1;
-    return memcpy(mp_alloc(mp, len), str, len);
+    return (char *)memcpy(mp_alloc(mp, len), str, len);
 }
 
 char *mp_strndup(MemoryPool *mp, const char *str, int len)
 {
-    char *s = memcpy(mp_alloc(mp, len + 1), str, len);
+    char *s = (char *)memcpy(mp_alloc(mp, len + 1), str, len);
     s[len] = '\0';
     return s;
 }

data/ext/mempool.h

@@ -1,10 +1,14 @@
 #ifndef FRT_MEM_POOL_H
 #define FRT_MEM_POOL_H
 
-#define MP_BUF_SIZE 65536
-#define MP_INIT_CAPA 4
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define FRT_MP_BUF_SIZE 65536
+#define FRT_MP_INIT_CAPA 4
 
-typedef struct MemoryPool {
+typedef struct FrtMemoryPool {
     int buf_alloc;
     int buf_capa;
     int buf_pointer;
@@ -12,24 +16,28 @@ typedef struct MemoryPool {
     int chunk_size;
     char *curr_buffer;
     char **buffers;
-} MemoryPool;
-
-extern MemoryPool *mp_new();
-extern MemoryPool *mp_new_capa(int chunk_size, int init_capa);
-extern INLINE void *mp_alloc(MemoryPool *mp, int size);
-extern void mp_reset(MemoryPool *mp);
-extern void mp_destroy(MemoryPool *mp);
-extern char *mp_strdup(MemoryPool *mp, const char *str);
-extern char *mp_strndup(MemoryPool *mp, const char *str, int len);
-extern void *mp_memdup(MemoryPool *mp, const void *p, int len);
-extern int mp_used(MemoryPool *mp);
-
-#define MP_ALLOC_N(mp,type,n) (type *)mp_alloc(mp, sizeof(type)*(n))
-#define MP_ALLOC(mp,type) (type *)mp_alloc(mp, sizeof(type))
-
-#define MP_ALLOC_AND_ZERO(mp,type)\
-    (type*)memset(mp_alloc(mp, sizeof(type)), 0, sizeof(type))
-#define MP_ALLOC_AND_ZERO_N(mp,type,n)\
-    (type*)ZEROSET_N(mp_alloc(mp, sizeof(type)*(n)), type, n)
+} FrtMemoryPool;
+
+extern FrtMemoryPool *frt_mp_new();
+extern FrtMemoryPool *frt_mp_new_capa(int chunk_size, int init_capa);
+extern FRT_INLINE void *frt_mp_alloc(FrtMemoryPool *mp, int size);
+extern void frt_mp_reset(FrtMemoryPool *mp);
+extern void frt_mp_destroy(FrtMemoryPool *mp);
+extern char *frt_mp_strdup(FrtMemoryPool *mp, const char *str);
+extern char *frt_mp_strndup(FrtMemoryPool *mp, const char *str, int len);
+extern void *frt_mp_memdup(FrtMemoryPool *mp, const void *p, int len);
+extern int frt_mp_used(FrtMemoryPool *mp);
+
+#define FRT_MP_ALLOC_N(mp,type,n) (type *)frt_mp_alloc(mp, sizeof(type)*(n))
+#define FRT_MP_ALLOC(mp,type) (type *)frt_mp_alloc(mp, sizeof(type))
+
+#define FRT_MP_ALLOC_AND_ZERO(mp,type)\
+    (type*)memset(frt_mp_alloc(mp, sizeof(type)), 0, sizeof(type))
+#define FRT_MP_ALLOC_AND_ZERO_N(mp,type,n)\
+    (type*)FRT_ZEROSET_N(frt_mp_alloc(mp, sizeof(type)*(n)), type, n)
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
 
 #endif

data/ext/modules.h

@@ -4,7 +4,8 @@
  * Do not edit manually.
  *
  * Modules included by this file are: danish, dutch, english, finnish, french,
- * german, italian, norwegian, porter, portuguese, russian, spanish, swedish
+ * german, hungarian, italian, norwegian, porter, portuguese, romanian,
+ * russian, spanish, swedish, turkish
  */
 
 #include "stem_ISO_8859_1_danish.h"
@@ -19,6 +20,8 @@
 #include "stem_UTF_8_french.h"
 #include "stem_ISO_8859_1_german.h"
 #include "stem_UTF_8_german.h"
+#include "stem_ISO_8859_1_hungarian.h"
+#include "stem_UTF_8_hungarian.h"
 #include "stem_ISO_8859_1_italian.h"
 #include "stem_UTF_8_italian.h"
 #include "stem_ISO_8859_1_norwegian.h"
@@ -27,34 +30,39 @@
 #include "stem_UTF_8_porter.h"
 #include "stem_ISO_8859_1_portuguese.h"
 #include "stem_UTF_8_portuguese.h"
+#include "stem_ISO_8859_2_romanian.h"
+#include "stem_UTF_8_romanian.h"
 #include "stem_KOI8_R_russian.h"
 #include "stem_UTF_8_russian.h"
 #include "stem_ISO_8859_1_spanish.h"
 #include "stem_UTF_8_spanish.h"
 #include "stem_ISO_8859_1_swedish.h"
 #include "stem_UTF_8_swedish.h"
+#include "stem_UTF_8_turkish.h"
 
 typedef enum {
-  ENC_UNKNOWN,
+  ENC_UNKNOWN=0,
   ENC_ISO_8859_1,
+  ENC_ISO_8859_2,
   ENC_KOI8_R,
   ENC_UTF_8
-} stemmer_encoding;
+} stemmer_encoding_t;
 
 struct stemmer_encoding {
   const char * name;
-  stemmer_encoding enc;
+  stemmer_encoding_t enc;
 };
 static struct stemmer_encoding encodings[] = {
   {"ISO_8859_1", ENC_ISO_8859_1},
+  {"ISO_8859_2", ENC_ISO_8859_2},
   {"KOI8_R", ENC_KOI8_R},
   {"UTF_8", ENC_UTF_8},
-  {0,0}
+  {0,ENC_UNKNOWN}
 };
 
 struct stemmer_modules {
   const char * name;
-  stemmer_encoding enc; 
+  stemmer_encoding_t enc; 
   struct SN_env * (*create)(void);
   void (*close)(struct SN_env *);
   int (*stem)(struct SN_env *);
@@ -102,6 +110,12 @@ static struct stemmer_modules modules[] = {
   {"ger", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
   {"german", ENC_ISO_8859_1, german_ISO_8859_1_create_env, german_ISO_8859_1_close_env, german_ISO_8859_1_stem},
   {"german", ENC_UTF_8, german_UTF_8_create_env, german_UTF_8_close_env, german_UTF_8_stem},
+  {"hu", ENC_ISO_8859_1, hungarian_ISO_8859_1_create_env, hungarian_ISO_8859_1_close_env, hungarian_ISO_8859_1_stem},
+  {"hu", ENC_UTF_8, hungarian_UTF_8_create_env, hungarian_UTF_8_close_env, hungarian_UTF_8_stem},
+  {"hun", ENC_ISO_8859_1, hungarian_ISO_8859_1_create_env, hungarian_ISO_8859_1_close_env, hungarian_ISO_8859_1_stem},
+  {"hun", ENC_UTF_8, hungarian_UTF_8_create_env, hungarian_UTF_8_close_env, hungarian_UTF_8_stem},
+  {"hungarian", ENC_ISO_8859_1, hungarian_ISO_8859_1_create_env, hungarian_ISO_8859_1_close_env, hungarian_ISO_8859_1_stem},
+  {"hungarian", ENC_UTF_8, hungarian_UTF_8_create_env, hungarian_UTF_8_close_env, hungarian_UTF_8_stem},
   {"it", ENC_ISO_8859_1, italian_ISO_8859_1_create_env, italian_ISO_8859_1_close_env, italian_ISO_8859_1_stem},
   {"it", ENC_UTF_8, italian_UTF_8_create_env, italian_UTF_8_close_env, italian_UTF_8_stem},
   {"ita", ENC_ISO_8859_1, italian_ISO_8859_1_create_env, italian_ISO_8859_1_close_env, italian_ISO_8859_1_stem},
@@ -126,8 +140,16 @@ static struct stemmer_modules modules[] = {
   {"portuguese", ENC_UTF_8, portuguese_UTF_8_create_env, portuguese_UTF_8_close_env, portuguese_UTF_8_stem},
   {"pt", ENC_ISO_8859_1, portuguese_ISO_8859_1_create_env, portuguese_ISO_8859_1_close_env, portuguese_ISO_8859_1_stem},
   {"pt", ENC_UTF_8, portuguese_UTF_8_create_env, portuguese_UTF_8_close_env, portuguese_UTF_8_stem},
+  {"ro", ENC_ISO_8859_2, romanian_ISO_8859_2_create_env, romanian_ISO_8859_2_close_env, romanian_ISO_8859_2_stem},
+  {"ro", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
+  {"romanian", ENC_ISO_8859_2, romanian_ISO_8859_2_create_env, romanian_ISO_8859_2_close_env, romanian_ISO_8859_2_stem},
+  {"romanian", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
+  {"ron", ENC_ISO_8859_2, romanian_ISO_8859_2_create_env, romanian_ISO_8859_2_close_env, romanian_ISO_8859_2_stem},
+  {"ron", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
   {"ru", ENC_KOI8_R, russian_KOI8_R_create_env, russian_KOI8_R_close_env, russian_KOI8_R_stem},
   {"ru", ENC_UTF_8, russian_UTF_8_create_env, russian_UTF_8_close_env, russian_UTF_8_stem},
+  {"rum", ENC_ISO_8859_2, romanian_ISO_8859_2_create_env, romanian_ISO_8859_2_close_env, romanian_ISO_8859_2_stem},
+  {"rum", ENC_UTF_8, romanian_UTF_8_create_env, romanian_UTF_8_close_env, romanian_UTF_8_stem},
   {"rus", ENC_KOI8_R, russian_KOI8_R_create_env, russian_KOI8_R_close_env, russian_KOI8_R_stem},
   {"rus", ENC_UTF_8, russian_UTF_8_create_env, russian_UTF_8_close_env, russian_UTF_8_stem},
   {"russian", ENC_KOI8_R, russian_KOI8_R_create_env, russian_KOI8_R_close_env, russian_KOI8_R_stem},
@@ -142,7 +164,10 @@ static struct stemmer_modules modules[] = {
   {"swe", ENC_UTF_8, swedish_UTF_8_create_env, swedish_UTF_8_close_env, swedish_UTF_8_stem},
   {"swedish", ENC_ISO_8859_1, swedish_ISO_8859_1_create_env, swedish_ISO_8859_1_close_env, swedish_ISO_8859_1_stem},
   {"swedish", ENC_UTF_8, swedish_UTF_8_create_env, swedish_UTF_8_close_env, swedish_UTF_8_stem},
-  {0,0,0,0,0}
+  {"tr", ENC_UTF_8, turkish_UTF_8_create_env, turkish_UTF_8_close_env, turkish_UTF_8_stem},
+  {"tur", ENC_UTF_8, turkish_UTF_8_create_env, turkish_UTF_8_close_env, turkish_UTF_8_stem},
+  {"turkish", ENC_UTF_8, turkish_UTF_8_create_env, turkish_UTF_8_close_env, turkish_UTF_8_stem},
+  {0,ENC_UNKNOWN,0,0,0}
 };
 static const char * algorithm_names[] = {
   "danish", 
@@ -151,12 +176,15 @@ static const char * algorithm_names[] = {
   "finnish", 
   "french", 
   "german", 
+  "hungarian", 
   "italian", 
   "norwegian", 
   "porter", 
   "portuguese", 
+  "romanian", 
   "russian", 
   "spanish", 
   "swedish", 
+  "turkish", 
   0
 };

data/ext/multimapper.c

@@ -2,6 +2,7 @@
 #include "array.h"
 #include "bitvector.h"
 #include <string.h>
+#include "internal.h"
 
 #define St(state) ((State *)(state))
 #define UCtoI(val) ((int)(unsigned char)(val))
@@ -161,7 +162,8 @@ static INLINE void mulmap_bv_set_states(BitVector *bv, int *states, int cnt)
 
 static DeterministicState *mulmap_process_state(MultiMapper *self, BitVector *bv)
 {
-    DeterministicState *current_state = h_get(self->dstates_map, bv);
+    DeterministicState *current_state
+        = (DeterministicState *)h_get(self->dstates_map, bv);
     if (current_state == NULL) {
         int bit, i;
         int match_len = 0, max_match_len = 0;
@@ -263,7 +265,7 @@ int mulmap_map_len(MultiMapper *self, char *to, char *from, int capa)
     DeterministicState *state = start;
     char *s = from, *d = to, *end = to + capa - 1;
     if (self->d_size == 0) {
-        RAISE(STATE_ERROR, "You forgot to compile your MultiMapper");
+        mulmap_compile(self);
     }
     while (*s && d < end) {
         state = state->next[UCtoI(*s)];
@@ -292,6 +294,45 @@ char *mulmap_map(MultiMapper *self, char *to, char *from, int capa)
     return to;
 }
 
+/* Maps a string to a dynamically allocated string */
+char *mulmap_dynamic_map(MultiMapper *self, char *from)
+{
+    DeterministicState *start = self->dstates[0];
+    DeterministicState *state = start;
+    int capa = strlen(from);
+    char *to = (char *)ecalloc(capa);
+    char *s = from, *d = to, *end = to + capa - 1;
+    if (self->d_size == 0) {
+        mulmap_compile(self);
+    }
+    do {
+        while (*s && d < end) {
+            state = state->next[UCtoI(*s)];
+            if (state->mapping) {
+                int len = state->mapping_len;
+                d -= (state->longest_match - 1);
+                if ((d + len) > end) {
+                    len = end - d;
+                }
+                memcpy(d, state->mapping, len);
+                d += len;
+                state = start;
+            }
+            else {
+                *(d++) = *s;
+            }
+            s++;
+        }
+        if (*s) {
+            capa += 1024;
+            erealloc(to, capa);
+            end = to + capa - 1;
+        }
+    } while(*s);
+    *d = '\0';
+    return to;
+}
+
 void mulmap_destroy(MultiMapper *self)
 {
     if (--(self->ref_cnt) <= 0) {

data/ext/multimapper.h

@@ -1,51 +1,60 @@
 #ifndef FRT_MAPPER_H
 #define FRT_MAPPER_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
 #include "hash.h"
 
-typedef struct State
+typedef struct FrtState
 {
-    int  (*next)(struct State *self, int c, int *states);
-    void (*destroy_i)(struct State *self);
-    int  (*is_match)(struct State *self, char **mapping);
-} State;
+    int  (*next)(struct FrtState *self, int c, int *states);
+    void (*destroy_i)(struct FrtState *self);
+    int  (*is_match)(struct FrtState *self, char **mapping);
+} FrtState;
 
-typedef struct DeterministicState
+typedef struct FrtDeterministicState
 {
-    struct DeterministicState *next[256];
+    struct FrtDeterministicState *next[256];
     int longest_match;
     char *mapping;
     int mapping_len;
-} DeterministicState;
+} FrtDeterministicState;
 
-typedef struct Mapping
+typedef struct FrtMapping
 {
     char *pattern;
     char *replacement;
-} Mapping;
+} FrtMapping;
 
-typedef struct MultiMapper
+typedef struct FrtMultiMapper
 {
-    Mapping **mappings;
+    FrtMapping **mappings;
     int size;
     int capa;
-    DeterministicState **dstates;
+    FrtDeterministicState **dstates;
     int d_size;
     int d_capa;
     unsigned char alphabet[256];
     int a_size;
-    HashTable *dstates_map;
-    State **nstates;
+    FrtHash *dstates_map;
+    FrtState **nstates;
     int nsize;
     int *next_states;
     int ref_cnt;
-} MultiMapper;
-
-extern MultiMapper *mulmap_new();
-extern void mulmap_add_mapping(MultiMapper *self, const char *p, const char *r);
-extern void mulmap_compile(MultiMapper *self);
-extern char *mulmap_map(MultiMapper *self, char *to, char *from, int capa);
-extern int mulmap_map_len(MultiMapper *self, char *to, char *from, int capa);
-extern void mulmap_destroy(MultiMapper *self);
+} FrtMultiMapper;
+
+extern FrtMultiMapper *frt_mulmap_new();
+extern void frt_mulmap_add_mapping(FrtMultiMapper *self, const char *p, const char *r);
+extern void frt_mulmap_compile(FrtMultiMapper *self);
+extern char *frt_mulmap_map(FrtMultiMapper *self, char *to, char *from, int capa);
+extern char *frt_mulmap_dynamic_map(FrtMultiMapper *self, char *from);
+extern int frt_mulmap_map_len(FrtMultiMapper *self, char *to, char *from, int capa);
+extern void frt_mulmap_destroy(FrtMultiMapper *self);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
 
 #endif