ferret 0.9.0 → 0.9.1

data/ext/global.c

@@ -7,6 +7,9 @@
 #include <math.h>
 #include <ctype.h>
 
+static char * const ESTRDUP_MEM_ERROR_MSG = "estrdup failed";
+static char * const MEM_ERROR_MSG = "malloc failed";
+
 const char *EMPTY_STRING = "";
 
 int min3(int a, int b, int c)
@@ -153,7 +156,7 @@ char *estrdup(const char *s)
 
   t = (char *)malloc(strlen(s) + 1);
   if (t == NULL)
-    eprintf(MEM_ERROR, "estrdup(\"%.20s\") failed:", s);
+    RAISE(MEM_ERROR, ESTRDUP_MEM_ERROR_MSG);
 
   strcpy(t, s);
   return t;
@@ -165,7 +168,7 @@ void *emalloc(size_t size)
   void *p;
   p = malloc(size);
   if (p == NULL)
-    eprintf(MEM_ERROR, "malloc of %u bytes failed:", size);
+    RAISE(MEM_ERROR, MEM_ERROR_MSG);
   return p;
 }
 
@@ -174,7 +177,7 @@ void *erealloc(void *ptr, size_t size)
   void *p;
   p = realloc(ptr, size);
   if (p == NULL)
-    eprintf(MEM_ERROR, "malloc of %u bytes failed:", size);
+    RAISE(MEM_ERROR, MEM_ERROR_MSG);
   return p;
 }
 
@@ -217,3 +220,69 @@ void lower_str(char *str)
     str++;
   }
 }
+
+/* strfmt: like sprintf except that it allocates memory for the string */
+char *strfmt(const char *fmt, ...)
+{
+  char *string;
+  char *p = (char *)fmt, *q;
+  va_list args;
+  int len = strlen(fmt) + 1;
+  int slen;
+  char *s;
+  long i;
+  double d;
+
+  q = string = ALLOC_N(char, len);
+
+  va_start(args, fmt);
+  while (*p) {
+    if (*p == '%') {
+      p++;
+      switch (*p) {
+        case 's':
+          p++;
+          s = va_arg(args, char *);
+          if (s) {
+            slen = strlen(s);
+            len += slen;
+            *q = 0;
+            REALLOC_N(string, char, len);
+            q = string + strlen(string);
+            sprintf(q, s);
+            q += slen;
+          }
+          continue;
+        case 'f':
+          p++;
+          len += 32;
+          *q = 0;
+          REALLOC_N(string, char, len);
+          q = string + strlen(string);
+          d = va_arg(args, double);
+          dbl_to_s(q, d);
+          q += strlen(q);
+          continue;
+        case 'd':
+          p++;
+          len += 20;
+          *q = 0;
+          REALLOC_N(string, char, len);
+          q = string + strlen(string);
+          i = va_arg(args, long);
+          sprintf(q, "%ld", i);
+          q += strlen(q);
+          continue;
+        default:
+          break;
+      }
+    }
+    *q = *p;
+    p++;
+    q++;
+  }
+  va_end(args);
+  *q = 0;
+
+  return string;
+}

data/ext/global.h

@@ -4,9 +4,6 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <assert.h>
-#include <pthread.h>
-#include "lang.h"
-
 //#define DEBUG
 #define VALGRIND
 
@@ -19,6 +16,9 @@ typedef unsigned int uint;
 
 typedef void (*destroy_func_t)(void *p);
 
+#include "lang.h"
+#include "except.h"
+
 #define MAX_WORD_SIZE 255
 #define MAX_PATH 1024
 #define MAX_BUFFER_SIZE 1024
@@ -70,4 +70,5 @@ int max3(int a, int b, int c);
 
 char *dbl_to_s(char *buf, double num);
 void lower_str(char *str);
+char *strfmt(const char *fmt, ...);
 #endif

data/ext/hash.c

@@ -388,7 +388,8 @@ int h_resize(HshTable *ht, int min_newsize) {
     ht->table = ALLOC_N(HshEntry, newsize);
   }
   memset(ht->table, 0, sizeof(HshEntry) * newsize);
-  i = ht->fill;
+  i = ht->used;
+  //i = ht->fill;
   ht->fill = ht->used;
   ht->mask = newsize - 1;
   int j = 0;
@@ -399,8 +400,8 @@ int h_resize(HshTable *ht, int min_newsize) {
       he_new->key = he_old->key;
       he_new->value = he_old->value;
       i--;
-    } else if (he_old->key == dummy_key) { //dummy entry
-      i--;
+    //} else if (he_old->key == dummy_key) { //dummy entry
+     // i--;
     } // else empty entry so nothing to do
   }
   if (oldtable != smallcopy && oldtable != ht->smalltable)
@@ -444,3 +445,43 @@ int h_has_key(HshTable *ht, const void *key)
     return HASH_KEY_EQUAL;
   }
 }
+
+void h_each(HshTable *ht,
+    void (*each_kv)(void *key, void *value, void *arg),
+    void *arg)
+{
+  HshEntry *he;
+  int i = ht->used;
+  for (he = ht->table; i > 0; he++) {
+    if (he->value != NULL) {// active entry
+      each_kv(he->key, he->value, arg);
+      i--;
+    }
+  }
+}
+
+HshTable *h_clone(HshTable *ht,
+    h_clone_func_t clone_key,
+    h_clone_func_t clone_value)
+{
+  void *key, *value;
+  HshEntry *he;
+  int i = ht->used;
+  HshTable *ht_clone;
+  if (ht->lookup == &h_lookup_str) {
+    ht_clone = h_new_str(ht->free_key, ht->free_value);
+  } else {
+    ht_clone = h_new(ht->hash, ht->eq, ht->free_key, ht->free_value);
+  }
+
+  for (he = ht->table; i > 0; he++) {
+    if (he->value != NULL) {// active entry
+      key = clone_key ? clone_key(he->key) : he->key;
+      value = clone_value ? clone_value(he->value) : he->value;
+      h_set(ht_clone, key, value);
+      i--;
+    }
+  }
+  return ht_clone;
+}
+

data/ext/hash.h

@@ -74,6 +74,15 @@ int h_set_safe(HshTable *ht, const void *key, void *value);
 int h_has_key(HshTable *ht, const void *key);
 unsigned int str_hash(const char *const str);
 
+void h_each(HshTable *ht,
+    void (*each_kv)(void *key, void *value, void *arg),
+    void *arg);
+
+typedef void *(*h_clone_func_t)(void *val);
+HshTable *h_clone(HshTable *ht,
+    h_clone_func_t clone_key,
+    h_clone_func_t clone_value);
+
 void dummy_free(void *p);
 HshEntry *h_lookup_str(HshTable *ht, register const void *key_p);
 

data/ext/header.h

@@ -0,0 +1,58 @@
+
+#include <limits.h>
+
+#include "api.h"
+
+#define MAXINT INT_MAX
+#define MININT INT_MIN
+
+#define HEAD 2*sizeof(int)
+
+#define SIZE(p)        ((int *)(p))[-1]
+#define SET_SIZE(p, n) ((int *)(p))[-1] = n
+#define CAPACITY(p)    ((int *)(p))[-2]
+
+struct among
+{   int s_size;     /* number of chars in string */
+    symbol * s;       /* search string */
+    int substring_i;/* index to longest matching substring */
+    int result;     /* result of the lookup */
+    int (* function)(struct SN_env *);
+};
+
+extern symbol * create_s(void);
+extern void lose_s(symbol * p);
+
+extern int skip_utf8(const symbol * p, int c, int lb, int l, int n);
+
+extern int in_grouping_U(struct SN_env * z, unsigned char * s, int min, int max);
+extern int in_grouping_b_U(struct SN_env * z, unsigned char * s, int min, int max);
+extern int out_grouping_U(struct SN_env * z, unsigned char * s, int min, int max);
+extern int out_grouping_b_U(struct SN_env * z, unsigned char * s, int min, int max);
+
+extern int in_grouping(struct SN_env * z, unsigned char * s, int min, int max);
+extern int in_grouping_b(struct SN_env * z, unsigned char * s, int min, int max);
+extern int out_grouping(struct SN_env * z, unsigned char * s, int min, int max);
+extern int out_grouping_b(struct SN_env * z, unsigned char * s, int min, int max);
+
+extern int eq_s(struct SN_env * z, int s_size, symbol * s);
+extern int eq_s_b(struct SN_env * z, int s_size, symbol * s);
+extern int eq_v(struct SN_env * z, symbol * p);
+extern int eq_v_b(struct SN_env * z, symbol * p);
+
+extern int find_among(struct SN_env * z, struct among * v, int v_size);
+extern int find_among_b(struct SN_env * z, struct among * v, int v_size);
+
+extern int replace_s(struct SN_env * z, int c_bra, int c_ket, int s_size, const symbol * s, int * adjustment);
+extern int slice_from_s(struct SN_env * z, int s_size, symbol * s);
+extern int slice_from_v(struct SN_env * z, symbol * p);
+extern int slice_del(struct SN_env * z);
+
+extern int insert_s(struct SN_env * z, int bra, int ket, int s_size, symbol * s);
+extern int insert_v(struct SN_env * z, int bra, int ket, symbol * p);
+
+extern symbol * slice_to(struct SN_env * z, symbol * p);
+extern symbol * assign_to(struct SN_env * z, symbol * p);
+
+extern void debug(struct SN_env * z, int number, int line_count);
+

data/ext/inc/except.h

@@ -0,0 +1,88 @@
+#ifndef FRT_EXCEPT_H
+#define FRT_EXCEPT_H
+
+#include <setjmp.h>
+#include <ruby.h>
+
+#define BODY 0
+#define FINALLY -1
+#define EXCEPTION 1
+#define ERROR 1
+#define IO_ERROR 2
+#define ARG_ERROR 3
+#define EOF_ERROR 4
+#define UNSUPPORTED_ERROR 5
+#define STATE_ERROR 6
+#define PARSE_ERROR 7
+#define MEM_ERROR 8
+
+typedef struct xcontext_t {
+  jmp_buf jbuf;
+  struct xcontext_t *next;
+  char *msg;
+  volatile int excode;
+  int handled : 1;
+  int in_finally : 1;
+} xcontext_t;
+
+RUBY_EXTERN int rb_thread_critical;
+extern xcontext_t *xtop_context;
+
+#define TRY\
+  xcontext_t xcontext;\
+  rb_thread_critical = Qtrue;\
+  xcontext.next = xtop_context;\
+  xtop_context = &xcontext;\
+  xcontext.handled = true;\
+  xcontext.in_finally = false;\
+  switch (setjmp(xcontext.jbuf)) {\
+    case BODY:
+
+
+#define XENDTRY\
+  }\
+  xtop_context = xcontext.next;\
+  if (!xcontext.handled) {\
+    RAISE(xcontext.excode, xcontext.msg);\
+  }\
+  rb_thread_critical = 0;
+
+#define ENDTRY\
+  }\
+  if (!xcontext.in_finally) {\
+    xtop_context = xcontext.next;\
+    if (!xcontext.handled) {\
+      RAISE(xcontext.excode, xcontext.msg);\
+    }\
+    xcontext.in_finally = 1;\
+    longjmp(xcontext.jbuf, FINALLY);\
+  }\
+  rb_thread_critical = 0;
+
+#define XFINALLY default: xcontext.in_finally = 1;
+
+#define XCATCHALL break; default: xcontext.in_finally = 1;
+
+//fprintf(stderr,"Error occured in %s, %d: %s\n", __FILE__, __LINE__, __func__);
+#define RAISE(xexcode, xmsg) \
+  do {\
+    if (!xtop_context) {\
+      eprintf(EXCEPTION_CODE, "Error: exception %d not handled: %s", xexcode, xmsg);\
+    } else if (!xtop_context->in_finally) {\
+      xtop_context->msg = xmsg;\
+      xtop_context->excode = xexcode;\
+      xtop_context->handled = false;\
+      longjmp(xtop_context->jbuf, xexcode);\
+    } else if (xtop_context->handled) {\
+      xtop_context->msg = xmsg;\
+      xtop_context->excode = xexcode;\
+      xtop_context->handled = false;\
+    }\
+  } while (0)
+
+#define HANDLED() xcontext.handled = 1 /* true */
+
+extern char * const UNSUPPORTED_ERROR_MSG;
+extern char * const EOF_ERROR_MSG;
+
+#endif

data/ext/inc/lang.h

@@ -2,6 +2,7 @@
 #define FRT_LANG_H
 
 #include <ruby.h>
+#include "hash.h"
 
 #define FERRET_EXT
 
@@ -14,28 +15,37 @@ extern void setprogname(const char *str);
 
 extern VALUE cQueryParseException;
 
-#define ERROR rb_eException
-#define IO_ERROR rb_eIOError
-#define ARG_ERROR rb_eArgError
-#define EOF_ERROR rb_eEOFError
-#define UNSUPPORTED_ERROR rb_eNotImpError
-#define STATE_ERROR rb_eException
-#define PARSE_ERROR cQueryParseException
-#define MEM_ERROR rb_eNoMemError
+#define EXCEPTION_CODE rb_eException
+//#define IO_ERROR rb_eIOError
+//#define ARG_ERROR rb_eArgError
+//#define EOF_ERROR rb_eEOFError
+//#define UNSUPPORTED_ERROR rb_eNotImpError
+//#define STATE_ERROR rb_eException
+//#define PARSE_ERROR cQueryParseException
+//#define MEM_ERROR rb_eNoMemError
 
 typedef void * mutex_t;
-typedef void * thread_key_t;
+typedef struct HshTable * thread_key_t;
+typedef int thread_once_t;
 #define MUTEX_INITIALIZER NULL
 #define MUTEX_RECURSIVE_INITIALIZER NULL
+#define THREAD_ONCE_INIT 1;
 #define mutex_init(a, b)
 #define mutex_lock(a)
 #define mutex_trylock(a)
 #define mutex_unlock(a)
 #define mutex_destroy(a)
-#define thread_key_create(a, b)
-#define thread_key_delete(a)
-#define thread_setspecific(a, b)
-#define thread_getspecific(a) NULL
+#define thread_key_create(a, b) frt_thread_key_create(a, b)
+#define thread_key_delete(a) frt_thread_key_delete(a)
+#define thread_setspecific(a, b) frt_thread_setspecific(a, b)
+#define thread_getspecific(a) frt_thread_getspecific(a)
 #define thread_exit(a)
+#define thread_once(a, b) frt_thread_once(a, b)
+
+void frt_thread_once(int *once_control, void (*init_routine) (void));
+void frt_thread_key_create(thread_key_t *key, void (*destr_function) (void *));
+void frt_thread_key_delete(thread_key_t key);
+void frt_thread_setspecific(thread_key_t key, const void *pointer);
+void *frt_thread_getspecific(thread_key_t key);
 
 #endif

data/ext/ind.c

@@ -1,6 +1,8 @@
 #include <string.h>
 #include "search.h"
 
+static char * const NON_UNIQUE_KEY_ERROR_MSG = "Tried to use a key that was not unique";
+
 static const char *ID_STRING = "id";
 
 #define INDEX_CLOSE_READER(self) do {\
@@ -53,7 +55,7 @@ Index *index_create(Store *store, Analyzer *analyzer, HashSet *def_fields,
     self->analyzer = analyzer;
     self->close_analyzer = false;
   } else {
-    self->analyzer = standard_analyzer_create();
+    self->analyzer = mb_standard_analyzer_create(true);
     self->close_analyzer = true;
   }
   self->use_compound_file = true;
@@ -69,6 +71,7 @@ Index *index_create(Store *store, Analyzer *analyzer, HashSet *def_fields,
   self->id_field = (char *)ID_STRING;
   self->def_field = (char *)ID_STRING;
   self->auto_flush = false;
+  self->check_latest = true;
 
   self->qp = qp_create(all_fields, def_fields, self->analyzer);
   /* Index is a convenience class so set qp convenience options */
@@ -117,7 +120,7 @@ inline void ensure_writer_open(Index *self)
 inline void ensure_reader_open(Index *self)
 {
   if (self->ir) {
-    if (!ir_is_latest(self->ir)) {
+    if (self->check_latest && !ir_is_latest(self->ir)) {
       INDEX_CLOSE_READER(self);
       self->ir = ir_open(self->store, false);
     }
@@ -196,7 +199,7 @@ static void inline index_add_doc_i(Index *self, Document *doc)
     td = sea_search(self->sea, q, 0, 1, NULL, NULL);
     if (td->total_hits > 1) {
       td_destroy(td);
-      eprintf(ARG_ERROR, "Tried to use a key that was not unique");
+      RAISE(ARG_ERROR, NON_UNIQUE_KEY_ERROR_MSG);
     } else if (td->total_hits == 1) {
       ir_delete_doc(self->ir, td->hits[0]->doc);
     }
@@ -230,7 +233,7 @@ void index_add_doc(Index *self, Document *doc)
 void index_add_string(Index *self, char *str, Analyzer *analyzer)
 {
   Document *doc = doc_create();
-  doc_add_field(doc, df_create(self->id_field, estrdup(str),
+  doc_add_field(doc, df_create(self->def_field, estrdup(str),
         DF_STORE_YES, DF_INDEX_TOKENIZED, DF_TERM_VECTOR_NO));
   if (analyzer) index_add_doc_a(self, doc, analyzer);
   else index_add_doc(self, doc);
@@ -242,7 +245,7 @@ void index_add_array(Index *self, Array *ary, Analyzer *analyzer)
   int i;
   Document *doc = doc_create();
   for (i = 0; i < ary->size; i++) {
-    doc_add_field(doc, df_create(self->id_field, estrdup(ary->elems[i]),
+    doc_add_field(doc, df_create(self->def_field, estrdup(ary->elems[i]),
         DF_STORE_YES, DF_INDEX_TOKENIZED, DF_TERM_VECTOR_NO));
   }
   if (analyzer) index_add_doc_a(self, doc, analyzer);
@@ -341,11 +344,14 @@ void index_delete_term(Index *self, Term *term)
   mutex_lock(&self->store->ext_mutex);
   ensure_reader_open(self);
   tde = ir_term_docs_for(self->ir, term);
-  while (tde->next(tde)) {
-    ir_delete_doc(self->ir, tde->doc_num(tde));
-    AUTOFLUSH_IR;
-  }
-  tde->close(tde);
+  TRY
+    while (tde->next(tde)) {
+      ir_delete_doc(self->ir, tde->doc_num(tde));
+      AUTOFLUSH_IR;
+    }
+  XFINALLY
+    tde->close(tde);
+  XENDTRY
   mutex_unlock(&self->store->ext_mutex);
 }