ferret 0.10.6 → 0.10.7

data/ext/hash.h

@@ -28,7 +28,7 @@ enum HashSetValues
  */
 typedef struct
 {
-    ulong hash;
+    unsigned long hash;
     void *key;
     void *value;
 } HashEntry;
@@ -58,11 +58,11 @@ typedef struct HashTable
 
     /* the following function pointers are used internally and should not be
      * used outside of the HashTable methods */
-    HashEntry *(*lookup_i)(struct HashTable *self, register const void *key);
-    ulong      (*hash_i)(const void *key);
-    int        (*eq_i)(const void *key1, const void *key2);
-    void       (*free_key_i)(void *p);
-    void       (*free_value_i)(void *p);
+    HashEntry    *(*lookup_i)(struct HashTable *self, register const void *key);
+    unsigned long (*hash_i)(const void *key);
+    int           (*eq_i)(const void *key1, const void *key2);
+    void          (*free_key_i)(void *p);
+    void          (*free_value_i)(void *p);
 } HashTable;
 
 /**
@@ -72,7 +72,7 @@ typedef struct HashTable
  * @param key object to hash
  * @return an unsigned 32-bit integer hash value
  */
-typedef ulong (*hash_ft)(const void *key);
+typedef unsigned long (*hash_ft)(const void *key);
 
 /**
  * Equals function type used by HashTable. A function of this type must be
@@ -86,15 +86,33 @@ typedef int (*eq_ft)(const void *key1, const void *key2);
  * function used to add integers to a HashTable, either as the key or the
  * value.
  */
-extern ulong *imalloc(ulong value);
+extern unsigned long *imalloc(unsigned long value);
 
 /**
  * Determine a hash value for a string. The string must be null terminated
  *
  * @param str string to hash
- * @return an unsigned 32-bit integer hash value
+ * @return an unsigned long integer hash value
+ */
+extern unsigned long str_hash(const char *const str);
+
+/**
+ * Determine a hash value for a pointer. Just cast the pointer to an unsigned
+ * long.
+ *
+ * @param ptr pointer to hash
+ * @return an unsigned long integer hash value
+ */
+extern unsigned long ptr_hash(const void *const ptr);
+
+/**
+ * Determine if two pointers point to the same point in memory.
+ *
+ * @param q1 first pointer
+ * @param q2 second pointer
+ * @return true if the pointers are equal
  */
-extern ulong str_hash(const char *const str);
+extern int ptr_eq(const void *q1, const void *q2);
 
 /**
  * Create a new HashTable that uses any type of object as it's key. The
@@ -280,7 +298,7 @@ extern int h_has_key(HashTable *self, const void *key);
  * @return the value referenced by the key +key+. If there is no value
  *   referenced by that key, NULL is returned.
  */
-extern void *h_get_int(HashTable *self, const ulong key);
+extern void *h_get_int(HashTable *self, const unsigned long key);
 
 /**
  * Delete the value in HashTable referenced by the integer key +key+. When the
@@ -297,7 +315,7 @@ extern void *h_get_int(HashTable *self, const ulong key);
  * @return true if the object was successfully deleted or false if the key was
  *   not found
  */
-extern int h_del_int(HashTable *self, const ulong key);
+extern int h_del_int(HashTable *self, const unsigned long key);
 
 /**
  * Remove the value in HashTable referenced by the integer key +key+. When the
@@ -311,7 +329,7 @@ extern int h_del_int(HashTable *self, const ulong key);
  * @param key the integer key to lookup
  * @return the value referenced by +key+ if it can be found or NULL otherwise
  */
-extern void *h_rem_int(HashTable *self, const ulong key);
+extern void *h_rem_int(HashTable *self, const unsigned long key);
 
 /**
  * WARNING: this function may destroy an old value if the key already exists
@@ -341,7 +359,7 @@ extern void *h_rem_int(HashTable *self, const ulong key);
  *                              the existing key so no key was freed
  *   </pre>
  */
-extern int h_set_int(HashTable *self, const ulong key, void *value);
+extern int h_set_int(HashTable *self, const unsigned long key, void *value);
 
 /**
  * Add the value +value+ to the HashTable referencing it with integer key
@@ -353,7 +371,7 @@ extern int h_set_int(HashTable *self, const ulong key, void *value);
  * @param value the value to add to the HashTable
  * @return true if the value was successfully added or false otherwise
  */
-extern int h_set_safe_int(HashTable *self, const ulong key, void *value);
+extern int h_set_safe_int(HashTable *self, const unsigned long key, void *value);
 /**
  * Check whether integer key +key+ exists in the HashTable.
  *
@@ -361,7 +379,7 @@ extern int h_set_safe_int(HashTable *self, const ulong key, void *value);
  * @param key the integer key to check for in the HashTable
  * @return true if the key exists in the HashTable, false otherwise.
  */
-extern int h_has_key_int(HashTable *self, const ulong key);
+extern int h_has_key_int(HashTable *self, const unsigned long key);
 
 typedef void (*h_each_key_val_ft)(void *key, void *value, void *arg);
 

data/ext/hashset.c

@@ -17,7 +17,7 @@ static HashSet *hs_alloc(void (*free_elem) (void *p))
     return hs;
 }
 
-HashSet *hs_new(ulong (*hash)(const void *p),
+HashSet *hs_new(unsigned long (*hash)(const void *p),
                 int (*eq)(const void *p1, const void *p2),
                 void (*free_elem)(void *p))
 {
@@ -129,11 +129,14 @@ void *hs_rem(HashSet *hs, void *elem)
     }
     else {
         int i = *index;
+        int j;
         ret_elem = hs->elems[i];
         h_del(hs->ht, elem);
         hs->size--;
-        memmove(&hs->elems[i], &hs->elems[i + 1],
-                sizeof(void *) * (hs->size - i));
+        for (j = i; j < hs->size; j++) {
+            hs->elems[j] = hs->elems[j+1];
+            h_set(hs->ht, hs->elems[j], imalloc(j));
+        }
         return ret_elem;
     }
 }

data/ext/hashset.h

@@ -36,7 +36,7 @@ typedef struct HashSet
  *   when the HashSet if destroyed or duplicate elements are added to the Set
  * @return a newly allocated HashSet structure
  */
-extern HashSet *hs_new(ulong (*hash)(const void *p),
+extern HashSet *hs_new(unsigned long (*hash)(const void *p),
                        int (*eq)(const void *p1, const void *p2),
                        void (*free_elem)(void *p));
 

data/ext/index.c

@@ -47,9 +47,9 @@ static char *ste_next(TermEnum *te);
  *
  ***************************************************************************/
 
-static ulong co_hash(const void *key)
+static unsigned long co_hash(const void *key)
 {
-    return (ulong)key;
+    return (unsigned long)key;
 }
 
 static int co_eq(const void *key1, const void *key2)
@@ -2675,7 +2675,9 @@ static bool mtdpe_next(TermDocEnum *tde)
     do {
         freq += sub_tde->freq(sub_tde);
         if (freq > mtdpe->pos_queue_capa) {
-            mtdpe->pos_queue_capa <<= 1;
+            do {
+                mtdpe->pos_queue_capa <<= 1;
+            } while (freq > mtdpe->pos_queue_capa);
             REALLOC_N(mtdpe->pos_queue, int, mtdpe->pos_queue_capa);
         }
 
@@ -2804,7 +2806,7 @@ void ir_acquire_write_lock(IndexReader *ir)
     }
 
     if (ir->write_lock == NULL) {
-        ir->write_lock = ir->store->open_lock(ir->store, WRITE_LOCK_NAME);
+        ir->write_lock = open_lock(ir->store, WRITE_LOCK_NAME);
         if (!ir->write_lock->obtain(ir->write_lock)) {/* obtain write lock */
             RAISE(LOCK_ERROR, "Could not obtain write lock when trying to "
                               "write changes to the index. Check that there "
@@ -2819,7 +2821,7 @@ void ir_acquire_write_lock(IndexReader *ir)
         if (sis_read_current_version(ir->store) > ir->sis->version) {
             ir->is_stale = true;
             ir->write_lock->release(ir->write_lock);
-            ir->store->close_lock(ir->write_lock);
+            close_lock(ir->write_lock);
             ir->write_lock = NULL;
             RAISE(STATE_ERROR, "IndexReader out of date and no longer valid "
                                "for delete, undelete, or set_norm operations. "
@@ -3016,7 +3018,7 @@ void ir_commit_i(IndexReader *ir)
         Lock *commit_lock;
 
         mutex_lock(&ir->store->mutex);
-        commit_lock = ir->store->open_lock(ir->store, COMMIT_LOCK_NAME);
+        commit_lock = open_lock(ir->store, COMMIT_LOCK_NAME);
         if (!commit_lock->obtain(commit_lock)) { /* obtain write lock */
             RAISE(LOCK_ERROR, "Error trying to commit the index. Commit "
                               "lock already obtained");
@@ -3026,13 +3028,13 @@ void ir_commit_i(IndexReader *ir)
         sis_write(ir->sis, ir->store);
 
         commit_lock->release(commit_lock);
-        ir->store->close_lock(commit_lock);
+        close_lock(commit_lock);
         mutex_unlock(&ir->store->mutex);
 
         if (ir->write_lock != NULL) {
             /* release write lock */
             ir->write_lock->release(ir->write_lock);
-            ir->store->close_lock(ir->write_lock);
+            close_lock(ir->write_lock);
             ir->write_lock = NULL;
         }
         ir->has_changes = false;
@@ -3092,15 +3094,15 @@ bool ir_is_latest(IndexReader *ir)
 {
     volatile bool is_latest = false;
 
-    Lock *commit_lock = ir->store->open_lock(ir->store, COMMIT_LOCK_NAME);
+    Lock *commit_lock = open_lock(ir->store, COMMIT_LOCK_NAME);
     if (!commit_lock->obtain(commit_lock)) {
-        ir->store->close_lock(commit_lock);
+        close_lock(commit_lock);
         RAISE(LOCK_ERROR, "Error detecting if the current index is latest "
               "version. Commit lock currently obtained");
     }
     is_latest = (sis_read_current_version(ir->store) == ir->sis->version);
     commit_lock->release(commit_lock);
-    ir->store->close_lock(commit_lock);
+    close_lock(commit_lock);
 
     return is_latest;
 }
@@ -5156,7 +5158,7 @@ static void iw_merge_segments(IndexWriter *iw, const int min_seg,
     si->doc_cnt = sm_merge(merger);
 
     mutex_lock(&iw->store->mutex);
-    commit_lock = iw->store->open_lock(iw->store, COMMIT_LOCK_NAME);
+    commit_lock = open_lock(iw->store, COMMIT_LOCK_NAME);
 
     /* *** OBTAIN COMMIT LOCK *** */
     if (!commit_lock->obtain(commit_lock)) {
@@ -5179,7 +5181,7 @@ static void iw_merge_segments(IndexWriter *iw, const int min_seg,
         iw_commit_compound_file(iw, si->name, commit_lock);
     }
 
-    iw->store->close_lock(commit_lock);
+    close_lock(commit_lock);
 
     mutex_unlock(&iw->store->mutex);
 
@@ -5233,7 +5235,7 @@ static void iw_flush_ram_segment(IndexWriter *iw)
     dw_flush(iw->dw);
 
     mutex_lock(&iw->store->mutex);
-    commit_lock = iw->store->open_lock(iw->store, COMMIT_LOCK_NAME);
+    commit_lock = open_lock(iw->store, COMMIT_LOCK_NAME);
 
     if (!commit_lock->obtain(commit_lock)) {
         RAISE(LOCK_ERROR, "Couldn't obtain commit lock to write segments file");
@@ -5247,7 +5249,7 @@ static void iw_flush_ram_segment(IndexWriter *iw)
     if (iw->config.use_compound_file) {
         iw_commit_compound_file(iw, si->name, commit_lock);
     }
-    iw->store->close_lock(commit_lock);
+    close_lock(commit_lock);
     mutex_unlock(&iw->store->mutex);
 
     iw_maybe_merge_segments(iw);
@@ -5346,7 +5348,7 @@ void iw_close(IndexWriter *iw)
     sim_destroy(iw->similarity);
 
     iw->write_lock->release(iw->write_lock);
-    iw->store->close_lock(iw->write_lock);
+    close_lock(iw->write_lock);
     store_deref(iw->store);
 
     mutex_destroy(&iw->mutex);
@@ -5364,7 +5366,7 @@ IndexWriter *iw_open(Store *store, Analyzer *analyzer, const Config *config)
     iw->config = *config;
 
     TRY
-        iw->write_lock = store->open_lock(store, WRITE_LOCK_NAME);
+        iw->write_lock = open_lock(store, WRITE_LOCK_NAME);
         if (!iw->write_lock->obtain(iw->write_lock)) {
             RAISE(LOCK_ERROR,
                   "Couldn't obtain write lock when opening IndexWriter");
@@ -5376,7 +5378,7 @@ IndexWriter *iw_open(Store *store, Analyzer *analyzer, const Config *config)
     XCATCHALL
         if (iw->write_lock) {
             iw->write_lock->release(iw->write_lock);
-            iw->store->close_lock(iw->write_lock);
+            close_lock(iw->write_lock);
         }
         if (iw->sis) sis_destroy(iw->sis);
         if (iw->fis) fis_deref(iw->fis);
@@ -5677,7 +5679,7 @@ void iw_add_readers(IndexWriter *iw, IndexReader **readers, const int r_cnt)
     }
 
     mutex_lock(&iw->store->mutex);
-    commit_lock = iw->store->open_lock(iw->store, COMMIT_LOCK_NAME);
+    commit_lock = open_lock(iw->store, COMMIT_LOCK_NAME);
 
     if (!commit_lock->obtain(commit_lock)) {
         RAISE(LOCK_ERROR, "Couldn't obtain commit lock to write segments file");
@@ -5686,7 +5688,7 @@ void iw_add_readers(IndexWriter *iw, IndexReader **readers, const int r_cnt)
     fis_write(iw->fis, iw->store);
     sis_write(iw->sis, iw->store);
     commit_lock->release(commit_lock);
-    iw->store->close_lock(commit_lock);
+    close_lock(commit_lock);
     mutex_unlock(&iw->store->mutex);
 
     iw_optimize_i(iw);

data/ext/q_boolean.c

@@ -1285,7 +1285,7 @@ void bc_deref(BooleanClause *self)
     }
 }
 
-static ulong bc_hash(BooleanClause *self)
+static unsigned long bc_hash(BooleanClause *self)
 {
     return ((q_hash(self->query) << 2) | self->occur);
 }
@@ -1492,10 +1492,10 @@ static Similarity *bq_get_similarity(Query *self, Searcher *searcher)
     return BQ(self)->similarity;
 }
 
-static ulong bq_hash(Query *self)
+static unsigned long bq_hash(Query *self)
 {
     int i;
-    ulong hash = 0;
+    unsigned long hash = 0;
     for (i = 0; i < BQ(self)->clause_cnt; i++) {
         hash ^= bc_hash(BQ(self)->clauses[i]);
     }

data/ext/q_const_score.c

@@ -129,7 +129,7 @@ static void csq_destroy(Query *self)
     q_destroy_i(self);
 }
 
-static ulong csq_hash(Query *self)
+static unsigned long csq_hash(Query *self)
 {
     return filt_hash(CScQ(self)->filter);
 }

data/ext/q_fuzzy.c

@@ -223,7 +223,7 @@ static void fuzq_destroy(Query *self)
     q_destroy_i(self);
 }
 
-static ulong fuzq_hash(Query *self)
+static unsigned long fuzq_hash(Query *self)
 {
     return str_hash(FzQ(self)->term) ^ str_hash(FzQ(self)->field)
         ^ float2int(FzQ(self)->min_sim) ^ FzQ(self)->pre_len;

data/ext/q_match_all.c

@@ -120,7 +120,7 @@ char *maq_to_s(Query *self, const char *field)
     }
 }
 
-static ulong maq_hash(Query *self)
+static unsigned long maq_hash(Query *self)
 {
     (void)self;
     return 0;

data/ext/q_multi_term.c

@@ -572,10 +572,10 @@ static void multi_tq_extract_terms(Query *self, HashSet *terms)
     }
 }
 
-static ulong multi_tq_hash(Query *self)
+static unsigned long multi_tq_hash(Query *self)
 {
     int i;
-    ulong hash = str_hash(MTQ(self)->field);
+    unsigned long hash = str_hash(MTQ(self)->field);
     PriorityQueue *boosted_terms = MTQ(self)->boosted_terms;
     for (i = boosted_terms->size; i > 0; i--) {
         BoostedTerm *bt = (BoostedTerm *)boosted_terms->heap[i];

data/ext/q_parser.c

@@ -1852,12 +1852,19 @@ static int yyerror(QParser *qp, char const *msg)
 
 static TokenStream *get_cached_ts(QParser *qp, char *field, char *text)
 {
-    TokenStream *ts = h_get(qp->ts_cache, field);
-    if (!ts) {
-        ts = a_get_ts(qp->analyzer, field, text);
-        h_set(qp->ts_cache, estrdup(field), ts);
+    TokenStream *ts;
+    if (!qp->tokenized_fields || hs_exists(qp->tokenized_fields, field)) {
+        ts = h_get(qp->ts_cache, field);
+        if (!ts) {
+            ts = a_get_ts(qp->analyzer, field, text);
+            h_set(qp->ts_cache, estrdup(field), ts);
+        }
+        else {
+            ts->reset(ts, text);
+        }
     }
     else {
+        ts = qp->non_tokenizer;
         ts->reset(ts, text);
     }
     return ts;
@@ -2054,7 +2061,8 @@ static Query *get_wild_q(QParser *qp, char *field, char *pattern)
     char *p;
     int len = (int)strlen(pattern);
 
-    if (qp->wild_lower) {
+    if (qp->wild_lower
+        && (!qp->tokenized_fields || hs_exists(qp->tokenized_fields, field))) {
         lower_str(pattern);
     }
     
@@ -2277,15 +2285,20 @@ void qp_destroy(QParser *self)
     if (self->close_def_fields) {
         hs_destroy(self->def_fields);
     }
+    if (self->tokenized_fields) {
+        hs_destroy(self->tokenized_fields);
+    }
     hs_destroy(self->all_fields);
     hs_destroy(self->fields_buf);
     h_destroy(self->field_cache);
     h_destroy(self->ts_cache);
+    tk_destroy(self->non_tokenizer);
     a_deref(self->analyzer);
     free(self);
 }
 
-QParser *qp_new(HashSet *all_fields, HashSet *def_fields, Analyzer *analyzer)
+QParser *qp_new(HashSet *all_fields, HashSet *def_fields,
+                HashSet *tokenized_fields, Analyzer *analyzer)
 {
     int i;
     QParser *self = ALLOC(QParser);
@@ -2298,6 +2311,7 @@ QParser *qp_new(HashSet *all_fields, HashSet *def_fields, Analyzer *analyzer)
     self->def_slop = 0;
     self->fields_buf = hs_new_str(NULL);
     self->all_fields = all_fields;
+    self->tokenized_fields = tokenized_fields;
     if (def_fields) {
         self->def_fields = def_fields;
         for (i = 0; i < self->def_fields->size; i++) {
@@ -2321,6 +2335,7 @@ QParser *qp_new(HashSet *all_fields, HashSet *def_fields, Analyzer *analyzer)
     self->analyzer = analyzer;
     self->ts_cache = h_new_str(&free, (free_ft)&ts_deref);
     self->buf_index = 0;
+    self->non_tokenizer = non_tokenizer_new();
     mutex_init(&self->mutex, NULL);
     return self;
 }