ferret 0.10.6 → 0.10.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. data/ext/analysis.c +136 -107
  2. data/ext/analysis.h +4 -0
  3. data/ext/bitvector.c +2 -2
  4. data/ext/bitvector.h +1 -1
  5. data/ext/compound_io.c +4 -4
  6. data/ext/defines.h +0 -2
  7. data/ext/filter.c +3 -3
  8. data/ext/fs_store.c +4 -4
  9. data/ext/hash.c +29 -18
  10. data/ext/hash.h +34 -16
  11. data/ext/hashset.c +6 -3
  12. data/ext/hashset.h +1 -1
  13. data/ext/index.c +22 -20
  14. data/ext/q_boolean.c +3 -3
  15. data/ext/q_const_score.c +1 -1
  16. data/ext/q_fuzzy.c +1 -1
  17. data/ext/q_match_all.c +1 -1
  18. data/ext/q_multi_term.c +2 -2
  19. data/ext/q_parser.c +21 -6
  20. data/ext/q_phrase.c +2 -2
  21. data/ext/q_prefix.c +1 -1
  22. data/ext/q_range.c +3 -3
  23. data/ext/q_span.c +8 -8
  24. data/ext/q_term.c +1 -1
  25. data/ext/q_wildcard.c +1 -1
  26. data/ext/r_analysis.c +10 -4
  27. data/ext/r_index.c +89 -12
  28. data/ext/r_qparser.c +67 -4
  29. data/ext/r_search.c +11 -1
  30. data/ext/r_store.c +51 -35
  31. data/ext/ram_store.c +18 -18
  32. data/ext/search.c +1 -1
  33. data/ext/search.h +25 -23
  34. data/ext/similarity.c +1 -1
  35. data/ext/sort.c +1 -1
  36. data/ext/store.c +22 -3
  37. data/ext/store.h +8 -2
  38. data/lib/ferret/index.rb +14 -4
  39. data/lib/ferret_version.rb +1 -1
  40. data/test/test_helper.rb +3 -0
  41. data/test/unit/analysis/tc_analyzer.rb +5 -5
  42. data/test/unit/analysis/tc_token_stream.rb +3 -3
  43. data/test/unit/index/tc_index_writer.rb +1 -1
  44. data/test/unit/query_parser/tc_query_parser.rb +7 -5
  45. data/test/unit/search/tc_filter.rb +1 -1
  46. data/test/unit/search/tc_fuzzy_query.rb +1 -1
  47. data/test/unit/search/tc_index_searcher.rb +1 -1
  48. data/test/unit/search/tc_multi_searcher.rb +1 -1
  49. data/test/unit/search/tc_search_and_sort.rb +1 -1
  50. data/test/unit/search/tc_spans.rb +1 -1
  51. metadata +4 -3
data/ext/hash.h CHANGED
@@ -28,7 +28,7 @@ enum HashSetValues
28
28
  */
29
29
  typedef struct
30
30
  {
31
- ulong hash;
31
+ unsigned long hash;
32
32
  void *key;
33
33
  void *value;
34
34
  } HashEntry;
@@ -58,11 +58,11 @@ typedef struct HashTable
58
58
 
59
59
  /* the following function pointers are used internally and should not be
60
60
  * used outside of the HashTable methods */
61
- HashEntry *(*lookup_i)(struct HashTable *self, register const void *key);
62
- ulong (*hash_i)(const void *key);
63
- int (*eq_i)(const void *key1, const void *key2);
64
- void (*free_key_i)(void *p);
65
- void (*free_value_i)(void *p);
61
+ HashEntry *(*lookup_i)(struct HashTable *self, register const void *key);
62
+ unsigned long (*hash_i)(const void *key);
63
+ int (*eq_i)(const void *key1, const void *key2);
64
+ void (*free_key_i)(void *p);
65
+ void (*free_value_i)(void *p);
66
66
  } HashTable;
67
67
 
68
68
  /**
@@ -72,7 +72,7 @@ typedef struct HashTable
72
72
  * @param key object to hash
73
73
  * @return an unsigned 32-bit integer hash value
74
74
  */
75
- typedef ulong (*hash_ft)(const void *key);
75
+ typedef unsigned long (*hash_ft)(const void *key);
76
76
 
77
77
  /**
78
78
  * Equals function type used by HashTable. A function of this type must be
@@ -86,15 +86,33 @@ typedef int (*eq_ft)(const void *key1, const void *key2);
86
86
  * function used to add integers to a HashTable, either as the key or the
87
87
  * value.
88
88
  */
89
- extern ulong *imalloc(ulong value);
89
+ extern unsigned long *imalloc(unsigned long value);
90
90
 
91
91
  /**
92
92
  * Determine a hash value for a string. The string must be null terminated
93
93
  *
94
94
  * @param str string to hash
95
- * @return an unsigned 32-bit integer hash value
95
+ * @return an unsigned long integer hash value
96
+ */
97
+ extern unsigned long str_hash(const char *const str);
98
+
99
+ /**
100
+ * Determine a hash value for a pointer. Just cast the pointer to an unsigned
101
+ * long.
102
+ *
103
+ * @param ptr pointer to hash
104
+ * @return an unsigned long integer hash value
105
+ */
106
+ extern unsigned long ptr_hash(const void *const ptr);
107
+
108
+ /**
109
+ * Determine if two pointers point to the same point in memory.
110
+ *
111
+ * @param q1 first pointer
112
+ * @param q2 second pointer
113
+ * @return true if the pointers are equal
96
114
  */
97
- extern ulong str_hash(const char *const str);
115
+ extern int ptr_eq(const void *q1, const void *q2);
98
116
 
99
117
  /**
100
118
  * Create a new HashTable that uses any type of object as it's key. The
@@ -280,7 +298,7 @@ extern int h_has_key(HashTable *self, const void *key);
280
298
  * @return the value referenced by the key +key+. If there is no value
281
299
  * referenced by that key, NULL is returned.
282
300
  */
283
- extern void *h_get_int(HashTable *self, const ulong key);
301
+ extern void *h_get_int(HashTable *self, const unsigned long key);
284
302
 
285
303
  /**
286
304
  * Delete the value in HashTable referenced by the integer key +key+. When the
@@ -297,7 +315,7 @@ extern void *h_get_int(HashTable *self, const ulong key);
297
315
  * @return true if the object was successfully deleted or false if the key was
298
316
  * not found
299
317
  */
300
- extern int h_del_int(HashTable *self, const ulong key);
318
+ extern int h_del_int(HashTable *self, const unsigned long key);
301
319
 
302
320
  /**
303
321
  * Remove the value in HashTable referenced by the integer key +key+. When the
@@ -311,7 +329,7 @@ extern int h_del_int(HashTable *self, const ulong key);
311
329
  * @param key the integer key to lookup
312
330
  * @return the value referenced by +key+ if it can be found or NULL otherwise
313
331
  */
314
- extern void *h_rem_int(HashTable *self, const ulong key);
332
+ extern void *h_rem_int(HashTable *self, const unsigned long key);
315
333
 
316
334
  /**
317
335
  * WARNING: this function may destroy an old value if the key already exists
@@ -341,7 +359,7 @@ extern void *h_rem_int(HashTable *self, const ulong key);
341
359
  * the existing key so no key was freed
342
360
  * </pre>
343
361
  */
344
- extern int h_set_int(HashTable *self, const ulong key, void *value);
362
+ extern int h_set_int(HashTable *self, const unsigned long key, void *value);
345
363
 
346
364
  /**
347
365
  * Add the value +value+ to the HashTable referencing it with integer key
@@ -353,7 +371,7 @@ extern int h_set_int(HashTable *self, const ulong key, void *value);
353
371
  * @param value the value to add to the HashTable
354
372
  * @return true if the value was successfully added or false otherwise
355
373
  */
356
- extern int h_set_safe_int(HashTable *self, const ulong key, void *value);
374
+ extern int h_set_safe_int(HashTable *self, const unsigned long key, void *value);
357
375
  /**
358
376
  * Check whether integer key +key+ exists in the HashTable.
359
377
  *
@@ -361,7 +379,7 @@ extern int h_set_safe_int(HashTable *self, const ulong key, void *value);
361
379
  * @param key the integer key to check for in the HashTable
362
380
  * @return true if the key exists in the HashTable, false otherwise.
363
381
  */
364
- extern int h_has_key_int(HashTable *self, const ulong key);
382
+ extern int h_has_key_int(HashTable *self, const unsigned long key);
365
383
 
366
384
  typedef void (*h_each_key_val_ft)(void *key, void *value, void *arg);
367
385
 
data/ext/hashset.c CHANGED
@@ -17,7 +17,7 @@ static HashSet *hs_alloc(void (*free_elem) (void *p))
17
17
  return hs;
18
18
  }
19
19
 
20
- HashSet *hs_new(ulong (*hash)(const void *p),
20
+ HashSet *hs_new(unsigned long (*hash)(const void *p),
21
21
  int (*eq)(const void *p1, const void *p2),
22
22
  void (*free_elem)(void *p))
23
23
  {
@@ -129,11 +129,14 @@ void *hs_rem(HashSet *hs, void *elem)
129
129
  }
130
130
  else {
131
131
  int i = *index;
132
+ int j;
132
133
  ret_elem = hs->elems[i];
133
134
  h_del(hs->ht, elem);
134
135
  hs->size--;
135
- memmove(&hs->elems[i], &hs->elems[i + 1],
136
- sizeof(void *) * (hs->size - i));
136
+ for (j = i; j < hs->size; j++) {
137
+ hs->elems[j] = hs->elems[j+1];
138
+ h_set(hs->ht, hs->elems[j], imalloc(j));
139
+ }
137
140
  return ret_elem;
138
141
  }
139
142
  }
data/ext/hashset.h CHANGED
@@ -36,7 +36,7 @@ typedef struct HashSet
36
36
  * when the HashSet if destroyed or duplicate elements are added to the Set
37
37
  * @return a newly allocated HashSet structure
38
38
  */
39
- extern HashSet *hs_new(ulong (*hash)(const void *p),
39
+ extern HashSet *hs_new(unsigned long (*hash)(const void *p),
40
40
  int (*eq)(const void *p1, const void *p2),
41
41
  void (*free_elem)(void *p));
42
42
 
data/ext/index.c CHANGED
@@ -47,9 +47,9 @@ static char *ste_next(TermEnum *te);
47
47
  *
48
48
  ***************************************************************************/
49
49
 
50
- static ulong co_hash(const void *key)
50
+ static unsigned long co_hash(const void *key)
51
51
  {
52
- return (ulong)key;
52
+ return (unsigned long)key;
53
53
  }
54
54
 
55
55
  static int co_eq(const void *key1, const void *key2)
@@ -2675,7 +2675,9 @@ static bool mtdpe_next(TermDocEnum *tde)
2675
2675
  do {
2676
2676
  freq += sub_tde->freq(sub_tde);
2677
2677
  if (freq > mtdpe->pos_queue_capa) {
2678
- mtdpe->pos_queue_capa <<= 1;
2678
+ do {
2679
+ mtdpe->pos_queue_capa <<= 1;
2680
+ } while (freq > mtdpe->pos_queue_capa);
2679
2681
  REALLOC_N(mtdpe->pos_queue, int, mtdpe->pos_queue_capa);
2680
2682
  }
2681
2683
 
@@ -2804,7 +2806,7 @@ void ir_acquire_write_lock(IndexReader *ir)
2804
2806
  }
2805
2807
 
2806
2808
  if (ir->write_lock == NULL) {
2807
- ir->write_lock = ir->store->open_lock(ir->store, WRITE_LOCK_NAME);
2809
+ ir->write_lock = open_lock(ir->store, WRITE_LOCK_NAME);
2808
2810
  if (!ir->write_lock->obtain(ir->write_lock)) {/* obtain write lock */
2809
2811
  RAISE(LOCK_ERROR, "Could not obtain write lock when trying to "
2810
2812
  "write changes to the index. Check that there "
@@ -2819,7 +2821,7 @@ void ir_acquire_write_lock(IndexReader *ir)
2819
2821
  if (sis_read_current_version(ir->store) > ir->sis->version) {
2820
2822
  ir->is_stale = true;
2821
2823
  ir->write_lock->release(ir->write_lock);
2822
- ir->store->close_lock(ir->write_lock);
2824
+ close_lock(ir->write_lock);
2823
2825
  ir->write_lock = NULL;
2824
2826
  RAISE(STATE_ERROR, "IndexReader out of date and no longer valid "
2825
2827
  "for delete, undelete, or set_norm operations. "
@@ -3016,7 +3018,7 @@ void ir_commit_i(IndexReader *ir)
3016
3018
  Lock *commit_lock;
3017
3019
 
3018
3020
  mutex_lock(&ir->store->mutex);
3019
- commit_lock = ir->store->open_lock(ir->store, COMMIT_LOCK_NAME);
3021
+ commit_lock = open_lock(ir->store, COMMIT_LOCK_NAME);
3020
3022
  if (!commit_lock->obtain(commit_lock)) { /* obtain write lock */
3021
3023
  RAISE(LOCK_ERROR, "Error trying to commit the index. Commit "
3022
3024
  "lock already obtained");
@@ -3026,13 +3028,13 @@ void ir_commit_i(IndexReader *ir)
3026
3028
  sis_write(ir->sis, ir->store);
3027
3029
 
3028
3030
  commit_lock->release(commit_lock);
3029
- ir->store->close_lock(commit_lock);
3031
+ close_lock(commit_lock);
3030
3032
  mutex_unlock(&ir->store->mutex);
3031
3033
 
3032
3034
  if (ir->write_lock != NULL) {
3033
3035
  /* release write lock */
3034
3036
  ir->write_lock->release(ir->write_lock);
3035
- ir->store->close_lock(ir->write_lock);
3037
+ close_lock(ir->write_lock);
3036
3038
  ir->write_lock = NULL;
3037
3039
  }
3038
3040
  ir->has_changes = false;
@@ -3092,15 +3094,15 @@ bool ir_is_latest(IndexReader *ir)
3092
3094
  {
3093
3095
  volatile bool is_latest = false;
3094
3096
 
3095
- Lock *commit_lock = ir->store->open_lock(ir->store, COMMIT_LOCK_NAME);
3097
+ Lock *commit_lock = open_lock(ir->store, COMMIT_LOCK_NAME);
3096
3098
  if (!commit_lock->obtain(commit_lock)) {
3097
- ir->store->close_lock(commit_lock);
3099
+ close_lock(commit_lock);
3098
3100
  RAISE(LOCK_ERROR, "Error detecting if the current index is latest "
3099
3101
  "version. Commit lock currently obtained");
3100
3102
  }
3101
3103
  is_latest = (sis_read_current_version(ir->store) == ir->sis->version);
3102
3104
  commit_lock->release(commit_lock);
3103
- ir->store->close_lock(commit_lock);
3105
+ close_lock(commit_lock);
3104
3106
 
3105
3107
  return is_latest;
3106
3108
  }
@@ -5156,7 +5158,7 @@ static void iw_merge_segments(IndexWriter *iw, const int min_seg,
5156
5158
  si->doc_cnt = sm_merge(merger);
5157
5159
 
5158
5160
  mutex_lock(&iw->store->mutex);
5159
- commit_lock = iw->store->open_lock(iw->store, COMMIT_LOCK_NAME);
5161
+ commit_lock = open_lock(iw->store, COMMIT_LOCK_NAME);
5160
5162
 
5161
5163
  /* *** OBTAIN COMMIT LOCK *** */
5162
5164
  if (!commit_lock->obtain(commit_lock)) {
@@ -5179,7 +5181,7 @@ static void iw_merge_segments(IndexWriter *iw, const int min_seg,
5179
5181
  iw_commit_compound_file(iw, si->name, commit_lock);
5180
5182
  }
5181
5183
 
5182
- iw->store->close_lock(commit_lock);
5184
+ close_lock(commit_lock);
5183
5185
 
5184
5186
  mutex_unlock(&iw->store->mutex);
5185
5187
 
@@ -5233,7 +5235,7 @@ static void iw_flush_ram_segment(IndexWriter *iw)
5233
5235
  dw_flush(iw->dw);
5234
5236
 
5235
5237
  mutex_lock(&iw->store->mutex);
5236
- commit_lock = iw->store->open_lock(iw->store, COMMIT_LOCK_NAME);
5238
+ commit_lock = open_lock(iw->store, COMMIT_LOCK_NAME);
5237
5239
 
5238
5240
  if (!commit_lock->obtain(commit_lock)) {
5239
5241
  RAISE(LOCK_ERROR, "Couldn't obtain commit lock to write segments file");
@@ -5247,7 +5249,7 @@ static void iw_flush_ram_segment(IndexWriter *iw)
5247
5249
  if (iw->config.use_compound_file) {
5248
5250
  iw_commit_compound_file(iw, si->name, commit_lock);
5249
5251
  }
5250
- iw->store->close_lock(commit_lock);
5252
+ close_lock(commit_lock);
5251
5253
  mutex_unlock(&iw->store->mutex);
5252
5254
 
5253
5255
  iw_maybe_merge_segments(iw);
@@ -5346,7 +5348,7 @@ void iw_close(IndexWriter *iw)
5346
5348
  sim_destroy(iw->similarity);
5347
5349
 
5348
5350
  iw->write_lock->release(iw->write_lock);
5349
- iw->store->close_lock(iw->write_lock);
5351
+ close_lock(iw->write_lock);
5350
5352
  store_deref(iw->store);
5351
5353
 
5352
5354
  mutex_destroy(&iw->mutex);
@@ -5364,7 +5366,7 @@ IndexWriter *iw_open(Store *store, Analyzer *analyzer, const Config *config)
5364
5366
  iw->config = *config;
5365
5367
 
5366
5368
  TRY
5367
- iw->write_lock = store->open_lock(store, WRITE_LOCK_NAME);
5369
+ iw->write_lock = open_lock(store, WRITE_LOCK_NAME);
5368
5370
  if (!iw->write_lock->obtain(iw->write_lock)) {
5369
5371
  RAISE(LOCK_ERROR,
5370
5372
  "Couldn't obtain write lock when opening IndexWriter");
@@ -5376,7 +5378,7 @@ IndexWriter *iw_open(Store *store, Analyzer *analyzer, const Config *config)
5376
5378
  XCATCHALL
5377
5379
  if (iw->write_lock) {
5378
5380
  iw->write_lock->release(iw->write_lock);
5379
- iw->store->close_lock(iw->write_lock);
5381
+ close_lock(iw->write_lock);
5380
5382
  }
5381
5383
  if (iw->sis) sis_destroy(iw->sis);
5382
5384
  if (iw->fis) fis_deref(iw->fis);
@@ -5677,7 +5679,7 @@ void iw_add_readers(IndexWriter *iw, IndexReader **readers, const int r_cnt)
5677
5679
  }
5678
5680
 
5679
5681
  mutex_lock(&iw->store->mutex);
5680
- commit_lock = iw->store->open_lock(iw->store, COMMIT_LOCK_NAME);
5682
+ commit_lock = open_lock(iw->store, COMMIT_LOCK_NAME);
5681
5683
 
5682
5684
  if (!commit_lock->obtain(commit_lock)) {
5683
5685
  RAISE(LOCK_ERROR, "Couldn't obtain commit lock to write segments file");
@@ -5686,7 +5688,7 @@ void iw_add_readers(IndexWriter *iw, IndexReader **readers, const int r_cnt)
5686
5688
  fis_write(iw->fis, iw->store);
5687
5689
  sis_write(iw->sis, iw->store);
5688
5690
  commit_lock->release(commit_lock);
5689
- iw->store->close_lock(commit_lock);
5691
+ close_lock(commit_lock);
5690
5692
  mutex_unlock(&iw->store->mutex);
5691
5693
 
5692
5694
  iw_optimize_i(iw);
data/ext/q_boolean.c CHANGED
@@ -1285,7 +1285,7 @@ void bc_deref(BooleanClause *self)
1285
1285
  }
1286
1286
  }
1287
1287
 
1288
- static ulong bc_hash(BooleanClause *self)
1288
+ static unsigned long bc_hash(BooleanClause *self)
1289
1289
  {
1290
1290
  return ((q_hash(self->query) << 2) | self->occur);
1291
1291
  }
@@ -1492,10 +1492,10 @@ static Similarity *bq_get_similarity(Query *self, Searcher *searcher)
1492
1492
  return BQ(self)->similarity;
1493
1493
  }
1494
1494
 
1495
- static ulong bq_hash(Query *self)
1495
+ static unsigned long bq_hash(Query *self)
1496
1496
  {
1497
1497
  int i;
1498
- ulong hash = 0;
1498
+ unsigned long hash = 0;
1499
1499
  for (i = 0; i < BQ(self)->clause_cnt; i++) {
1500
1500
  hash ^= bc_hash(BQ(self)->clauses[i]);
1501
1501
  }
data/ext/q_const_score.c CHANGED
@@ -129,7 +129,7 @@ static void csq_destroy(Query *self)
129
129
  q_destroy_i(self);
130
130
  }
131
131
 
132
- static ulong csq_hash(Query *self)
132
+ static unsigned long csq_hash(Query *self)
133
133
  {
134
134
  return filt_hash(CScQ(self)->filter);
135
135
  }
data/ext/q_fuzzy.c CHANGED
@@ -223,7 +223,7 @@ static void fuzq_destroy(Query *self)
223
223
  q_destroy_i(self);
224
224
  }
225
225
 
226
- static ulong fuzq_hash(Query *self)
226
+ static unsigned long fuzq_hash(Query *self)
227
227
  {
228
228
  return str_hash(FzQ(self)->term) ^ str_hash(FzQ(self)->field)
229
229
  ^ float2int(FzQ(self)->min_sim) ^ FzQ(self)->pre_len;
data/ext/q_match_all.c CHANGED
@@ -120,7 +120,7 @@ char *maq_to_s(Query *self, const char *field)
120
120
  }
121
121
  }
122
122
 
123
- static ulong maq_hash(Query *self)
123
+ static unsigned long maq_hash(Query *self)
124
124
  {
125
125
  (void)self;
126
126
  return 0;
data/ext/q_multi_term.c CHANGED
@@ -572,10 +572,10 @@ static void multi_tq_extract_terms(Query *self, HashSet *terms)
572
572
  }
573
573
  }
574
574
 
575
- static ulong multi_tq_hash(Query *self)
575
+ static unsigned long multi_tq_hash(Query *self)
576
576
  {
577
577
  int i;
578
- ulong hash = str_hash(MTQ(self)->field);
578
+ unsigned long hash = str_hash(MTQ(self)->field);
579
579
  PriorityQueue *boosted_terms = MTQ(self)->boosted_terms;
580
580
  for (i = boosted_terms->size; i > 0; i--) {
581
581
  BoostedTerm *bt = (BoostedTerm *)boosted_terms->heap[i];
data/ext/q_parser.c CHANGED
@@ -1852,12 +1852,19 @@ static int yyerror(QParser *qp, char const *msg)
1852
1852
 
1853
1853
  static TokenStream *get_cached_ts(QParser *qp, char *field, char *text)
1854
1854
  {
1855
- TokenStream *ts = h_get(qp->ts_cache, field);
1856
- if (!ts) {
1857
- ts = a_get_ts(qp->analyzer, field, text);
1858
- h_set(qp->ts_cache, estrdup(field), ts);
1855
+ TokenStream *ts;
1856
+ if (!qp->tokenized_fields || hs_exists(qp->tokenized_fields, field)) {
1857
+ ts = h_get(qp->ts_cache, field);
1858
+ if (!ts) {
1859
+ ts = a_get_ts(qp->analyzer, field, text);
1860
+ h_set(qp->ts_cache, estrdup(field), ts);
1861
+ }
1862
+ else {
1863
+ ts->reset(ts, text);
1864
+ }
1859
1865
  }
1860
1866
  else {
1867
+ ts = qp->non_tokenizer;
1861
1868
  ts->reset(ts, text);
1862
1869
  }
1863
1870
  return ts;
@@ -2054,7 +2061,8 @@ static Query *get_wild_q(QParser *qp, char *field, char *pattern)
2054
2061
  char *p;
2055
2062
  int len = (int)strlen(pattern);
2056
2063
 
2057
- if (qp->wild_lower) {
2064
+ if (qp->wild_lower
2065
+ && (!qp->tokenized_fields || hs_exists(qp->tokenized_fields, field))) {
2058
2066
  lower_str(pattern);
2059
2067
  }
2060
2068
 
@@ -2277,15 +2285,20 @@ void qp_destroy(QParser *self)
2277
2285
  if (self->close_def_fields) {
2278
2286
  hs_destroy(self->def_fields);
2279
2287
  }
2288
+ if (self->tokenized_fields) {
2289
+ hs_destroy(self->tokenized_fields);
2290
+ }
2280
2291
  hs_destroy(self->all_fields);
2281
2292
  hs_destroy(self->fields_buf);
2282
2293
  h_destroy(self->field_cache);
2283
2294
  h_destroy(self->ts_cache);
2295
+ tk_destroy(self->non_tokenizer);
2284
2296
  a_deref(self->analyzer);
2285
2297
  free(self);
2286
2298
  }
2287
2299
 
2288
- QParser *qp_new(HashSet *all_fields, HashSet *def_fields, Analyzer *analyzer)
2300
+ QParser *qp_new(HashSet *all_fields, HashSet *def_fields,
2301
+ HashSet *tokenized_fields, Analyzer *analyzer)
2289
2302
  {
2290
2303
  int i;
2291
2304
  QParser *self = ALLOC(QParser);
@@ -2298,6 +2311,7 @@ QParser *qp_new(HashSet *all_fields, HashSet *def_fields, Analyzer *analyzer)
2298
2311
  self->def_slop = 0;
2299
2312
  self->fields_buf = hs_new_str(NULL);
2300
2313
  self->all_fields = all_fields;
2314
+ self->tokenized_fields = tokenized_fields;
2301
2315
  if (def_fields) {
2302
2316
  self->def_fields = def_fields;
2303
2317
  for (i = 0; i < self->def_fields->size; i++) {
@@ -2321,6 +2335,7 @@ QParser *qp_new(HashSet *all_fields, HashSet *def_fields, Analyzer *analyzer)
2321
2335
  self->analyzer = analyzer;
2322
2336
  self->ts_cache = h_new_str(&free, (free_ft)&ts_deref);
2323
2337
  self->buf_index = 0;
2338
+ self->non_tokenizer = non_tokenizer_new();
2324
2339
  mutex_init(&self->mutex, NULL);
2325
2340
  return self;
2326
2341
  }