ferret 0.3.2 → 0.9.0

data/ext/termdocs.c

@@ -0,0 +1,599 @@
+#include <index.h>
+#include <string.h>
+
+/****************************************************************************
+ *
+ * SegmentTermDocEnum
+ *
+ ****************************************************************************/
+
+
+void stde_close(TermDocEnum *tde)
+{
+  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
+
+  is_close(stde->freq_in);
+  if (stde->skip_in != NULL)
+    is_close(stde->skip_in);
+
+  free(stde);
+  free(tde);
+}
+
+void stde_seek_ti(TermDocEnum *tde, TermInfo *ti)
+{
+  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
+  if (ti == NULL) {
+    stde->doc_freq = 0;
+  } else {
+    stde->count = 0;
+    stde->doc_freq = ti->doc_freq;
+    stde->doc_num = 0;
+    stde->skip_doc = 0;
+    stde->skip_count = 0;
+    stde->num_skips = stde->doc_freq / stde->skip_interval;
+    stde->freq_pointer = ti->freq_pointer;
+    stde->prox_pointer = ti->prox_pointer;
+    stde->skip_pointer = ti->freq_pointer + ti->skip_offset;
+    is_seek(stde->freq_in, ti->freq_pointer);
+    stde->have_skipped = false;
+  }
+}
+
+void stde_seek(TermDocEnum *tde, Term *term)
+{
+  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
+  TermInfo *ti = tir_get_ti(stde->parent->tir, term);
+  stde_seek_ti(tde, ti);
+  ti_destroy(ti);
+}
+
+void stde_skip_prox(SegmentTermDocEnum *stde) { }
+void stde_seek_prox(SegmentTermDocEnum *stde, int prox_pointer) { }
+
+bool stde_next(TermDocEnum *tde)
+{
+  int doc_code;
+  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
+  while (true) { 
+
+    if (stde->count >= stde->doc_freq)
+      return false;
+
+    doc_code = is_read_vint(stde->freq_in);
+    stde->doc_num += doc_code >> 1;    // shift off low bit
+    if ((doc_code & 1) != 0) {         // if low bit is set
+      stde->freq = 1;                  // freq is one
+    } else {
+      stde->freq = is_read_vint(stde->freq_in); // else read freq
+    }
+
+    stde->count++;
+
+    if (stde->deleted_docs == NULL ||
+        bv_get(stde->deleted_docs, stde->doc_num) == 0)
+      break; // We found an undeleted doc so return
+      
+    stde->skip_prox(stde);
+  }
+  return true;
+}
+
+int stde_doc_num(TermDocEnum *tde)
+{ return ((SegmentTermDocEnum *)tde->data)->doc_num; }
+
+int stde_freq(TermDocEnum *tde)
+{ return ((SegmentTermDocEnum *)tde->data)->freq; }
+
+bool stde_skip_to(TermDocEnum *tde, int target_doc_num)
+{
+  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
+  if (stde->doc_freq >= stde->skip_interval) { // optimized case
+
+    if (stde->skip_in == NULL)
+      stde->skip_in = is_clone(stde->freq_in); // lazily clone
+
+    if (!stde->have_skipped) {                 // lazily seek skip stream
+      is_seek(stde->skip_in, stde->skip_pointer);
+      stde->have_skipped = true;
+    }
+
+    // scan skip data
+    int last_skip_doc = stde->skip_doc;
+    int last_freq_pointer = is_pos(stde->freq_in);
+    int last_prox_pointer = -1;
+    int num_skipped = -1 - (stde->count % stde->skip_interval);
+
+    while (target_doc_num > stde->skip_doc) {
+      last_skip_doc = stde->skip_doc;
+      last_freq_pointer = stde->freq_pointer;
+      last_prox_pointer = stde->prox_pointer;
+
+      if (stde->skip_doc != 0 && stde->skip_doc >= stde->doc_num)
+        num_skipped += stde->skip_interval;
+      
+      if(stde->skip_count >= stde->num_skips)
+        break;
+
+      stde->skip_doc += is_read_vint(stde->skip_in);
+      stde->freq_pointer += is_read_vint(stde->skip_in);
+      stde->prox_pointer += is_read_vint(stde->skip_in);
+
+      stde->skip_count++;
+    }
+
+    // if we found something to skip, so skip it
+    if (last_freq_pointer > is_pos(stde->freq_in)) {
+      is_seek(stde->freq_in, last_freq_pointer);
+      stde->seek_prox(stde, last_prox_pointer);
+
+      stde->doc_num = last_skip_doc;
+      stde->count += num_skipped;
+    }
+  }
+
+  // done skipping, now just scan
+  do { 
+    if (! tde->next(tde)) {
+      return false;
+    }
+  } while (target_doc_num > ((SegmentTermDocEnum *)tde->data)->doc_num);
+  return true;
+}
+
+int stde_read(TermDocEnum *tde, int *docs, int *freqs, int req_num)
+{
+  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
+  int i = 0, doc_code;
+  while (i < req_num && stde->count < stde->doc_freq) {
+    // manually inlined call to next() for speed
+    doc_code = is_read_vint(stde->freq_in);
+    stde->doc_num += doc_code >> 1;                 // shift off low bit
+    if ((doc_code & 1) != 0)                        // if low bit is set
+      stde->freq = 1;                               // freq is one
+    else
+      stde->freq = is_read_vint(stde->freq_in);   // else read freq
+
+    stde->count++;
+
+    if (stde->deleted_docs == NULL ||
+        bv_get(stde->deleted_docs, stde->doc_num) == 0) {
+      docs[i] = stde->doc_num;
+      freqs[i] = stde->freq;
+      i++;
+    }
+  }
+  return i;
+}
+
+TermDocEnum *stde_create(IndexReader *ir)
+{
+  SegmentReader *sr = (SegmentReader *)ir->data;
+  TermDocEnum *tde = ALLOC(TermDocEnum);
+  tde->seek = &stde_seek;
+  tde->doc_num = &stde_doc_num;
+  tde->freq = &stde_freq;
+  tde->next = &stde_next;
+  tde->read = &stde_read;
+  tde->skip_to = &stde_skip_to;
+  tde->next_position = NULL;
+  tde->close = &stde_close;
+
+  SegmentTermDocEnum *stde = ALLOC(SegmentTermDocEnum);
+  ZEROSET(stde, SegmentTermDocEnum, 1); // set all values to 0
+  tde->data = stde;
+  stde->parent = sr;
+  stde->freq_in = is_clone(sr->freq_in);
+  stde->deleted_docs = sr->deleted_docs;
+  stde->skip_interval = sr->tir->skip_interval;
+  stde->skip_in = NULL;
+  stde->have_skipped = false;
+  stde->skip_prox = &stde_skip_prox;
+  stde->seek_prox = &stde_seek_prox;
+  return tde;
+}
+
+/****************************************************************************
+ *
+ * SegmentTermPosEnum
+ *
+ ****************************************************************************/
+
+void stpe_seek(TermDocEnum *tde, Term *term)
+{
+  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
+  TermInfo *ti = tir_get_ti(stde->parent->tir, term);
+  stde_seek_ti(tde, ti);
+  if (ti != NULL) {
+    is_seek(stde->prox_in, ti->prox_pointer);
+  }
+  stde->prox_cnt = 0;
+  ti_destroy(ti);
+}
+
+void stpe_close(TermDocEnum *tde)
+{
+  // super
+  is_close(((SegmentTermDocEnum *)tde->data)->prox_in);
+  ((SegmentTermDocEnum *)tde->data)->prox_in = NULL;
+  stde_close(tde);
+}
+
+void stpe_skip_prox(SegmentTermDocEnum *stde)
+{
+  int i;
+  for (i = 0; i < stde->freq; i++)
+    is_read_vint(stde->prox_in);
+}
+
+void stpe_seek_prox(SegmentTermDocEnum *stde, int prox_pointer)
+{
+  is_seek(stde->prox_in, prox_pointer);
+  stde->prox_cnt = 0;
+}
+
+bool stpe_next(TermDocEnum *tde)
+{
+  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
+  int i;
+  for (i = 0; i < stde->prox_cnt; i++)
+    is_read_vint(stde->prox_in);
+
+  // if super
+  if (stde_next(tde)) {
+    stde->prox_cnt = stde->freq;
+    stde->position = 0;
+    return true;
+  }
+  return false;
+}
+
+int stpe_read(TermDocEnum *tde, int *docs, int *freqs, int req_num)
+{
+  eprintf(ARG_ERROR, "TermPosEnum does not handle processing multiple documents in one call. Use TermDocEnum instead.");
+  return -1;
+}
+
+int stpe_next_position(TermDocEnum *tde)
+{
+  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
+  stde->prox_cnt--;
+  return stde->position += is_read_vint(stde->prox_in);
+}
+
+TermDocEnum *stpe_create(IndexReader *ir)
+{
+  SegmentReader *sr = (SegmentReader *)ir->data;
+  TermDocEnum *tde = stde_create(ir);
+  tde->close = &stpe_close;
+  tde->seek = &stpe_seek;
+  tde->next = &stpe_next;
+  tde->read = &stpe_read;
+  tde->next_position = &stpe_next_position;
+
+  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
+  stde->prox_in = is_clone(sr->prox_in);
+  stde->prox_cnt = 0;
+  stde->position = 0;
+  stde->skip_prox = &stpe_skip_prox;
+  stde->seek_prox = &stpe_seek_prox;
+
+  return tde;
+}
+
+/****************************************************************************
+ *
+ * MultiTermDocEnum
+ *
+ ****************************************************************************/
+
+void mtde_close(TermDocEnum *tde)
+{
+  MultiTermDocEnum *mtde = (MultiTermDocEnum *)tde->data;
+  TermDocEnum *tmp_tde;
+  int i;
+  for (i = 0; i < mtde->ir_cnt; i++) {
+    if ((tmp_tde = mtde->irs_tde[i]) != NULL)
+      tmp_tde->close(tmp_tde);
+  }
+  if (mtde->term != NULL) term_destroy(mtde->term);
+  free(mtde->irs_tde);
+  free(mtde);
+  free(tde);
+}
+
+void mtde_seek(TermDocEnum *tde, Term *term)
+{
+  MultiTermDocEnum *mtde = (MultiTermDocEnum *)tde->data;
+  if (mtde->term != NULL) term_destroy(mtde->term);
+  mtde->term = term_create(term->field, term->text);
+  mtde->base = 0;
+  mtde->pointer = 0;
+  mtde->curr_tde = NULL;
+}
+
+TermDocEnum *mtde_term_docs_from_reader(IndexReader *ir)
+{
+  return ir->term_docs(ir);
+}
+
+TermDocEnum *mtde_term_docs(MultiTermDocEnum *mtde, int i)
+{
+  if (mtde->term == NULL)
+    return NULL;
+
+  TermDocEnum *tde = mtde->irs_tde[i];
+  if (tde == NULL) {
+    tde = mtde->irs_tde[i] = mtde->term_docs_from_reader(mtde->irs[i]);
+  }
+
+  tde->seek(tde, mtde->term);
+  return tde;
+}
+
+bool mtde_next(TermDocEnum *tde)
+{
+  MultiTermDocEnum *mtde = (MultiTermDocEnum *)tde->data;
+  if (mtde->curr_tde != NULL && mtde->curr_tde->next(mtde->curr_tde)) {
+    return true;
+  } else if (mtde->pointer < mtde->ir_cnt) {
+    mtde->base = mtde->starts[mtde->pointer];
+    mtde->curr_tde = mtde_term_docs(mtde, mtde->pointer);
+    mtde->pointer++;
+    return mtde_next(tde);
+  } else {
+    return false;
+  }
+}
+
+int mtde_doc_num(TermDocEnum *tde)
+{
+  MultiTermDocEnum *mtde = (MultiTermDocEnum *)tde->data;
+  return mtde->base + mtde->curr_tde->doc_num(mtde->curr_tde);
+}
+
+int mtde_freq(TermDocEnum *tde)
+{
+  MultiTermDocEnum *mtde = (MultiTermDocEnum *)tde->data;
+  return mtde->curr_tde->freq(mtde->curr_tde);
+}
+
+bool mtde_skip_to(TermDocEnum *tde, int target_doc_num)
+{
+  MultiTermDocEnum *mtde = (MultiTermDocEnum *)tde->data;
+  while (mtde->pointer < mtde->ir_cnt) {
+    if ((target_doc_num < mtde->starts[mtde->pointer]) &&
+       (mtde->curr_tde->skip_to(mtde->curr_tde, target_doc_num - mtde->base))) {
+      return true;
+    }
+
+    mtde->base = mtde->starts[mtde->pointer];
+    mtde->curr_tde = mtde_term_docs(mtde, mtde->pointer);
+    mtde->pointer++;
+  }
+  if (mtde->curr_tde) {
+    return mtde->curr_tde->skip_to(mtde->curr_tde, target_doc_num - mtde->base);
+  } else {
+    return false;
+  }
+}
+
+int mtde_read(TermDocEnum *tde, int *docs, int *freqs, int req_num)
+{
+  int i, end = 0, last_end = 0, b;
+  MultiTermDocEnum *mtde = (MultiTermDocEnum *)tde->data;
+  while (true) {
+    while (mtde->curr_tde == NULL) {
+      if (mtde->pointer < mtde->ir_cnt) {      // try next segment
+        mtde->base = mtde->starts[mtde->pointer];
+        mtde->curr_tde = mtde_term_docs(mtde, mtde->pointer++);
+      } else {
+        return end;
+      }
+    }
+    end += mtde->curr_tde->read(mtde->curr_tde,
+        &docs[last_end], &freqs[last_end], req_num - last_end);
+    if (end == last_end) {          // none left in segment
+      mtde->curr_tde = NULL;
+    } else {            // got some
+      b = mtde->base;        // adjust doc numbers
+      for (i = last_end; i < end; i++)
+        docs[i] += b;
+      if (end == req_num)
+        return end;
+      else
+        last_end = end;
+    }
+  }
+}
+
+TermDocEnum *mtde_create(IndexReader **irs, int *starts, int ir_cnt)
+{
+  TermDocEnum *tde = ALLOC(TermDocEnum);
+  tde->close = &mtde_close;
+  tde->seek = &mtde_seek;
+  tde->next = &mtde_next;
+  tde->doc_num = &mtde_doc_num;
+  tde->freq = &mtde_freq;
+  tde->skip_to = &mtde_skip_to;
+  tde->read = &mtde_read;
+  tde->next_position = NULL;
+
+  MultiTermDocEnum *mtde = ALLOC(MultiTermDocEnum);
+  ZEROSET(mtde, MultiTermDocEnum, 1); // set all values to 0
+  tde->data = mtde;
+  mtde->irs = irs;
+  mtde->starts = starts;
+  mtde->ir_cnt = ir_cnt;
+  mtde->irs_tde = ALLOC_N(TermDocEnum *, ir_cnt);
+  ZEROSET(mtde->irs_tde, TermDocEnum *, ir_cnt);
+  mtde->term_docs_from_reader = &mtde_term_docs_from_reader;
+  
+  return tde;
+}
+
+/****************************************************************************
+ *
+ * MultiTermPosEnum
+ *
+ ****************************************************************************/
+
+TermDocEnum *mtpe_term_docs_from_reader(IndexReader *ir)
+{
+  return ir->term_positions(ir);
+}
+
+
+int mtpe_next_position(TermDocEnum *tde)
+{
+  TermDocEnum *curr_tde = ((MultiTermDocEnum *)tde->data)->curr_tde;
+  return curr_tde->next_position(curr_tde);
+}
+
+TermDocEnum *mtpe_create(IndexReader **irs, int *starts, int ir_cnt)
+{
+  TermDocEnum *tde = mtde_create(irs, starts, ir_cnt);
+  tde->next_position = &mtpe_next_position;
+  ((MultiTermDocEnum *)tde->data)->term_docs_from_reader = &mtpe_term_docs_from_reader;
+  return tde;
+}
+
+/****************************************************************************
+ *
+ * MultipleTermDocPosEnum
+ *
+ ****************************************************************************/
+
+#define GET_MTDPE MultipleTermDocPosEnum *mtdpe = (MultipleTermDocPosEnum *)self->data
+void tde_destroy(void *p) {
+  TermDocEnum *self = (TermDocEnum *)p;
+  self->close(self);
+}
+
+void mtdpe_close(TermDocEnum *self)
+{
+  GET_MTDPE;
+
+  pq_clear(mtdpe->pq);
+  pq_destroy(mtdpe->pq);
+  free(mtdpe->pos_queue);
+  free(mtdpe);
+  free(self);
+}
+
+void mtdpe_seek(TermDocEnum *tde, Term *term)
+{ eprintf(UNSUPPORTED_ERROR, "Unsupported op seek on MultipleTDPE");}
+
+bool mtdpe_next(TermDocEnum *self)
+{
+  TermDocEnum *tde;
+  int i = 0, freq = 0;
+  int doc;
+  GET_MTDPE;
+
+  if (mtdpe->pq->count == 0) return false;
+
+  tde = (TermDocEnum *)pq_top(mtdpe->pq);
+  doc = tde->doc_num(tde);
+
+  do {
+    freq += tde->freq(tde);
+    if (freq > mtdpe->pos_queue_capa) {
+      mtdpe->pos_queue_capa *= 2;
+      REALLOC_N(mtdpe->pos_queue, int, mtdpe->pos_queue_capa);
+    }
+
+    for (; i < freq; i++) {
+      mtdpe->pos_queue[i] = tde->next_position(tde);
+    }
+
+    if (tde->next(tde)) {
+      pq_down(mtdpe->pq);
+    } else {
+      tde = pq_pop(mtdpe->pq);
+      tde->close(tde);
+    }
+    tde = (TermDocEnum *)pq_top(mtdpe->pq);
+  } while ((mtdpe->pq->count > 0) && (tde->doc_num(tde) == doc));
+
+  qsort(mtdpe->pos_queue, freq, sizeof(int), &icmp_risky);
+
+  mtdpe->pos_queue_index = 0;
+  mtdpe->freq = freq;
+  mtdpe->doc_num = doc;
+
+  return true;
+}
+
+int mtdpe_doc_num(TermDocEnum *self)
+{ return ((MultipleTermDocPosEnum *)self->data)->doc_num; }
+
+int mtdpe_freq(TermDocEnum *self)
+{ return ((MultipleTermDocPosEnum *)self->data)->freq; }
+
+
+bool tdpe_less_than(void *p1, void *p2)
+{
+  return ((TermDocEnum *)p1)->doc_num((TermDocEnum *)p1) <
+    ((TermDocEnum *)p2)->doc_num((TermDocEnum *)p2);
+}
+
+bool mtdpe_skip_to(TermDocEnum *self, int target_doc_num)
+{
+  GET_MTDPE;
+  TermDocEnum *tde;
+  while ((tde = pq_top(mtdpe->pq)) != NULL &&
+      (target_doc_num > tde->doc_num(tde))) {
+    if (tde->skip_to(tde, target_doc_num)) {
+      pq_down(mtdpe->pq);
+    } else {
+      tde = pq_pop(mtdpe->pq);
+      tde->close(tde);
+    }
+  }
+  return self->next(self);
+}
+
+int mtdpe_read(TermDocEnum *tde, int *docs, int *freqs, int req_num)
+{
+  eprintf(UNSUPPORTED_ERROR, "Unsupported op read on MultipleTDPE");
+  return -1;
+}
+
+int mtdpe_next_position(TermDocEnum *self)
+{
+  GET_MTDPE;
+  return mtdpe->pos_queue[mtdpe->pos_queue_index++];
+}
+
+TermDocEnum *mtdpe_create(IndexReader *ir, Term **terms, int t_cnt)
+{
+  TermDocEnum *self = ALLOC(TermDocEnum);
+  MultipleTermDocPosEnum *mtdpe = ALLOC(MultipleTermDocPosEnum);
+  PriorityQueue *pq;
+  TermDocEnum *tpe;
+  int i;
+
+  self->close = &mtdpe_close;
+  self->seek = &mtdpe_seek;
+  self->next = &mtdpe_next;
+  self->doc_num = &mtdpe_doc_num;
+  self->freq = &mtdpe_freq;
+  self->skip_to = &mtdpe_skip_to;
+  self->read = &mtdpe_read;
+  self->next_position = &mtdpe_next_position;
+
+  ZEROSET(mtdpe, MultipleTermDocPosEnum, 1); // set all values to 0
+  self->data = mtdpe;
+  pq = mtdpe->pq = pq_create(t_cnt, &tdpe_less_than);
+  mtdpe->pos_queue_capa = MTDPE_POS_QUEUE_INIT_CAPA;
+  mtdpe->pos_queue = ALLOC_N(int, MTDPE_POS_QUEUE_INIT_CAPA);
+  for (i = 0; i < t_cnt; i++) {
+    tpe = ir_term_positions_for(ir, terms[i]);
+    if (tpe->next(tpe)) pq_push(pq, tpe);
+  }
+  pq->free_elem = &tde_destroy;
+
+  return self;
+}
+