ferret 0.9.6 → 0.10.0

data/ext/termdocs.c

@@ -1,611 +0,0 @@
-#include "index.h"
-#include <string.h>
-
-static char * const TPE_VS_TDE_ERROR_MSG = "TermPosEnum does not handle processing multiple documents in one call. Use TermDocEnum instead.";
-
-/****************************************************************************
- *
- * SegmentTermDocEnum
- *
- ****************************************************************************/
-
-
-void stde_close(TermDocEnum *tde)
-{
-  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
-
-  is_close(stde->freq_in);
-  if (stde->skip_in != NULL)
-    is_close(stde->skip_in);
-
-  free(stde);
-  free(tde);
-}
-
-void stde_seek_ti(TermDocEnum *tde, TermInfo *ti)
-{
-  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
-  if (ti == NULL) {
-    stde->doc_freq = 0;
-  } else {
-    stde->count = 0;
-    stde->doc_freq = ti->doc_freq;
-    stde->doc_num = 0;
-    stde->skip_doc = 0;
-    stde->skip_count = 0;
-    stde->num_skips = stde->doc_freq / stde->skip_interval;
-    stde->freq_pointer = ti->freq_pointer;
-    stde->prox_pointer = ti->prox_pointer;
-    stde->skip_pointer = ti->freq_pointer + ti->skip_offset;
-    is_seek(stde->freq_in, ti->freq_pointer);
-    stde->have_skipped = false;
-  }
-}
-
-void stde_seek(TermDocEnum *tde, Term *term)
-{
-  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
-  TermInfo *ti = tir_get_ti(stde->parent->tir, term);
-  stde_seek_ti(tde, ti);
-  ti_destroy(ti);
-}
-
-void stde_skip_prox(SegmentTermDocEnum *stde) { }
-void stde_seek_prox(SegmentTermDocEnum *stde, int prox_pointer) { }
-
-bool stde_next(TermDocEnum *tde)
-{
-  int doc_code;
-  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
-  while (true) { 
-
-    if (stde->count >= stde->doc_freq) {
-      return false;
-    }
-
-    doc_code = (int)is_read_vint(stde->freq_in);
-    stde->doc_num += doc_code >> 1;    /* shift off low bit */
-    if ((doc_code & 1) != 0) {         /* if low bit is set */
-      stde->freq = 1;                  /* freq is one */
-    } else {
-      stde->freq = (int)is_read_vint(stde->freq_in); /* read freq */
-    }
-
-    stde->count++;
-
-    if (stde->deleted_docs == NULL ||
-        bv_get(stde->deleted_docs, stde->doc_num) == 0) {
-      break; /* We found an undeleted doc so return */
-    }
-      
-    stde->skip_prox(stde);
-  }
-  return true;
-}
-
-int stde_doc_num(TermDocEnum *tde)
-{ return ((SegmentTermDocEnum *)tde->data)->doc_num; }
-
-int stde_freq(TermDocEnum *tde)
-{ return ((SegmentTermDocEnum *)tde->data)->freq; }
-
-bool stde_skip_to(TermDocEnum *tde, int target_doc_num)
-{
-  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
-
-  if (stde->doc_freq >= stde->skip_interval) { /* optimized case */
-    int last_skip_doc;
-    int last_freq_pointer;
-    int last_prox_pointer;
-    int num_skipped;
-
-    if (stde->skip_in == NULL) {
-      stde->skip_in = is_clone(stde->freq_in); /* lazily clone */
-    }
-
-    if (!stde->have_skipped) {                 /* lazily seek skip stream */
-      is_seek(stde->skip_in, stde->skip_pointer);
-      stde->have_skipped = true;
-    }
-
-    /* scan skip data */
-    last_skip_doc = stde->skip_doc;
-    last_freq_pointer = is_pos(stde->freq_in);
-    last_prox_pointer = -1;
-    num_skipped = -1 - (stde->count % stde->skip_interval);
-
-    while (target_doc_num > stde->skip_doc) {
-      last_skip_doc = stde->skip_doc;
-      last_freq_pointer = stde->freq_pointer;
-      last_prox_pointer = stde->prox_pointer;
-
-      if (stde->skip_doc != 0 && stde->skip_doc >= stde->doc_num) {
-        num_skipped += stde->skip_interval;
-      }
-      
-      if(stde->skip_count >= stde->num_skips) {
-        break;
-      }
-
-      stde->skip_doc += (int)is_read_vint(stde->skip_in);
-      stde->freq_pointer += (int)is_read_vint(stde->skip_in);
-      stde->prox_pointer += (int)is_read_vint(stde->skip_in);
-
-      stde->skip_count++;
-    }
-
-    /* if we found something to skip, so skip it */
-    if (last_freq_pointer > is_pos(stde->freq_in)) {
-      is_seek(stde->freq_in, last_freq_pointer);
-      stde->seek_prox(stde, last_prox_pointer);
-
-      stde->doc_num = last_skip_doc;
-      stde->count += num_skipped;
-    }
-  }
-
-  /* done skipping, now just scan */
-  do { 
-    if (! tde->next(tde)) {
-      return false;
-    }
-  } while (target_doc_num > ((SegmentTermDocEnum *)tde->data)->doc_num);
-  return true;
-}
-
-int stde_read(TermDocEnum *tde, int *docs, int *freqs, int req_num)
-{
-  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
-  int i = 0, doc_code;
-  while (i < req_num && stde->count < stde->doc_freq) {
-    /* manually inlined call to next() for speed */
-    doc_code = (int)is_read_vint(stde->freq_in);
-    stde->doc_num += (doc_code >> 1);            /* shift off low bit */
-    if ((doc_code & 1) != 0) {                   /* if low bit is set */
-      stde->freq = 1;                            /* freq is one */
-    } else {
-      stde->freq = (int)is_read_vint(stde->freq_in);  /* else read freq */
-    }
-
-    stde->count++;
-
-    if (stde->deleted_docs == NULL ||
-        bv_get(stde->deleted_docs, stde->doc_num) == 0) {
-      docs[i] = stde->doc_num;
-      freqs[i] = stde->freq;
-      i++;
-    }
-  }
-  return i;
-}
-
-TermDocEnum *stde_create(IndexReader *ir)
-{
-  SegmentTermDocEnum *stde = ALLOC_AND_ZERO(SegmentTermDocEnum);
-  SegmentReader *sr = (SegmentReader *)ir->data;
-  TermDocEnum *tde = ALLOC(TermDocEnum);
-  tde->seek = &stde_seek;
-  tde->doc_num = &stde_doc_num;
-  tde->freq = &stde_freq;
-  tde->next = &stde_next;
-  tde->read = &stde_read;
-  tde->skip_to = &stde_skip_to;
-  tde->next_position = NULL;
-  tde->close = &stde_close;
-
-  tde->data = stde;
-  stde->parent = sr;
-  stde->freq_in = is_clone(sr->freq_in);
-  stde->deleted_docs = sr->deleted_docs;
-  stde->skip_interval = sr->tir->skip_interval;
-  stde->skip_in = NULL;
-  stde->have_skipped = false;
-  stde->skip_prox = &stde_skip_prox;
-  stde->seek_prox = &stde_seek_prox;
-  return tde;
-}
-
-/****************************************************************************
- *
- * SegmentTermPosEnum
- *
- ****************************************************************************/
-
-void stpe_seek(TermDocEnum *tde, Term *term)
-{
-  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
-  TermInfo *ti = tir_get_ti(stde->parent->tir, term);
-  stde_seek_ti(tde, ti);
-  if (ti != NULL) {
-    is_seek(stde->prox_in, ti->prox_pointer);
-  }
-  stde->prox_cnt = 0;
-  ti_destroy(ti);
-}
-
-void stpe_close(TermDocEnum *tde)
-{
-  // super
-  is_close(((SegmentTermDocEnum *)tde->data)->prox_in);
-  ((SegmentTermDocEnum *)tde->data)->prox_in = NULL;
-  stde_close(tde);
-}
-
-void stpe_skip_prox(SegmentTermDocEnum *stde)
-{
-  is_skip_vints(stde->prox_in, stde->freq);
-}
-
-void stpe_seek_prox(SegmentTermDocEnum *stde, int prox_pointer)
-{
-  is_seek(stde->prox_in, prox_pointer);
-  stde->prox_cnt = 0;
-}
-
-bool stpe_next(TermDocEnum *tde)
-{
-  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
-  is_skip_vints(stde->prox_in, stde->prox_cnt);
-
-  /* if super */
-  if (stde_next(tde)) {
-    stde->prox_cnt = stde->freq;
-    stde->position = 0;
-    return true;
-  }
-  return false;
-}
-
-int stpe_read(TermDocEnum *tde, int *docs, int *freqs, int req_num)
-{
-  RAISE(ARG_ERROR, TPE_VS_TDE_ERROR_MSG);
-  return -1;
-}
-
-int stpe_next_position(TermDocEnum *tde)
-{
-  SegmentTermDocEnum *stde = (SegmentTermDocEnum *)tde->data;
-  stde->prox_cnt--;
-  return stde->position += (int)is_read_vint(stde->prox_in);
-}
-
-TermDocEnum *stpe_create(IndexReader *ir)
-{
-  SegmentTermDocEnum *stde;
-  SegmentReader *sr = (SegmentReader *)ir->data;
-  TermDocEnum *tde = stde_create(ir);
-  tde->close = &stpe_close;
-  tde->seek = &stpe_seek;
-  tde->next = &stpe_next;
-  tde->read = &stpe_read;
-  tde->next_position = &stpe_next_position;
-
-  stde = (SegmentTermDocEnum *)tde->data;
-  stde->prox_in = is_clone(sr->prox_in);
-  stde->prox_cnt = 0;
-  stde->position = 0;
-  stde->skip_prox = &stpe_skip_prox;
-  stde->seek_prox = &stpe_seek_prox;
-
-  return tde;
-}
-
-/****************************************************************************
- *
- * MultiTermDocEnum
- *
- ****************************************************************************/
-
-void mtde_close(TermDocEnum *tde)
-{
-  MultiTermDocEnum *mtde = (MultiTermDocEnum *)tde->data;
-  TermDocEnum *tmp_tde;
-  int i;
-  for (i = 0; i < mtde->ir_cnt; i++) {
-    if ((tmp_tde = mtde->irs_tde[i]) != NULL)
-      tmp_tde->close(tmp_tde);
-  }
-  if (mtde->term != NULL) term_destroy(mtde->term);
-  free(mtde->irs_tde);
-  free(mtde);
-  free(tde);
-}
-
-void mtde_seek(TermDocEnum *tde, Term *term)
-{
-  MultiTermDocEnum *mtde = (MultiTermDocEnum *)tde->data;
-  if (mtde->term != NULL) term_destroy(mtde->term);
-  mtde->term = term_create(term->field, term->text);
-  mtde->base = 0;
-  mtde->pointer = 0;
-  mtde->curr_tde = NULL;
-}
-
-TermDocEnum *mtde_term_docs_from_reader(IndexReader *ir)
-{
-  return ir->term_docs(ir);
-}
-
-TermDocEnum *mtde_term_docs(MultiTermDocEnum *mtde, int i)
-{
-  if (mtde->term == NULL) {
-    return NULL;
-  } else {
-
-    TermDocEnum *tde = mtde->irs_tde[i];
-    if (tde == NULL) {
-      tde = mtde->irs_tde[i] = mtde->term_docs_from_reader(mtde->irs[i]);
-    }
-
-    tde->seek(tde, mtde->term);
-    return tde;
-  }
-}
-
-bool mtde_next(TermDocEnum *tde)
-{
-  MultiTermDocEnum *mtde = (MultiTermDocEnum *)tde->data;
-  if (mtde->curr_tde != NULL && mtde->curr_tde->next(mtde->curr_tde)) {
-    return true;
-  } else if (mtde->pointer < mtde->ir_cnt) {
-    mtde->base = mtde->starts[mtde->pointer];
-    mtde->curr_tde = mtde_term_docs(mtde, mtde->pointer);
-    mtde->pointer++;
-    return mtde_next(tde);
-  } else {
-    return false;
-  }
-}
-
-int mtde_doc_num(TermDocEnum *tde)
-{
-  MultiTermDocEnum *mtde = (MultiTermDocEnum *)tde->data;
-  return mtde->base + mtde->curr_tde->doc_num(mtde->curr_tde);
-}
-
-int mtde_freq(TermDocEnum *tde)
-{
-  MultiTermDocEnum *mtde = (MultiTermDocEnum *)tde->data;
-  return mtde->curr_tde->freq(mtde->curr_tde);
-}
-
-bool mtde_skip_to(TermDocEnum *tde, int target_doc_num)
-{
-  MultiTermDocEnum *mtde = (MultiTermDocEnum *)tde->data;
-  while (mtde->pointer < mtde->ir_cnt) {
-    if ((target_doc_num < mtde->starts[mtde->pointer]) &&
-       (mtde->curr_tde->skip_to(mtde->curr_tde, target_doc_num - mtde->base))) {
-      return true;
-    }
-
-    mtde->base = mtde->starts[mtde->pointer];
-    mtde->curr_tde = mtde_term_docs(mtde, mtde->pointer);
-    mtde->pointer++;
-  }
-  if (mtde->curr_tde) {
-    return mtde->curr_tde->skip_to(mtde->curr_tde, target_doc_num - mtde->base);
-  } else {
-    return false;
-  }
-}
-
-int mtde_read(TermDocEnum *tde, int *docs, int *freqs, int req_num)
-{
-  int i, end = 0, last_end = 0, b;
-  MultiTermDocEnum *mtde = (MultiTermDocEnum *)tde->data;
-  while (true) {
-    while (mtde->curr_tde == NULL) {
-      if (mtde->pointer < mtde->ir_cnt) {      // try next segment
-        mtde->base = mtde->starts[mtde->pointer];
-        mtde->curr_tde = mtde_term_docs(mtde, mtde->pointer++);
-      } else {
-        return end;
-      }
-    }
-    end += mtde->curr_tde->read(mtde->curr_tde,
-        &docs[last_end], &freqs[last_end], req_num - last_end);
-    if (end == last_end) {          // none left in segment
-      mtde->curr_tde = NULL;
-    } else {            // got some
-      b = mtde->base;        // adjust doc numbers
-      for (i = last_end; i < end; i++)
-        docs[i] += b;
-      if (end == req_num)
-        return end;
-      else
-        last_end = end;
-    }
-  }
-}
-
-TermDocEnum *mtde_create(IndexReader **irs, int *starts, int ir_cnt)
-{
-  MultiTermDocEnum *mtde = ALLOC_AND_ZERO(MultiTermDocEnum);
-  TermDocEnum *tde = ALLOC(TermDocEnum);
-  tde->close = &mtde_close;
-  tde->seek = &mtde_seek;
-  tde->next = &mtde_next;
-  tde->doc_num = &mtde_doc_num;
-  tde->freq = &mtde_freq;
-  tde->skip_to = &mtde_skip_to;
-  tde->read = &mtde_read;
-  tde->next_position = NULL;
-
-  tde->data = mtde;
-  mtde->irs = irs;
-  mtde->starts = starts;
-  mtde->ir_cnt = ir_cnt;
-  mtde->irs_tde = ALLOC_N(TermDocEnum *, ir_cnt);
-  ZEROSET(mtde->irs_tde, TermDocEnum *, ir_cnt);
-  mtde->term_docs_from_reader = &mtde_term_docs_from_reader;
-  
-  return tde;
-}
-
-/****************************************************************************
- *
- * MultiTermPosEnum
- *
- ****************************************************************************/
-
-TermDocEnum *mtpe_term_docs_from_reader(IndexReader *ir)
-{
-  return ir->term_positions(ir);
-}
-
-
-int mtpe_next_position(TermDocEnum *tde)
-{
-  TermDocEnum *curr_tde = ((MultiTermDocEnum *)tde->data)->curr_tde;
-  return curr_tde->next_position(curr_tde);
-}
-
-TermDocEnum *mtpe_create(IndexReader **irs, int *starts, int ir_cnt)
-{
-  TermDocEnum *tde = mtde_create(irs, starts, ir_cnt);
-  tde->next_position = &mtpe_next_position;
-  ((MultiTermDocEnum *)tde->data)->term_docs_from_reader = &mtpe_term_docs_from_reader;
-  return tde;
-}
-
-/****************************************************************************
- *
- * MultipleTermDocPosEnum
- *
- ****************************************************************************/
-
-#define GET_MTDPE MultipleTermDocPosEnum *mtdpe = (MultipleTermDocPosEnum *)self->data
-void tde_destroy(TermDocEnum *self) {
-  self->close(self);
-}
-
-void mtdpe_close(TermDocEnum *self)
-{
-  GET_MTDPE;
-
-  pq_clear(mtdpe->pq);
-  pq_destroy(mtdpe->pq);
-  free(mtdpe->pos_queue);
-  free(mtdpe);
-  free(self);
-}
-
-void mtdpe_seek(TermDocEnum *tde, Term *term)
-{ RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);}
-
-bool mtdpe_next(TermDocEnum *self)
-{
-  TermDocEnum *tde;
-  int i = 0, freq = 0;
-  int doc;
-  GET_MTDPE;
-
-  if (mtdpe->pq->count == 0) return false;
-
-  tde = (TermDocEnum *)pq_top(mtdpe->pq);
-  doc = tde->doc_num(tde);
-
-  do {
-    freq += tde->freq(tde);
-    if (freq > mtdpe->pos_queue_capa) {
-      mtdpe->pos_queue_capa *= 2;
-      REALLOC_N(mtdpe->pos_queue, int, mtdpe->pos_queue_capa);
-    }
-
-    for (; i < freq; i++) {
-      mtdpe->pos_queue[i] = tde->next_position(tde);
-    }
-
-    if (tde->next(tde)) {
-      pq_down(mtdpe->pq);
-    } else {
-      tde = pq_pop(mtdpe->pq);
-      tde->close(tde);
-    }
-    tde = (TermDocEnum *)pq_top(mtdpe->pq);
-  } while ((mtdpe->pq->count > 0) && (tde->doc_num(tde) == doc));
-
-  qsort(mtdpe->pos_queue, freq, sizeof(int), &icmp_risky);
-
-  mtdpe->pos_queue_index = 0;
-  mtdpe->freq = freq;
-  mtdpe->doc_num = doc;
-
-  return true;
-}
-
-int mtdpe_doc_num(TermDocEnum *self)
-{ return ((MultipleTermDocPosEnum *)self->data)->doc_num; }
-
-int mtdpe_freq(TermDocEnum *self)
-{ return ((MultipleTermDocPosEnum *)self->data)->freq; }
-
-
-bool tdpe_less_than(void *p1, void *p2)
-{
-  return ((TermDocEnum *)p1)->doc_num((TermDocEnum *)p1) <
-    ((TermDocEnum *)p2)->doc_num((TermDocEnum *)p2);
-}
-
-bool mtdpe_skip_to(TermDocEnum *self, int target_doc_num)
-{
-  GET_MTDPE;
-  TermDocEnum *tde;
-  while ((tde = pq_top(mtdpe->pq)) != NULL &&
-      (target_doc_num > tde->doc_num(tde))) {
-    if (tde->skip_to(tde, target_doc_num)) {
-      pq_down(mtdpe->pq);
-    } else {
-      tde = pq_pop(mtdpe->pq);
-      tde->close(tde);
-    }
-  }
-  return self->next(self);
-}
-
-int mtdpe_read(TermDocEnum *tde, int *docs, int *freqs, int req_num)
-{
-  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
-  return -1;
-}
-
-int mtdpe_next_position(TermDocEnum *self)
-{
-  GET_MTDPE;
-  return mtdpe->pos_queue[mtdpe->pos_queue_index++];
-}
-
-TermDocEnum *mtdpe_create(IndexReader *ir, Term **terms, int t_cnt)
-{
-  int i;
-  TermDocEnum *self = ALLOC(TermDocEnum);
-  MultipleTermDocPosEnum *mtdpe = ALLOC_AND_ZERO_N(MultipleTermDocPosEnum, 1);
-  PriorityQueue *pq;
-  TermDocEnum *tpe;
-
-  pq = mtdpe->pq = pq_create(t_cnt, &tdpe_less_than);
-  mtdpe->pos_queue_capa = MTDPE_POS_QUEUE_INIT_CAPA;
-  mtdpe->pos_queue = ALLOC_N(int, MTDPE_POS_QUEUE_INIT_CAPA);
-  for (i = 0; i < t_cnt; i++) {
-    tpe = ir_term_positions_for(ir, terms[i]);
-    if (tpe->next(tpe)) {
-      pq_push(pq, tpe);
-    } else {
-      tpe->close(tpe);
-    }
-  }
-  pq->free_elem = (free_ft)&tde_destroy;
-
-  self->data = mtdpe;
-  self->close = &mtdpe_close;
-  self->seek = &mtdpe_seek;
-  self->next = &mtdpe_next;
-  self->doc_num = &mtdpe_doc_num;
-  self->freq = &mtdpe_freq;
-  self->skip_to = &mtdpe_skip_to;
-  self->read = &mtdpe_read;
-  self->next_position = &mtdpe_next_position;
-
-  return self;
-}
-