ferret 0.9.6 → 0.10.0

data/ext/vector.c

@@ -1,637 +0,0 @@
-#include "index.h"
-#include "helper.h"
-#include <string.h>
-
-static char * const NULL_POS_ERROR_MSG = "Trying to write positions that are null!";
-static char * const NULL_OFFSETS_ERROR_MSG = "Trying to write offsets that are null!";
-static char * const FIELD_OPEN_ERROR_MSG = "Field is still open while writing document";
-static char * const FORMAT_VERSION_ERROR_MSG = "Invalid format version";
-
-#define TERM_ARR_START_SIZE 16
-#define FIELD_ARR_START_SIZE 8
-
-TVOffsetInfo *tvoi_create(int start, int end)
-{
-  TVOffsetInfo *tvoi = ALLOC(TVOffsetInfo);
-  tvoi->start = start;
-  tvoi->end = end;
-  return tvoi;
-}
-
-void tvoi_destroy(void *p)
-{
-  free(p);
-}
-
-TVField *tvf_create(int number, int store_positions, int store_offsets)
-{
-  TVField *tvf = ALLOC(TVField);
-  tvf->tvf_pointer = 0;
-  tvf->number = number;
-  tvf->store_positions = store_positions;
-  tvf->store_offsets = store_offsets;
-  return tvf;
-}
-
-void tvf_destroy(void *p)
-{
-  free(p);
-}
-
-TVTerm *tvt_create(char *text, int freq, int *positions, TVOffsetInfo **offsets)
-{
-  TVTerm *tvt = ALLOC(TVTerm);
-  tvt->text = text;
-  tvt->freq = freq;
-  tvt->positions = positions;
-  tvt->offsets = offsets;
-  return tvt;
-}
-
-void tvt_destroy(void *p)
-{
-  free(p);
-}
-
-
-TermVectorsWriter *tvw_open(Store *store, char *segment, FieldInfos *fis)
-{
-  char fname[SEGMENT_NAME_MAX_LENGTH];
-  size_t segment_len = strlen(segment);
-  TermVectorsWriter *tvw = ALLOC(TermVectorsWriter);
-  OutStream *os;
-
-  tvw->curr_field = NULL;
-  tvw->curr_doc_pointer = -1;
- 
-  /* Open files for TermVector storage */
-  strcpy(fname, segment);
-
-  strcpy(fname + segment_len, TVX_EXTENSION);
-  os = tvw->tvx = store->create_output(store, fname);
-  os_write_int(os, FORMAT_VERSION);
-
-  strcpy(fname + segment_len, TVD_EXTENSION);
-  os = tvw->tvd = store->create_output(store, fname);
-  os_write_int(os, FORMAT_VERSION);
-
-  strcpy(fname + segment_len, TVF_EXTENSION);
-  os = tvw->tvf = store->create_output(store, fname);
-  os_write_int(os, FORMAT_VERSION);
-
-  tvw->fis = fis;
-
-  tvw->fields = NULL;
-  tvw->fcnt = 0;
-  tvw->fsize = 0;
-  tvw->terms = NULL;
-  tvw->tcnt = 0;
-  tvw->tsize = 0;
-
-  return tvw;
-}
-
-void tvw_write_field(TermVectorsWriter *tvw)
-{
-  int i, j, start, length;
-  char *last_term_text;
-  TVOffsetInfo *tmp_offset;
-  TVTerm **terms = tvw->terms;
-  TVTerm *term;
-  OutStream *tvf = tvw->tvf;
-  int store_positions = tvw->curr_field->store_positions;
-  int store_offsets = tvw->curr_field->store_offsets;
-  uchar bits = 0x0;
-
-  /* remember where this field is written */
-  tvw->curr_field->tvf_pointer = os_pos(tvf);
-  
-  /* write the number of terms */
-  os_write_vint(tvf, tvw->tcnt);
-  
-  if (store_positions) {
-    bits |= STORE_POSITIONS_WITH_TERMVECTOR;
-  }
-
-  if (store_offsets) {
-    bits |= STORE_OFFSET_WITH_TERMVECTOR;
-  }
-
-  os_write_byte(tvf, (uchar)bits);
-  
-  last_term_text = (char *)EMPTY_STRING;
-  for (i = 0; i < tvw->tcnt; i++) {
-    term = terms[i];
-    start = hlp_string_diff(last_term_text, term->text);
-    length = (int)strlen(term->text) - start;
-    os_write_vint(tvf, start);       /* write shared prefix length */
-    os_write_vint(tvf, length);      /* write delta length */
-    os_write_chars(tvf, term->text, start, length);  /* write delta chars */
-    os_write_vint(tvf, term->freq);
-    last_term_text = term->text;
-    
-    if (store_positions) {
-      int last_pos = 0;
-      
-      if (term->positions == NULL) {
-        RAISE(IO_ERROR, NULL_POS_ERROR_MSG);
-      }
-
-      /* use delta encoding for positions */
-      for (j = 0; j < term->freq; j++) {
-        os_write_vint(tvf, term->positions[j] - last_pos);
-        last_pos = term->positions[j];
-      }
-    }
-    
-    if (store_offsets) {
-      int last_end = 0;
-
-      if (term->offsets == NULL) {
-        RAISE(IO_ERROR, NULL_OFFSETS_ERROR_MSG);
-      }
-
-      /* use delta encoding for offsets */
-      for (j = 0; j < term->freq; j++) {
-        tmp_offset = term->offsets[j];
-        os_write_vint(tvf, tmp_offset->start - last_end);
-
-        /* save the diff between the two */
-        os_write_vint(tvf, tmp_offset->end - tmp_offset->start);
-        last_end = tmp_offset->end;
-      }
-    }
-  }
-}
-
-void tvw_close_field(TermVectorsWriter *tvw)
-{
-  int i;
-  if (tvw->curr_field != NULL) {
-    /* save field and terms */
-    tvw_write_field(tvw);
-
-    if (tvw->fcnt >= tvw->fsize) {
-      tvw->fsize *=2;
-      if (tvw->fsize < FIELD_ARR_START_SIZE) {
-        tvw->fsize = FIELD_ARR_START_SIZE;
-      }
-      REALLOC_N(tvw->fields, TVField *, tvw->fsize);
-    }
-    tvw->fields[tvw->fcnt] = tvw->curr_field;
-    tvw->fcnt++;
-
-    for (i = 0; i < tvw->tcnt; i++) {
-      tvt_destroy(tvw->terms[i]);
-    }
-    tvw->tcnt = 0;
-
-    tvw->curr_field = NULL;
-  }
-}
-
-void tvw_create_field(TermVectorsWriter *tvw,
-    int field_number, int store_position, int store_offset)
-{
-  tvw_close_field(tvw);
-  tvw->curr_field = tvf_create(field_number, store_position, store_offset);
-}
-
-void tvw_open_field(TermVectorsWriter *tvw, char *field)
-{
-  FieldInfo *fi = fis_get_fi(tvw->fis, field);
-  tvw_create_field(tvw, fi->number, fi->store_pos, fi->store_offset);
-}
-
-void tvw_write_doc(TermVectorsWriter *tvw)
-{
-  OutStream *tvd = tvw->tvd;
-  int i;
-  TVField **fields = tvw->fields;
-  int last_field_pointer = 0;
-
-  if (tvw->curr_field != NULL) {
-    RAISE(STATE_ERROR, FIELD_OPEN_ERROR_MSG);
-  }
-
-  //printf("Writing doc pointer: %d\n", tvw->curr_doc_pointer);
-  /*  write document index record */
-  os_write_long(tvw->tvx, tvw->curr_doc_pointer);
-
-  //printf("Writing field count: %ld, %d, %d -> ", (long long)tvw, tvw->fcnt, os_pos(tvd));
-  /* write the number of @fields */
-  os_write_vint(tvd, tvw->fcnt);
-  
-  /* write field numbers */
-  for (i = 0; i < tvw->fcnt; i++) {
-    os_write_vint(tvd, fields[i]->number);
-  }
-
-  /* write field pointers */
-  for (i = 0; i < tvw->fcnt; i++) {
-    os_write_vint(tvd, fields[i]->tvf_pointer - last_field_pointer);
-    last_field_pointer = fields[i]->tvf_pointer;
-  }
-  //printf("%d\n", os_pos(tvw->tvd));
-}
-
-void tvw_close_doc(TermVectorsWriter *tvw)
-{
-  int i;
-  if (tvw->curr_doc_pointer >= 0) {
-    tvw_close_field(tvw);
-    tvw_write_doc(tvw);
-
-    for (i = 0; i < tvw->fcnt; i++) {
-      tvf_destroy(tvw->fields[i]);
-    }
-    tvw->fcnt = 0;
-    tvw->curr_doc_pointer = -1;
-  }
-}
-
-void tvw_open_doc(TermVectorsWriter *tvw)
-{
-  tvw_close_doc(tvw);
-  tvw->curr_doc_pointer = os_pos(tvw->tvd);
-}
-
-void tvw_add_term(TermVectorsWriter *tvw,
-    char *text, int freq, int *positions, TVOffsetInfo **offsets) 
-{
-  if (tvw->tcnt >= tvw->tsize) {
-    tvw->tsize *= 2;
-    if (tvw->tsize < TERM_ARR_START_SIZE) {
-      tvw->tsize = TERM_ARR_START_SIZE;
-    }
-
-    REALLOC_N(tvw->terms, TVTerm *, tvw->tsize);
-  }
-  tvw->terms[tvw->tcnt] = tvt_create(text, freq, positions, offsets);
-  tvw->tcnt++;
-}
-
-void tvw_add_all_doc_vectors(TermVectorsWriter *tvw, Array *vectors)
-{
-  int i, j, store_positions, store_offsets;
-  TermVector *tv;
-
-  tvw_open_doc(tvw);
-
-  for (i = 0; i < vectors->size; i++) {
-    tv = vectors->elems[i];
-
-    store_positions = (tv->tcnt > 0 && tv->positions != NULL);
-    store_offsets = (tv->tcnt > 0 && tv->offsets != NULL);
-
-    tvw_create_field(tvw, (int)fis_get_number(tvw->fis, tv->field),
-                 store_positions, store_offsets);
-
-    if (store_positions && store_offsets) {
-      for (j = 0; j < tv->tcnt; j++) {
-        tvw_add_term(tvw, tv->terms[j], tv->freqs[j], tv->positions[j], tv->offsets[j]);
-      }
-    } else if (store_positions) {
-      for (j = 0; j < tv->tcnt; j++) {
-        tvw_add_term(tvw, tv->terms[j], tv->freqs[j], tv->positions[j], NULL);
-      }
-    } else if (store_offsets) {
-      for (j = 0; j < tv->tcnt; j++) {
-        tvw_add_term(tvw, tv->terms[j], tv->freqs[j], NULL, tv->offsets[j]);
-      }
-    } else {
-      for (j = 0; j < tv->tcnt; j++) {
-        tvw_add_term(tvw, tv->terms[j], tv->freqs[j], NULL, NULL);
-      }
-    }
-    tvw_close_field(tvw);
-  }
-  tvw_close_doc(tvw);
-}
-
-void tvw_close(TermVectorsWriter *tvw)
-{
-  /* put everything in the finally block so that even if they throw an
-   * exception, everything else will also be closed.  */
-  TRY
-  XFINALLY
-    tvw_close_doc(tvw);
-    os_close(tvw->tvx);
-    os_close(tvw->tvd);
-    os_close(tvw->tvf);
-  XENDTRY
-  free(tvw->terms);
-  free(tvw->fields);
-  free(tvw);
-}
-
-TermVector *tv_create(
-  const char *field,
-  char **terms,
-  int tcnt,
-  int *freqs,
-  int **positions,
-  TVOffsetInfo ***offsets)
-{
-  TermVector *tv =
-    ALLOC(TermVector);
-  tv->field = (char *)field;
-  tv->terms = terms;
-  tv->tcnt = tcnt;
-  tv->freqs = freqs;
-  tv->positions = positions;
-  tv->offsets = offsets;
-  return tv;
-}
-
-void tv_destroy(TermVector *tv)
-{
-  int i, j;
-  for (i = 0; i < tv->tcnt; i++) {
-    free(tv->terms[i]);
-  }
-  free(tv->terms);
-  if (tv->positions != NULL) {
-    for (i = 0; i < tv->tcnt; i++) {
-      free(tv->positions[i]);
-    }
-    free(tv->positions);
-  }
-  if (tv->offsets != NULL) {
-    for (i = 0; i < tv->tcnt; i++) {
-      for (j = 0; j < tv->freqs[i]; j++) {
-        tvoi_destroy(tv->offsets[i][j]);
-      }
-      free(tv->offsets[i]);
-    }
-    free(tv->offsets);
-  }
-  free(tv->freqs);
-  free(tv);
-}
-
-void tv_destroy_except_data(TermVector *tv)
-{
-  free(tv->terms);
-  if (tv->positions != NULL) {
-    free(tv->positions);
-  }
-  if (tv->offsets != NULL) {
-    free(tv->offsets);
-  }
-  free(tv->freqs);
-  free(tv);
-}
-
-int tvr_check_valid_format(InStream *is)
-{
-  int format = is_read_int(is);
-  if (format > FORMAT_VERSION)
-    RAISE(EXCEPTION, FORMAT_VERSION_ERROR_MSG);
-  return format;
-}
-
-TermVectorsReader *tvr_clone(TermVectorsReader *orig)
-{
-  TermVectorsReader *clone = NULL;
-  clone = ALLOC(TermVectorsReader);
-  memcpy(clone, orig, sizeof(TermVectorsReader));
-  if (orig->tvx && orig->tvd && orig->tvf) { 
-    clone->tvx = is_clone(orig->tvx);
-    clone->tvd = is_clone(orig->tvd);
-    clone->tvf = is_clone(orig->tvf);
-  }
-  return clone;
-}
-
-TermVectorsReader *tvr_open(Store *store, char *segment, FieldInfos *fis)
-{
-  TermVectorsReader *tvr = ALLOC(TermVectorsReader);
-  // Open files for TermVector storage
-  char fname[SEGMENT_NAME_MAX_LENGTH];
-  size_t segment_len = strlen(segment);
-  InStream *is;
-
-  strcpy(fname, segment);
-
-  strcpy(fname + segment_len, TVX_EXTENSION);
-  if (!store->exists(store, fname)) {
-    tvr->tvx = tvr->tvd = tvr->tvf = NULL;
-    tvr->size = 0;
-  } else {
-    is = tvr->tvx = store->open_input(store, fname);
-    tvr_check_valid_format(is);
-    tvr->size = is_length(is)/8;
-
-    strcpy(fname + segment_len, TVD_EXTENSION);
-    is = tvr->tvd = store->open_input(store, fname);
-    tvr->tvd_format = tvr_check_valid_format(is);
-
-    strcpy(fname + segment_len, TVF_EXTENSION);
-    is = tvr->tvf = store->open_input(store, fname);
-    tvr->tvf_format = tvr_check_valid_format(is);
-
-    tvr->fis = fis;
-  }
-  return tvr;
-}
-
-void tvr_close(TermVectorsReader *tvr)
-{
-  /* put everything in the finally block so that even if they throw an
-   * exception, everything else will also be closed.  */
-  TRY
-  XFINALLY
-    if (tvr->tvx) {
-      is_close(tvr->tvx);
-    }
-    if (tvr->tvd) {
-      is_close(tvr->tvd);
-    }
-    if (tvr->tvf) {
-      is_close(tvr->tvf);
-    }
-    free(tvr);
-  XENDTRY
-}
-
-TermVector *tvr_read_term_vector(TermVectorsReader *tvr,
-    char *field, int tvf_pointer)
-{
-  int i, j, store_positions, store_offsets, bits, num_terms;
-  char **terms;
-  int *term_freqs;
-  
-  /*  we may not need these, but declare them */
-  int **positions = NULL; 
-  TVOffsetInfo ***offsets = NULL;
-  int start, delta_length, total_length, freq, prev_pos;
-  int start_offset, end_offset, prev_offset;
-  int *pos;
-  TVOffsetInfo **offs;
-  char buffer[MAX_WORD_SIZE] = "";
-  
-  /* Now read the data from specified position. We don't need to offset
-   * offset by the FORMAT here since the pointer already includes the offset */
-  is_seek(tvr->tvf, tvf_pointer);
-  num_terms = (int)is_read_vint(tvr->tvf);
-  
-  /* If no terms - return a constant empty termvector. However, this should
-   * never occur! */
-  if (num_terms == 0) {
-    return tv_create(field, NULL, 0, NULL, NULL, NULL);
-  }
-  
-  if(tvr->tvf_format == FORMAT_VERSION) {
-    bits = is_read_byte(tvr->tvf);
-    store_positions = ((bits & STORE_POSITIONS_WITH_TERMVECTOR) != 0);
-    store_offsets = ((bits & STORE_OFFSET_WITH_TERMVECTOR) != 0);
-  } else {
-    is_read_vint(tvr->tvf);
-    store_positions = false;
-    store_offsets = false;
-  }
-
-  terms = ALLOC_N(char *, num_terms);
-  term_freqs = ALLOC_N(int, num_terms);
-
-  if (store_positions) {
-    positions = ALLOC_N(int *, num_terms);
-  }
-
-  if (store_offsets) {
-    offsets = ALLOC_N(TVOffsetInfo **, num_terms);
-  }
-  
-  
-  for (i = 0; i < num_terms; i++) {
-    start = (int)is_read_vint(tvr->tvf);
-    delta_length = (int)is_read_vint(tvr->tvf);
-    total_length = start + delta_length;
-    is_read_chars(tvr->tvf, buffer, start, delta_length);
-    buffer[total_length] = '\0';
-    terms[i] = estrdup(buffer);
-    freq = (int)is_read_vint(tvr->tvf);
-    term_freqs[i] = freq;
-    
-    if (store_positions) {/* read in the positions */
-      pos = ALLOC_N(int, freq);
-      positions[i] = pos;
-      prev_pos = 0;
-      for (j = 0; j < freq; j++) {
-        pos[j] = prev_pos + (int)is_read_vint(tvr->tvf);
-        prev_pos = pos[j];
-      }
-    }
-    
-    if (store_offsets) {
-      offs = ALLOC_N(TVOffsetInfo *, freq);
-      offsets[i] = offs;
-      prev_offset = 0;
-      for (j = 0; j < freq; j++) {
-        start_offset = prev_offset + (int)is_read_vint(tvr->tvf);
-        end_offset = start_offset + (int)is_read_vint(tvr->tvf);
-        offs[j] = tvoi_create(start_offset, end_offset);
-        prev_offset = end_offset;
-      }
-    }
-  }
-  return tv_create(field, terms, num_terms, term_freqs, positions, offsets);
-}
-
-Array *tvr_get_tv(TermVectorsReader *tvr, int doc_num)
-{
-  int i;
-  Array *tvs = NULL;
-  /* Check if no term vectors are available for this segment at all */
-  if (tvr->tvx != NULL) {
-    int position, field_count;
-    /* We need to offset by */
-    is_seek(tvr->tvx, (doc_num * 8) + FORMAT_SIZE);
-
-    position = (int)is_read_long(tvr->tvx);
-
-    is_seek(tvr->tvd, position);
-    field_count = (int)is_read_vint(tvr->tvd);
-
-    /* No fields are vectorized for this document */
-    if (field_count > 0) {
-      int number = 0;
-      int position = 0;
-      int *tvf_pointers = ALLOC_N(int, field_count);
-      char **fields = ALLOC_N(char *, field_count);
-      
-      for (i = 0; i < field_count; i++) {
-        if (tvr->tvd_format == FORMAT_VERSION) {
-          number = (int)is_read_vint(tvr->tvd);
-        } else {
-          number += (int)is_read_vint(tvr->tvd);
-        }
-
-        fields[i] = tvr->fis->by_number[number]->name;
-      }
-
-      /* Compute position in the tvf file */
-      for (i = 0; i < field_count; i++) {
-        position += (int)is_read_vint(tvr->tvd);
-        tvf_pointers[i] = position;
-      }
-
-      tvs = ary_create(field_count, (free_ft)&tv_destroy);
-      for (i = 0; i < field_count; i++) {
-        ary_append(tvs, tvr_read_term_vector(tvr, fields[i], tvf_pointers[i]));
-      }
-      free(fields);
-      free(tvf_pointers);
-    }
-  }
-  return tvs;
-}
-
-TermVector *tvr_get_field_tv(TermVectorsReader *tvr, int doc_num, char *field)
-{
-  int i;
-  /* Check if no term vectors are available for this segment at all */
-  int field_number = (int)fis_get_number(tvr->fis, field);
-  TermVector *tv = NULL;
-
-  if (tvr->tvx != NULL) {
-    int pos, field_count, number = 0, found = -1;
-    /* We need to account for the FORMAT_SIZE at when seeking in the @tvx
-     * We don't need to do this in other seeks because we already have the
-     * file pointer that was written in another file */
-    is_seek(tvr->tvx, (doc_num * 8) + FORMAT_SIZE);
-    //printf("TVX Pointer: %d\n", is_pos(tvr->tvx));
-    pos = (int)is_read_long(tvr->tvx);
-
-    is_seek(tvr->tvd, pos);
-    field_count = (int)is_read_vint(tvr->tvd);
-    //printf("Num Fields: %d\n", field_count);
-    /* There are only a few fields per document. We opt for a full scan
-     * rather then requiring that they be ordered. We need to read through
-     * all of the fields anyway to get to the tvf pointers. */
-    for (i = 0; i < field_count; i++) {
-      if (tvr->tvd_format == FORMAT_VERSION) {
-        number = (int)is_read_vint(tvr->tvd);
-      } else {
-        number += (int)is_read_vint(tvr->tvd);
-      }
-
-      if (number == field_number) {
-        found = i;
-      }
-    }
-
-    /* This field, although valid in the segment, was not found in this
-     * document */
-    if (found != -1) {
-      /* Compute pos in the tvf file */
-      pos = 0;
-      for (i = 0; i <= found; i++) {
-        pos += (int)is_read_vint(tvr->tvd);
-      }
-      tv = tvr_read_term_vector(tvr, field, pos);
-    }
-  }
-  return tv;
-}