ferret 0.3.2 → 0.9.0

data/ext/vector.c

@@ -0,0 +1,594 @@
+#include <index.h>
+#include <string.h>
+#include <helper.h>
+
+#define TERM_ARR_START_SIZE 16
+#define FIELD_ARR_START_SIZE 8
+
+TVOffsetInfo *tvoi_create(int start, int end)
+{
+  TVOffsetInfo *tvoi = ALLOC(TVOffsetInfo);
+  tvoi->start = start;
+  tvoi->end = end;
+  return tvoi;
+}
+
+void tvoi_destroy(void *p)
+{
+  free(p);
+}
+
+TVField *tvf_create(int number, int store_positions, int store_offsets)
+{
+  TVField *tvf = ALLOC(TVField);
+  tvf->tvf_pointer = 0;
+  tvf->number = number;
+  tvf->store_positions = store_positions;
+  tvf->store_offsets = store_offsets;
+  return tvf;
+}
+
+void tvf_destroy(void *p)
+{
+  free(p);
+}
+
+TVTerm *tvt_create(char *text, int freq, int *positions, TVOffsetInfo **offsets)
+{
+  TVTerm *tvt = ALLOC(TVTerm);
+  tvt->text = text;
+  tvt->freq = freq;
+  tvt->positions = positions;
+  tvt->offsets = offsets;
+  return tvt;
+}
+
+void tvt_destroy(void *p)
+{
+  //int i;
+  //TVTerm *tvt = (TVTerm *)p;
+  //free(tvt->text);
+  //free(tvt->positions);
+  //if (tvt->offsets != NULL) {
+  //  for (i = 0; i < tvt->freq; i++) {
+  //    tvoi_destroy(tvt->offsets[i]);
+  //  }
+  //  free(tvt->offsets);
+  //}
+  free(p);
+}
+
+
+TermVectorsWriter *tvw_open(Store *store, char *segment, FieldInfos *fis)
+{
+  TermVectorsWriter *tvw = ALLOC(TermVectorsWriter);
+  tvw->curr_field = NULL;
+  tvw->curr_doc_pointer = -1;
+ 
+  // Open files for TermVector storage
+  char fname[SEGMENT_NAME_MAX_LENGTH];
+  int segment_len = strlen(segment);
+  strcpy(fname, segment);
+
+  strcpy(fname + segment_len, TVX_EXTENSION);
+  OutStream *os = tvw->tvx = store->create_output(store, fname);
+  os_write_int(os, FORMAT_VERSION);
+
+  strcpy(fname + segment_len, TVD_EXTENSION);
+  os = tvw->tvd = store->create_output(store, fname);
+  os_write_int(os, FORMAT_VERSION);
+
+  strcpy(fname + segment_len, TVF_EXTENSION);
+  os = tvw->tvf = store->create_output(store, fname);
+  os_write_int(os, FORMAT_VERSION);
+
+  tvw->fis = fis;
+
+  tvw->fields = NULL;
+  tvw->fcnt = 0;
+  tvw->fsize = 0;
+  tvw->terms = NULL;
+  tvw->tcnt = 0;
+  tvw->tsize = 0;
+
+  return tvw;
+}
+
+void tvw_write_field(TermVectorsWriter *tvw)
+{
+  int i, j, start, length;
+  char *last_term_text;
+  TVOffsetInfo *tmp_offset;
+  // remember where this field is written
+  OutStream *tvf = tvw->tvf;
+  tvw->curr_field->tvf_pointer = os_pos(tvf);
+  
+  // write the number of terms
+  os_write_vint(tvf, tvw->tcnt);
+  
+  int store_positions = tvw->curr_field->store_positions;
+  int store_offsets = tvw->curr_field->store_offsets;
+  int bits = 0x0;
+  if (store_positions) 
+    bits |= STORE_POSITIONS_WITH_TERMVECTOR;
+
+  if (store_offsets) 
+    bits |= STORE_OFFSET_WITH_TERMVECTOR;
+
+  os_write_byte(tvf, bits);
+  
+  last_term_text = (char *)EMPTY_STRING;
+  TVTerm **terms = tvw->terms;
+  TVTerm *term;
+  for (i = 0; i < tvw->tcnt; i++) {
+    term = terms[i];
+    start = hlp_string_diff(last_term_text, term->text);
+    length = strlen(term->text) - start;
+    os_write_vint(tvf, start);       // write shared prefix length
+    os_write_vint(tvf, length);      // write delta length
+    os_write_chars(tvf, term->text, start, length);  // write delta chars
+    os_write_vint(tvf, term->freq);
+    last_term_text = term->text;
+    
+    if (store_positions) {
+      if (term->positions == NULL)
+        eprintf(IO_ERROR, "Trying to write positions that are null!");
+      
+      // use delta encoding for positions
+      int last_pos = 0;
+      for (j = 0; j < term->freq; j++) {
+        os_write_vint(tvf, term->positions[j] - last_pos);
+        last_pos = term->positions[j];
+      }
+    }
+    
+    if (store_offsets) {
+      if (term->offsets == NULL)
+        eprintf(IO_ERROR, "Trying to write offsets that are null!");
+      
+      // use delta encoding for offsets
+      int last_end = 0;
+      for (j = 0; j < term->freq; j++) {
+        tmp_offset = term->offsets[j];
+        os_write_vint(tvf, tmp_offset->start - last_end);
+
+        // save the diff between the two.
+        os_write_vint(tvf, tmp_offset->end - tmp_offset->start);
+        last_end = tmp_offset->end;
+      }
+    }
+  }
+}
+
+void tvw_close_field(TermVectorsWriter *tvw)
+{
+  int i;
+  if (tvw->curr_field != NULL) {
+    // save field and terms
+    tvw_write_field(tvw);
+
+    if (tvw->fcnt >= tvw->fsize) {
+      tvw->fsize *=2;
+      if (tvw->fsize < FIELD_ARR_START_SIZE)
+        tvw->fsize = FIELD_ARR_START_SIZE;
+      REALLOC_N(tvw->fields, TVField *, tvw->fsize);
+    }
+    tvw->fields[tvw->fcnt] = tvw->curr_field;
+    tvw->fcnt++;
+
+    for (i = 0; i < tvw->tcnt; i++) {
+      tvt_destroy(tvw->terms[i]);
+    }
+    tvw->tcnt = 0;
+
+    tvw->curr_field = NULL;
+  }
+}
+
+void tvw_create_field(TermVectorsWriter *tvw,
+    int field_number, int store_position, int store_offset)
+{
+  tvw_close_field(tvw);
+  tvw->curr_field = tvf_create(field_number, store_position, store_offset);
+}
+
+void tvw_open_field(TermVectorsWriter *tvw, char *field)
+{
+  FieldInfo *fi = fis_get_fi(tvw->fis, field);
+  tvw_create_field(tvw, fi->number, fi->store_pos, fi->store_offset);
+}
+
+void tvw_write_doc(TermVectorsWriter *tvw)
+{
+  if (tvw->curr_field != NULL)
+    eprintf(STATE_ERROR, "Field is still open while writing document");
+
+  // puts("Writing doc pointer: " + @curr_doc_pointer)
+  //  write document index record
+  os_write_long(tvw->tvx, tvw->curr_doc_pointer);
+
+  OutStream *tvd = tvw->tvd;
+  // write the number of @fields
+  os_write_vint(tvd, tvw->fcnt);
+
+  // write field numbers
+  int i;
+  TVField **fields = tvw->fields;
+  for (i = 0; i < tvw->fcnt; i++) {
+    os_write_vint(tvd, fields[i]->number);
+  }
+
+  // write field pointers
+  int last_field_pointer = 0;
+  for (i = 0; i < tvw->fcnt; i++) {
+    os_write_vint(tvd, fields[i]->tvf_pointer - last_field_pointer);
+    last_field_pointer = fields[i]->tvf_pointer;
+  }
+}
+
+void tvw_close_doc(TermVectorsWriter *tvw)
+{
+  int i;
+  if (tvw->curr_doc_pointer >= 0) {
+    tvw_close_field(tvw);
+    tvw_write_doc(tvw);
+
+    for (i = 0; i < tvw->fcnt; i++) {
+      tvf_destroy(tvw->fields[i]);
+    }
+    tvw->fcnt = 0;
+    tvw->curr_doc_pointer = -1;
+  }
+}
+
+void tvw_open_doc(TermVectorsWriter *tvw)
+{
+  tvw_close_doc(tvw);
+  tvw->curr_doc_pointer = os_pos(tvw->tvd);
+}
+
+void tvw_add_term(TermVectorsWriter *tvw,
+    char *text, int freq, int *positions, TVOffsetInfo **offsets) 
+{
+  if (tvw->tcnt >= tvw->tsize) {
+    tvw->tsize *= 2;
+    if (tvw->tsize < TERM_ARR_START_SIZE) 
+      tvw->tsize = TERM_ARR_START_SIZE;
+    REALLOC_N(tvw->terms, TVTerm *, tvw->tsize);
+  }
+  tvw->terms[tvw->tcnt] = tvt_create(text, freq, positions, offsets);
+  tvw->tcnt++;
+}
+
+void tvw_add_all_doc_vectors(TermVectorsWriter *tvw, Array *vectors)
+{
+  tvw_open_doc(tvw);
+
+  int i, j, store_positions, store_offsets;
+  TermVector *tv;
+  for (i = 0; i < vectors->size; i++) {
+    tv = vectors->elems[i];
+
+    store_positions = (tv->tcnt > 0 && tv->positions != NULL);
+    store_offsets = (tv->tcnt > 0 && tv->offsets != NULL);
+
+    tvw_create_field(tvw, fis_get_number(tvw->fis, tv->field),
+                 store_positions, store_offsets);
+
+    if (store_positions && store_offsets) {
+      for (j = 0; j < tv->tcnt; j++)
+        tvw_add_term(tvw, tv->terms[j], tv->freqs[j], tv->positions[j], tv->offsets[j]);
+    } else if (store_positions) {
+      for (j = 0; j < tv->tcnt; j++)
+        tvw_add_term(tvw, tv->terms[j], tv->freqs[j], tv->positions[j], NULL);
+    } else if (store_offsets) {
+      for (j = 0; j < tv->tcnt; j++)
+        tvw_add_term(tvw, tv->terms[j], tv->freqs[j], NULL, tv->offsets[j]);
+    } else {
+      for (j = 0; j < tv->tcnt; j++)
+        tvw_add_term(tvw, tv->terms[j], tv->freqs[j], NULL, NULL);
+    }
+    tvw_close_field(tvw);
+  }
+  tvw_close_doc(tvw);
+}
+
+void tvw_close(TermVectorsWriter *tvw)
+{
+  tvw_close_doc(tvw);
+  os_close(tvw->tvx);
+  os_close(tvw->tvd);
+  os_close(tvw->tvf);
+  free(tvw->terms);
+  free(tvw->fields);
+  free(tvw);
+}
+
+TermVector *tv_create(
+  const char *field,
+  char **terms,
+  int tcnt,
+  int *freqs,
+  int **positions,
+  TVOffsetInfo ***offsets)
+{
+  TermVector *tv =
+    ALLOC(TermVector);
+  tv->field = (char *)field;
+  tv->terms = terms;
+  tv->tcnt = tcnt;
+  tv->freqs = freqs;
+  tv->positions = positions;
+  tv->offsets = offsets;
+  return tv;
+}
+
+void tv_destroy(void *p)
+{
+  int i, j;
+  TermVector *tv = (TermVector *)p;
+  for (i = 0; i < tv->tcnt; i++) {
+    free(tv->terms[i]);
+  }
+  free(tv->terms);
+  if (tv->positions != NULL) {
+    for (i = 0; i < tv->tcnt; i++) {
+      free(tv->positions[i]);
+    }
+    free(tv->positions);
+  }
+  if (tv->offsets != NULL) {
+    for (i = 0; i < tv->tcnt; i++) {
+      for (j = 0; j < tv->freqs[i]; j++) {
+        tvoi_destroy(tv->offsets[i][j]);
+      }
+      free(tv->offsets[i]);
+    }
+    free(tv->offsets);
+  }
+  free(tv->freqs);
+  free(p);
+}
+
+void tv_destroy_except_data(void *p)
+{
+  TermVector *tv = (TermVector *)p;
+  free(tv->terms);
+  if (tv->positions != NULL) {
+    free(tv->positions);
+  }
+  if (tv->offsets != NULL) {
+    free(tv->offsets);
+  }
+  free(tv->freqs);
+  free(p);
+}
+
+int tvr_check_valid_format(InStream *is)
+{
+  int format = is_read_int(is);
+  if (format > FORMAT_VERSION)
+    eprintf(ERROR, "Incompatible format version: %d expected %d or less",
+        format, FORMAT_VERSION);
+  return format;
+}
+
+TermVectorsReader *tvr_clone(TermVectorsReader *orig)
+{
+  TermVectorsReader *clone = NULL;
+  if (orig->tvx && orig->tvd && orig->tvf) { 
+    clone = ALLOC(TermVectorsReader);
+    memcpy(clone, orig, sizeof(TermVectorsReader));
+    clone->tvx = is_clone(orig->tvx);
+    clone->tvd = is_clone(orig->tvd);
+    clone->tvf = is_clone(orig->tvf);
+  }
+  return clone;
+}
+
+TermVectorsReader *tvr_open(Store *store, char *segment, FieldInfos *fis)
+{
+  TermVectorsReader *tvr = ALLOC(TermVectorsReader);
+  // Open files for TermVector storage
+  char fname[SEGMENT_NAME_MAX_LENGTH];
+  int segment_len = strlen(segment);
+  strcpy(fname, segment);
+
+  strcpy(fname + segment_len, TVX_EXTENSION);
+  InStream *is = tvr->tvx = store->open_input(store, fname);
+  tvr_check_valid_format(is);
+  tvr->size = is_length(is)/8;
+
+  strcpy(fname + segment_len, TVD_EXTENSION);
+  is = tvr->tvd = store->open_input(store, fname);
+  tvr->tvd_format = tvr_check_valid_format(is);
+
+  strcpy(fname + segment_len, TVF_EXTENSION);
+  is = tvr->tvf = store->open_input(store, fname);
+  tvr->tvf_format = tvr_check_valid_format(is);
+
+  tvr->fis = fis;
+  return tvr;
+}
+
+void tvr_close(TermVectorsReader *tvr)
+{
+  is_close(tvr->tvx);
+  is_close(tvr->tvd);
+  is_close(tvr->tvf);
+  free(tvr);
+}
+
+TermVector *tvr_read_term_vector(TermVectorsReader *tvr,
+    char *field, int tvf_pointer)
+{
+  int i, j, store_positions, store_offsets, bits;
+  // Now read the data from specified position
+  // We don't need to offset by the FORMAT here since the pointer
+  // already includes the offset
+  is_seek(tvr->tvf, tvf_pointer);
+
+  int num_terms = is_read_vint(tvr->tvf);
+  // If no terms - return a constant empty termvector. However, this should
+  // never occur!
+  if (num_terms == 0)
+    return tv_create(field, NULL, 0, NULL, NULL, NULL);
+  
+  if(tvr->tvf_format == FORMAT_VERSION) {
+    bits = is_read_byte(tvr->tvf);
+    store_positions = ((bits & STORE_POSITIONS_WITH_TERMVECTOR) != 0);
+    store_offsets = ((bits & STORE_OFFSET_WITH_TERMVECTOR) != 0);
+  } else {
+    is_read_vint(tvr->tvf);
+    store_positions = false;
+    store_offsets = false;
+  }
+
+  char **terms = ALLOC_N(char *, num_terms);
+  int *term_freqs = ALLOC_N(int, num_terms);
+  
+  //  we may not need these, but declare them
+  int **positions = NULL; 
+  TVOffsetInfo ***offsets = NULL;
+
+  if(store_positions)
+    positions = ALLOC_N(int *, num_terms);
+
+  if(store_offsets)
+    offsets = ALLOC_N(TVOffsetInfo **, num_terms);
+  
+  int start, delta_length, total_length, freq, prev_pos;
+  int start_offset, end_offset, prev_offset;
+  int *pos;
+  TVOffsetInfo **offs;
+  char buffer[MAX_WORD_SIZE] = "";
+  
+  for (i = 0; i < num_terms; i++) {
+    start = is_read_vint(tvr->tvf);
+    delta_length = is_read_vint(tvr->tvf);
+    total_length = start + delta_length;
+    is_read_chars(tvr->tvf, buffer, start, delta_length);
+    buffer[total_length] = '\0';
+    terms[i] = estrdup(buffer);
+    freq = is_read_vint(tvr->tvf);
+    term_freqs[i] = freq;
+    
+    if (store_positions) {//read in the positions
+      pos = ALLOC_N(int, freq);
+      positions[i] = pos;
+      prev_pos = 0;
+      for (j = 0; j < freq; j++) {
+        pos[j] = prev_pos + is_read_vint(tvr->tvf);
+        prev_pos = pos[j];
+      }
+    }
+    
+    if (store_offsets) {
+      offs = ALLOC_N(TVOffsetInfo *, freq);
+      offsets[i] = offs;
+      prev_offset = 0;
+      for (j = 0; j < freq; j++) {
+        start_offset = prev_offset + is_read_vint(tvr->tvf);
+        end_offset = start_offset + is_read_vint(tvr->tvf);
+        offs[j] = tvoi_create(start_offset, end_offset);
+        prev_offset = end_offset;
+      }
+    }
+  }
+  return tv_create(field, terms, num_terms, term_freqs, positions, offsets);
+}
+
+Array *tvr_get_tv(TermVectorsReader *tvr, int doc_num)
+{
+  int i;
+  Array *tvs = NULL;
+  // Check if no term vectors are available for this segment at all
+  if (tvr->tvx != NULL) {
+    // We need to offset by
+    is_seek(tvr->tvx, (doc_num * 8) + FORMAT_SIZE);
+
+    int position = is_read_long(tvr->tvx);
+
+    is_seek(tvr->tvd, position);
+    int field_count = is_read_vint(tvr->tvd);
+
+    // No fields are vectorized for this document
+    if (field_count > 0) {
+      int number = 0;
+      char **fields = ALLOC_N(char *, field_count);
+      
+      for (i = 0; i < field_count; i++) {
+        if (tvr->tvd_format == FORMAT_VERSION)
+          number = is_read_vint(tvr->tvd);
+        else
+          number += is_read_vint(tvr->tvd);
+
+        fields[i] = tvr->fis->by_number[number]->name;
+      }
+
+      // Compute position in the tvf file
+      int position = 0;
+      int *tvf_pointers = ALLOC_N(int, field_count);
+      for (i = 0; i < field_count; i++) {
+        position += is_read_vint(tvr->tvd);
+        tvf_pointers[i] = position;
+      }
+
+      tvs = ary_create(field_count, &tv_destroy);
+      for (i = 0; i < field_count; i++) {
+        ary_append(tvs, tvr_read_term_vector(tvr, fields[i], tvf_pointers[i]));
+      }
+      free(fields);
+      free(tvf_pointers);
+    }
+  }
+  return tvs;
+}
+
+TermVector *tvr_get_field_tv(TermVectorsReader *tvr, int doc_num, char *field)
+{
+  int i;
+  // Check if no term vectors are available for this segment at all
+  int field_number = fis_get_number(tvr->fis, field);
+  TermVector *tv = NULL;
+
+  if (tvr->tvx != NULL) {
+    // We need to account for the FORMAT_SIZE at when seeking in the @tvx
+    // We don't need to do this in other seeks because we already have the
+    // file pointer that was written in another file
+    is_seek(tvr->tvx, (doc_num * 8) + FORMAT_SIZE);
+    // puts("TVX Pointer: " + @tvx.pos())
+    int pos = is_read_long(tvr->tvx);
+
+    is_seek(tvr->tvd, pos);
+    int field_count = is_read_vint(tvr->tvd);
+    //puts("Num Fields: " + field_count)
+    // There are only a few fields per document. We opt for a full scan
+    // rather then requiring that they be ordered. We need to read through
+    // all of the fields anyway to get to the tvf pointers.
+    int number = 0;
+    int found = -1;
+
+    for (i = 0; i < field_count; i++) {
+      if (tvr->tvd_format == FORMAT_VERSION)
+        number = is_read_vint(tvr->tvd);
+      else
+        number += is_read_vint(tvr->tvd);
+
+      if (number == field_number)
+        found = i;
+    }
+
+    // This field, although valid in the segment, was not found in this
+    // document
+    if (found != -1) {
+      // Compute pos in the @tvf file
+      pos = 0;
+      for (i = 0; i <= found; i++)
+        pos += is_read_vint(tvr->tvd);
+
+      tv = tvr_read_term_vector(tvr, field, pos);
+    }
+  }
+  return tv;
+}