ferret 0.3.2 → 0.9.0

data/ext/analysis.h

@@ -0,0 +1,76 @@
+#ifndef FRT_ANALYSIS_H
+#define FRT_ANALYSIS_H
+
+#include <global.h>
+#include <hash.h>
+
+/****************************************************************************
+ *
+ * Token
+ *
+ ****************************************************************************/
+
+typedef struct Token {
+  char text[MAX_WORD_SIZE];
+  int start;
+  int end;
+  int pos_inc;
+} Token;
+
+Token *tk_create();
+void tk_destroy(void *p);
+Token *tk_set(Token *tk, char *text, int tlen, int start, int end, int pos_inc);
+Token *tk_set_no_len(Token *tk, char *text, int start, int end, int pos_inc);
+int tk_eq(Token *tk1, Token *tk2);
+int tk_cmp(Token *tk1, Token *tk2);
+
+/****************************************************************************
+ *
+ * TokenStream
+ *
+ ****************************************************************************/
+
+typedef struct TokenStream TokenStream;
+struct TokenStream {
+  void *data;
+  char *text;
+  int pos;
+  Token *token;
+  Token *(*next)(TokenStream *ts);
+  void (*reset)(TokenStream *ts, char *text);
+  void (*destroy)(void *p);
+  TokenStream *sub_ts; // used by filters
+};
+
+#define ts_next(mts) mts->next(mts)
+#define ts_destroy(mts) mts->destroy(mts)
+
+TokenStream *whitespace_tokenizer_create();
+TokenStream *letter_tokenizer_create();
+TokenStream *standard_tokenizer_create();
+TokenStream *lowercase_filter_create(TokenStream *ts);
+TokenStream *stop_filter_create_with_words(TokenStream *ts, char **words, int len);
+TokenStream *stop_filter_create(TokenStream *ts);
+
+/****************************************************************************
+ *
+ * Analyzer
+ *
+ ****************************************************************************/
+
+typedef struct Analyzer {
+  void *data;
+  TokenStream *current_ts;
+  TokenStream *(*get_ts)(struct Analyzer *a, char *field, char *text);
+  void (*destroy)(void *p);
+} Analyzer;
+
+#define a_destroy(ma) ma->destroy(ma)
+#define a_get_ts(ma, field, text) ma->get_ts(ma, field, text)
+
+Analyzer *whitespace_analyzer_create();
+Analyzer *letter_analyzer_create();
+Analyzer *standard_analyzer_create();
+Analyzer *standard_analyzer_create_with_words(char **words, int len);
+
+#endif

data/ext/array.c

@@ -0,0 +1,83 @@
+#include <global.h>
+#include <array.h>
+#include <string.h>
+
+Array *ary_create(int allocate, void (*free_elem)(void *p))
+{
+  Array *ary = ALLOC(Array);
+  if (allocate == 0) {
+    ary->elems = NULL;
+  } else {
+    ary->elems = ALLOC_N(void *, allocate);
+    memset(ary->elems, 0, sizeof(void *) * allocate);
+  }
+  ary->size = 0;
+  ary->allocated = allocate;
+  ary->free_elem = free_elem;
+
+  return ary;
+}
+
+void ary_destroy(void *p)
+{
+  Array *ary = (Array *)p;
+  int i;
+  for (i = 0; i < ary->size; i++) {
+    if (ary->free_elem != NULL && ary->elems[i] != NULL)
+      ary->free_elem(ary->elems[i]);
+  }
+  free(ary->elems);
+  free(ary);
+}
+
+void ary_set(Array *ary, int index, void *value)
+{
+  if (index >= ary->allocated) {
+    ary->allocated = (index + 1)*2;
+    REALLOC_N(ary->elems, void *, (ary->allocated));
+    memset((&ary->elems[ary->size]), 0, sizeof(void *) * (ary->allocated - ary->size));
+  }
+
+  if (index >= ary->size)
+    ary->size = index + 1;
+
+  if (ary->free_elem != NULL && ary->elems[index] != NULL)
+    ary->free_elem(ary->elems[index]);
+
+  ary->elems[index] = value;
+}
+
+void ary_append(Array *ary, void *value)
+{
+  ary_set(ary, ary->size, value);
+}
+
+void *ary_get(Array *ary, int index)
+{
+  if (index >= ary->size)
+    return NULL;
+  return ary->elems[index];
+}
+
+void ary_delete(Array *ary, int index)
+{
+  if (index >= ary->size)
+    return;
+  ary->free_elem(ary->elems[index]);
+  ary->elems[index] = NULL;
+  if (index == ary->size - 1)
+    ary->size--;
+}
+
+void *ary_remove(Array *ary, int index)
+{
+  void *p;
+  if (index >= ary->size)
+    return NULL;
+  p = ary->elems[index];
+  ary->elems[index] = NULL;
+  ary->size--;
+  memmove(&ary->elems[index], &ary->elems[index + 1],
+      sizeof(void *) *(ary->size - index));
+  return p;
+}

data/ext/array.h

@@ -0,0 +1,19 @@
+#ifndef FRT_ARRAY_H
+#define FRT_ARRAY_H
+
+typedef struct Array {
+  void **elems;
+  int size;
+  int allocated;
+  void (*free_elem)(void *p);
+} Array;
+
+Array *ary_create(int size, void (*free_elem)(void *p));
+void ary_destroy(void *p);
+void ary_set(Array *ary, int index, void *value);
+void ary_append(Array *ary, void *value);
+void *ary_get(Array *ary, int index);
+void ary_delete(Array *ary, int index);
+void *ary_remove(Array *ary, int index);
+
+#endif

data/ext/bitvector.c

@@ -0,0 +1,164 @@
+#include <bitvector.h>
+#include <string.h>
+
+BitVector *bv_create_size(int size)
+{
+  BitVector *bv = ALLOC(BitVector);
+
+  bv->capa = (size >> 3) + 1;
+  bv->bits = ALLOC_N(uchar, bv->capa);
+  memset(bv->bits, 0, bv->capa);
+
+  bv->size = 0;
+  bv->count = 0;
+  bv->curr_bit = -1;
+  return bv;
+}
+
+BitVector *bv_create()
+{
+  return bv_create_size(BV_INIT_CAPA);
+}
+
+void bv_destroy(void *p)
+{
+  BitVector *bv = (BitVector *)p;
+  free(bv->bits);
+  free(bv);
+}
+
+void bv_set(BitVector *bv, int bit)
+{
+  int byte = bit>>3;
+  uchar bitmask = 1<<(bit&7);
+  if (bv->size <= byte) {
+    bv->size = byte + 1;
+    if (bv->size >= bv->capa) {
+      int capa = bv->capa * 2;
+      while (capa < bv->size) capa *= 2;
+      REALLOC_N(bv->bits, uchar, capa);
+      memset(bv->bits + bv->capa, 0, capa - bv->capa);
+      bv->capa = capa;
+    }
+  }
+  uchar *byte_p = &(bv->bits[byte]);
+  if ((bitmask & *byte_p) == 0) {
+    bv->count++;
+    *byte_p |= bitmask;
+  }
+}
+
+int bv_get(BitVector *bv, int bit)
+{
+  int byte = bit>>3;
+  if (byte >= bv->size) return 0;
+  return (bv->bits[byte]>>(bit&7))&1;
+}
+
+void bv_clear(BitVector *bv)
+{
+  memset(bv->bits, 0, bv->size);
+  bv->count = 0;
+}
+
+void bv_unset(BitVector *bv, int bit)
+{
+  int byte = bit>>3;
+  if (byte >= bv->size) return;
+  uchar *byte_p = &(bv->bits[byte]);
+  uchar bitmask = 1<<(bit&7);
+  if ((bitmask & *byte_p) > 0) {
+    bv->count--;
+    *byte_p &= ~bitmask;
+  }
+}
+
+void bv_write(BitVector *bv, Store *store, char *name)
+{
+  OutStream *os = store->create_output(store, name);
+  os_write_vint(os, bv->size);
+  os_write_bytes(os, bv->bits, bv->size);
+  os_close(os);
+}
+
+const uchar BYTE_COUNTS[] = { // table of bits/char
+  0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4,
+  1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+  1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+  2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+  1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+  2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+  2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+  3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+  1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
+  2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+  2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+  3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+  2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6,
+  3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+  3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7,
+  4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7, 6, 7, 7, 8
+};
+
+int bv_count(BitVector *bv) 
+{
+  // if the vector has been modified
+  int i, c = 0;
+  uchar *bytes = bv->bits;
+  for (i = 0; i < bv->size; i++)
+    c += BYTE_COUNTS[bytes[i]]; // sum bits per char
+  bv->count = c;
+  return c;
+}
+
+BitVector *bv_read(Store *store, char *name)
+{
+  BitVector *bv = ALLOC(BitVector);
+  InStream *is = store->open_input(store, name);
+  bv->capa = bv->size = is_read_vint(is);
+  bv->bits = ALLOC_N(uchar, bv->capa);
+  is_read_bytes(is, bv->bits, 0, bv->size);
+  is_close(is);
+  bv_count(bv);
+  return bv;
+}
+
+void bv_scan_reset(BitVector *bv)
+{
+  bv->curr_bit = -1;
+}
+
+inline int bv_scan_next_from(BitVector *bv, register const int from)
+{
+  register const uchar *const bits = bv->bits;
+  register const int size = bv->size;
+  register int byte_pos = (from) >> 3;
+  register int inc = ((from) & 7);
+  register int bit = 1 << inc;
+  register int mask = 0xff << inc;
+  register int byte;
+
+  if (byte_pos >= size) return -1;
+  if ((bits[byte_pos]&mask) == 0) {
+    inc = 0;
+    bit = 1;
+    do {
+      byte_pos++;
+      if (byte_pos >= size) return -1;
+    } while (bits[byte_pos] == 0);
+  }
+
+  byte = bits[byte_pos];
+  while ((byte & bit) == 0) {
+    bit <<= 1;
+    inc++;
+  }
+
+  return bv->curr_bit = ((byte_pos << 3) + inc);
+}
+
+inline int bv_scan_next(BitVector *bv)
+{
+  return bv_scan_next_from(bv, bv->curr_bit+1);
+}
+

data/ext/bitvector.h

@@ -0,0 +1,29 @@
+#ifndef FRT_BIT_VECTOR_H
+#define FRT_BIT_VECTOR_H
+
+#include <global.h>
+#include <store.h>
+
+#define BV_INIT_CAPA 256 
+typedef struct BitVector {
+  uchar *bits;
+  int size;
+  int capa;
+  int count;
+  int curr_bit;
+} BitVector;
+
+BitVector *bv_create();;
+BitVector *bv_create_size(int size);
+void bv_destroy(void *bv);
+void bv_set(BitVector *bv, int bit);
+int bv_get(BitVector *bv, int bit);
+void bv_clear(BitVector *bv);
+void bv_unset(BitVector *bv, int bit);
+void bv_write(BitVector *bv, Store *store, char *name);
+BitVector *bv_read(Store *store, char *name);
+void bv_scan_reset(BitVector *bv);
+int bv_scan_next(BitVector *bv);
+int bv_scan_next_from(BitVector *bv, int from);
+
+#endif

data/ext/compound_io.c

@@ -0,0 +1,335 @@
+#include "index.h"
+
+/****************************************************************************
+ *
+ * CompoundStore
+ *
+ ****************************************************************************/
+
+typedef struct FileEntry {
+  int offset;
+  int length;
+} FileEntry;
+
+void cmpd_touch(Store *store, char *filename)
+{
+  store->dir.cmpd->store->touch(store->dir.cmpd->store, filename);
+}
+
+int cmpd_exists(Store *store, char *filename)
+{
+  if (h_get(store->dir.cmpd->entries, filename) != NULL)
+    return true;
+  else
+    return false;
+}
+
+int cmpd_remove(Store *store, char *filename)
+{
+  eprintf(UNSUPPORTED_ERROR, "Unsupported operation");
+  return 0;
+}
+
+int cmpd_rename(Store *store, char *from, char *to)
+{
+  eprintf(UNSUPPORTED_ERROR, "Unsupported operation");
+  return 0;
+}
+
+int cmpd_count(Store *store)
+{
+  return store->dir.cmpd->entries->used;
+}
+
+void cmpd_clear(Store *store)
+{
+  eprintf(UNSUPPORTED_ERROR, "Unsupported operation");
+}
+
+void cmpd_close(Store *store)
+{
+  mutex_lock(&store->mutex);
+  CompoundStore *cmpd = store->dir.cmpd;
+  if (cmpd->stream == NULL)
+    eprintf(IO_ERROR, "Already closed");
+
+  h_destroy(cmpd->entries);
+
+  is_close(cmpd->stream);
+  cmpd->stream = NULL;
+  free(store->dir.cmpd);
+  store_destroy(store);
+}
+
+int cmpd_length(Store *store, char *filename)
+{
+  FileEntry *fe = (FileEntry *)h_get(store->dir.cmpd->entries, filename);
+  if (fe != NULL)
+    return fe->length;
+  else
+    return 0;
+}
+
+void cmpdi_seek_internal(InStream *is, int pos) {}
+void cmpdi_close_internal(InStream *is)
+{
+  //is_close(is->d.cis->sub);
+  free(is->d.cis);
+}
+
+void cmpdi_clone_internal(InStream *is, InStream *new_is)
+{
+  CompoundInStream *cis = ALLOC(CompoundInStream);
+  //cis->sub = is_clone(is->d.cis->sub);
+  cis->sub = is->d.cis->sub;
+  cis->offset = is->d.cis->offset;
+  cis->length = is->d.cis->length;
+  new_is->d.cis = cis;
+}
+
+int cmpdi_length_internal(InStream *is)
+{
+  return (is->d.cis->length);
+}
+
+void cmpdi_read_internal(InStream *is, uchar *b, int offset, int len)
+{
+  CompoundInStream *cis = is->d.cis;
+  int start = is_pos(is);
+  if ((start + len) > cis->length)
+    eprintf(EOF_ERROR, "read past EOF");
+  is_seek(cis->sub, cis->offset + start);
+  is_read_bytes(cis->sub, b, offset, len);
+}
+
+InStream *cmpd_create_input(InStream *sub_is, int offset, int length)
+{
+  InStream *is = is_create();
+  CompoundInStream *cis = ALLOC(CompoundInStream);
+  //cis->sub = is_clone(sub_is);
+  cis->sub = sub_is;
+  cis->offset = offset;
+  cis->length = length;
+  is->d.cis = cis;
+  is->file = NULL;
+
+  is->read_internal = &cmpdi_read_internal;
+  is->seek_internal = &cmpdi_seek_internal;
+  is->close_internal = &cmpdi_close_internal;
+  is->clone_internal = &cmpdi_clone_internal;
+  is->length_internal = &cmpdi_length_internal;
+  return is;
+}
+
+InStream *cmpd_open_input(Store *store, const char *filename)
+{
+  CompoundStore *cmpd = store->dir.cmpd;
+  InStream *is;
+
+  mutex_lock(&store->mutex);
+  if (cmpd->stream == NULL) {
+    mutex_unlock(&store->mutex);
+    eprintf(IO_ERROR, "Stream closed");
+  }
+
+  FileEntry *entry = (FileEntry *)h_get(cmpd->entries, filename);
+  if (entry == NULL) {
+    mutex_unlock(&store->mutex);
+    eprintf(IO_ERROR, "No sub-file with id <%s> found", filename);
+  }
+
+  is = cmpd_create_input(cmpd->stream, entry->offset, entry->length);
+  mutex_unlock(&store->mutex);
+
+  return is;
+}
+
+OutStream *cmpd_create_output(Store *store, const char *filename)
+{
+  eprintf(UNSUPPORTED_ERROR, "Unsupported operation");
+  return NULL;
+}
+
+Lock *cmpd_open_lock(Store *store, char *lockname)
+{
+  eprintf(UNSUPPORTED_ERROR, "Unsupported operation");
+  return NULL;
+}
+
+void cmpd_close_lock(Lock *lock)
+{
+  eprintf(UNSUPPORTED_ERROR, "Unsupported operation");
+}
+
+Store *open_cmpd_store(Store *store, const char *name)
+{
+  CompoundStore *cmpd = ALLOC(CompoundStore);
+  Store *new_store = store_create();
+
+  cmpd->store   = store;
+  cmpd->name    = name;
+  cmpd->entries = h_new_str(&efree, &efree);
+  InStream *is = cmpd->stream = store->open_input(store, cmpd->name);
+
+  // read the directory and init files
+  int count = is_read_vint(is);
+  FileEntry *entry = NULL;
+  int i, offset;
+  char *fname;
+  for (i = 0; i < count; i++) {
+    offset = is_read_long(is);
+    fname = is_read_string(is);
+
+    if (entry != NULL) {
+      // set length of the previous entry
+      entry->length = offset - entry->offset;
+    }
+
+    entry = ALLOC(FileEntry);
+    entry->offset = offset;
+    h_set(cmpd->entries, fname, entry);
+  }
+
+  // set the length of the final entry
+  if (entry != NULL)
+    entry->length = is_length(is) - entry->offset;
+
+  new_store->dir.cmpd = cmpd;
+  new_store->touch    = &cmpd_touch;
+  new_store->exists   = &cmpd_exists;
+  new_store->remove   = &cmpd_remove;
+  new_store->rename   = &cmpd_rename;
+  new_store->count    = &cmpd_count;
+  new_store->close    = &cmpd_close;
+  new_store->clear    = &cmpd_clear;
+  new_store->length   = &cmpd_length;
+  new_store->create_output = &cmpd_create_output;
+  new_store->open_input    = &cmpd_open_input;
+  new_store->open_lock     = &cmpd_open_lock;
+  new_store->close_lock    = &cmpd_close_lock;
+  return new_store;
+}
+
+/****************************************************************************
+ *
+ * CompoundWriter
+ *
+ ****************************************************************************/
+
+typedef struct WFileEntry {
+  char *name;
+  int dir_offset;
+  int data_offset;
+} WFileEntry;
+
+WFileEntry *wfe_create(char *name)
+{
+  WFileEntry *wfe = ALLOC(WFileEntry);
+  wfe->name = name;
+  return wfe;
+}
+
+void wfe_destroy(void *p)
+{
+  WFileEntry *wfe = (WFileEntry *)p;
+  efree(wfe);
+}
+
+CompoundWriter *open_cw(Store *store, char *name)
+{
+  CompoundWriter *cw = ALLOC(CompoundWriter);
+  cw->store = store;
+  cw->name = name;
+  cw->ids = hs_str_create(NULL);
+  cw->file_entries = ary_create(1, &wfe_destroy);
+  cw->merged = false;
+  return cw;
+}
+
+void cw_add_file(CompoundWriter *cw, char *id)
+{
+  if (cw->merged) eprintf(STATE_ERROR, "Already merged");
+  if (hs_add(cw->ids, id) != HASH_KEY_DOES_NOT_EXIST)
+    eprintf(STATE_ERROR, "Already merged");
+
+  hs_add(cw->ids, id);
+  ary_append(cw->file_entries, wfe_create(id));
+}
+
+void cw_copy_file(CompoundWriter *cw, WFileEntry *src, OutStream *os)
+{
+
+  int start_ptr = os_pos(os);
+
+  InStream *is = cw->store->open_input(cw->store, src->name);
+  int remainder, length, len;
+  remainder = length = is_length(is);
+
+  uchar buffer[BUFFER_SIZE];
+  while (remainder > 0) {
+    len = MIN(remainder, BUFFER_SIZE);
+    is_read_bytes(is, buffer, 0, len);
+    os_write_bytes(os, buffer, len);
+    remainder -= len;
+  }
+
+  // Verify that remainder is 0
+  if (remainder != 0)
+    eprintf(IO_ERROR, "Non-zero remainder length after copying: %ld "
+        "(id:%s, length: %ld, buffer size: %ld\n", remainder,
+        src->name, length, BUFFER_SIZE);
+
+  // Verify that the output length diff is equal to original file
+  int end_ptr = os_pos(os);
+  int diff = end_ptr - start_ptr;
+  if (diff != length)
+    eprintf(IO_ERROR, "Difference in the output file offsets %ld "
+        " does not match the original file length ", diff, length);
+
+  is_close(is);
+}
+
+void cw_close(CompoundWriter *cw)
+{
+  if (cw->merged) eprintf(STATE_ERROR, "Already merged");
+  if (cw->ids->size <= 0)
+    eprintf(STATE_ERROR, "No Files to merge into the compound file");
+
+  cw->merged = true;
+
+  OutStream *os = cw->store->create_output(cw->store, cw->name);
+  os_write_vint(os, cw->file_entries->size);
+
+  /* Write the directory with all offsets at 0.
+   * Remember the positions of directory entries so that we can adjust the
+   * offsets later */
+  int i;
+  WFileEntry *wfe; 
+  for (i = 0; i < cw->file_entries->size; i++) {
+    wfe = (WFileEntry *)cw->file_entries->elems[i];
+    wfe->dir_offset = os_pos(os);
+    os_write_long(os, 0);  // for now
+    os_write_string(os, wfe->name);
+  }
+
+  /* Open the files and copy their data into the stream.  Remember the
+   * locations of each file's data section. */
+  for (i = 0; i < cw->file_entries->size; i++) {
+    wfe = (WFileEntry *)cw->file_entries->elems[i];
+    wfe->data_offset = os_pos(os);
+    cw_copy_file(cw, wfe, os);
+  }
+
+  /* Write the data offsets into the directory of the compound stream */
+  for (i = 0; i < cw->file_entries->size; i++) {
+    wfe = (WFileEntry *)cw->file_entries->elems[i];
+    os_seek(os, wfe->dir_offset);
+    os_write_long(os, wfe->data_offset);
+  }
+
+  os_close(os);
+  hs_destroy(cw->ids);
+  ary_destroy(cw->file_entries);
+  free(cw);
+}
+