ferret 0.3.2 → 0.9.0

data/ext/hash.h

@@ -0,0 +1,80 @@
+#ifndef FRT_HASH_H
+#define FRT_HASH_H
+
+#include "global.h"
+
+#define NUM_ENTRIES 256
+#define MULTIPLIER 31
+
+typedef struct HashEntry {
+  char      *name;
+  void      *value;
+  struct HashEntry *next;
+} HashEntry;
+
+HashEntry **ht_create();
+int ht_count(HashEntry **ht);
+void ht_destroy(HashEntry **ht);
+void ht_destroy_all(HashEntry **ht, void (*fn)(void *));
+void ht_set(HashEntry **ht, char *name, void *value);
+void *ht_get(HashEntry **ht, char *name);
+void *ht_delete(HashEntry **ht, char *name);
+
+/****************************************************************************
+ *
+ * HshTable
+ *
+ ****************************************************************************/
+
+#define Hsh_MINSIZE 8
+#define SLOW_DOWN 50000 // stop increasing the hash table so quickly to
+                        // conserve memory
+extern char *dummy_key;
+enum {
+  HASH_KEY_DOES_NOT_EXIST = 0,
+  HASH_KEY_SAME = 1,
+  HASH_KEY_EQUAL = 2
+};
+
+typedef struct {
+	int hash;      /* cached hash code of key */
+  void *key;
+	void *value;
+} HshEntry;
+
+typedef struct HshTable {
+	int fill;  /* # Active + # Dummy */
+	int used;  /* # Active */
+	int mask;
+
+	/* table points to smalltable for small tables, else to
+	 * additional malloc'ed memory. */
+	HshEntry *table;
+	HshEntry smalltable[Hsh_MINSIZE];
+  HshEntry *(*lookup)(struct HshTable *ht, const void *key);
+  unsigned int (*hash)(const void *key);
+  int (*eq)(const void *key1, const void *key2);
+  void (*free_key)(void *key);
+  void (*free_value)(void *value);
+} HshTable;
+
+HshTable *h_new_str(void (*free_key)(void *key), void (*free_value)(void *value));
+HshTable *h_new(unsigned int (*hash)(const void *key),
+    int (*eq)(const void *key1, const void *key2),
+    void (*free_key)(void *key),
+    void (*free_value)(void *value));
+void h_destroy(HshTable *ht);
+void h_clear(HshTable *ht);
+
+void *h_get(HshTable *ht, const void *key);
+int h_del(HshTable *ht, const void *key);
+void *h_rem(HshTable *ht, const void *key, bool del_key);
+int h_set(HshTable *ht, const void *key, void *value);
+int h_set_safe(HshTable *ht, const void *key, void *value);
+int h_has_key(HshTable *ht, const void *key);
+unsigned int str_hash(const char *const str);
+
+void dummy_free(void *p);
+HshEntry *h_lookup_str(HshTable *ht, register const void *key_p);
+
+#endif

data/ext/hashset.c

@@ -0,0 +1,141 @@
+#include <hashset.h>
+#include <string.h>
+#define HS_MIN_SIZE 4
+
+int *imalloc(int i)
+{
+  int *ip = ALLOC(int);
+  *ip = i;
+  return ip;
+}
+
+void hs_dummy_free(void *p){}
+
+HashSet *hs_create(unsigned int (*hash)(const void *p),
+    int (*eq)(const void *p1, const void *p2),
+    void (*free_elem)(void *p))
+{
+  HashSet *hs = ALLOC(HashSet);
+  hs->ht = h_new(hash, eq, NULL, &efree);
+  hs->elems = NULL;
+  hs->capa = hs->size = 0;
+  if (free_elem == NULL)
+    hs->free_elem = &hs_dummy_free;
+  else
+    hs->free_elem = free_elem;
+  return hs;
+}
+
+HashSet *hs_str_create(void (*free_elem)(void *p))
+{
+  HashSet *hs = ALLOC(HashSet);
+  hs->ht = h_new_str(NULL, &efree);
+  hs->elems = NULL;
+  hs->capa = hs->size = 0;
+  if (free_elem == NULL)
+    hs->free_elem = &hs_dummy_free;
+  else
+    hs->free_elem = free_elem;
+  return hs;
+}
+
+void hs_destroy(void *p)
+{
+  HashSet *hs = (HashSet *)p;
+  h_destroy(hs->ht);
+  free(hs->elems);
+  free(hs);
+}
+
+void hs_clear(HashSet *self)
+{
+  int i;
+  for (i = self->size - 1; i >= 0; i--)
+    hs_del(self, self->elems[i]);
+}
+
+void hs_destroy_all(void *p)
+{
+  int i;
+  HashSet *hs = (HashSet *)p;
+  if (hs->free_elem != &dummy_free)
+    for (i = 0; i < hs->size; i++)
+      hs->free_elem(hs->elems[i]);
+  hs_destroy(p);
+}
+
+int hs_add(HashSet *hs, void *elem)
+{
+  int has_elem = h_has_key(hs->ht, elem);
+  //printf("has_elem = %d %d:%d\n", has_elem, HASH_KEY_EQUAL, HASH_KEY_SAME);
+  if (has_elem == HASH_KEY_EQUAL) {
+    // We don't want to keep two of the same elem so free if necessary
+    hs->free_elem(elem);
+  } else if (has_elem == HASH_KEY_SAME) {
+    // No need to do anything
+  } else {
+    // add the elem to the array, resizing if necessary
+    if (hs->size >= hs->capa) {
+      if (hs->capa == 0)
+        hs->capa = HS_MIN_SIZE;
+      else
+        hs->capa *= 2;
+      REALLOC_N(hs->elems, void *, hs->capa);
+    }
+    hs->elems[hs->size] = elem;
+    h_set(hs->ht, elem, imalloc(hs->size));
+    hs->size++;
+  }
+  return has_elem;
+}
+
+int hs_del(HashSet *hs, void *elem)
+{
+  void *tmp_elem = hs_rem(hs, elem);
+  if (tmp_elem != NULL) {
+    hs->free_elem(tmp_elem);
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+void *hs_rem(HashSet *hs, void *elem)
+{
+  void *ret_elem;
+  int *index = (int *)h_get(hs->ht, elem);
+  if (index == NULL) {
+    return NULL;
+  } else {
+    int i = *index;
+    ret_elem = hs->elems[i];
+    h_del(hs->ht, elem);
+    hs->size--;
+    memmove(&hs->elems[i], &hs->elems[i+1], sizeof(void *) * (hs->size - i));
+    return ret_elem;
+  }
+}
+
+int hs_exists(HashSet *hs, void *elem)
+{
+  return h_has_key(hs->ht, elem);
+}
+
+HashSet *hs_merge(HashSet *hs, HashSet *other)
+{
+  int i;
+  for (i = 0; i < other->size; i++) {
+    hs_add(hs, other->elems[i]);
+  }
+  // Now free the other hashset. It is no longer needed. No need, however, to
+  // delete the elements as they are in the new hash set
+  hs_destroy(other);
+  return hs;
+}
+
+void *hs_orig(HashSet *hs, void *elem)
+{
+  int *i = h_get(hs->ht, elem);
+  if (i) return hs->elems[*i];
+  else return NULL;
+}

data/ext/hashset.h

@@ -0,0 +1,37 @@
+#ifndef FRT_HASHSET_H
+#define FRT_HASHSET_H
+
+#include "hash.h"
+#include "array.h"
+#include "global.h"
+
+typedef struct HashSet {
+  int capa;
+  int size;
+  void **elems;
+  HshTable *ht;
+  void (*free_elem)(void *p);
+} HashSet;
+
+HashSet *hs_create(unsigned int (*hash)(const void *p),
+                   int (*eq)(const void *p1, const void *p2),
+                   void (*free_elem)(void *p));
+HashSet *hs_str_create(void (*free_elem)(void *p));
+void hs_destroy(void *p);
+void hs_destroy_all(void *p);
+int hs_add(HashSet *hs, void *elem);
+int hs_del(HashSet *hs, void *elem);
+void *hs_rem(HashSet *hs, void *elem);
+int hs_exists(HashSet *hs, void *elem);
+HashSet *hs_merge(HashSet *hs, HashSet *other);
+void *hs_orig(HashSet *hs, void *elem);
+void hs_clear(HashSet *self);
+
+// TODO: finish these functions.
+//int hs_osf(HashSet *hs, void *elem);
+//HashSet hs_or(HashSet *hs1, HashSet *h2);
+//HashSet hs_excl_or(HashSet *hs1, HashSet *h2);
+//HashSet hs_and(HashSet *hs1, HashSet *h2);
+//HashSet hs_mask(HashSet *hs1, HashSet *h2);
+
+#endif

data/ext/helper.c

@@ -0,0 +1,11 @@
+#include <string.h>
+
+inline int hlp_string_diff(register const char *const s1,
+                           register const char *const s2)
+{
+  register int i = 0;
+  while (s1[i] && (s1[i] == s2[i])) {
+    i++;
+  }
+  return i;
+}

data/ext/helper.h

@@ -0,0 +1,5 @@
+#ifndef FRT_HELPER_H
+#define FRT_HELPER_H
+
+int hlp_string_diff(char *s1, char *s2);
+#endif

data/ext/inc/lang.h

@@ -0,0 +1,41 @@
+#ifndef FRT_LANG_H
+#define FRT_LANG_H
+
+#include <ruby.h>
+
+#define FERRET_EXT
+
+#define MAX_ERROR_LEN 2048
+#define eprintf(...) ft_raise(__FILE__, __LINE__, __VA_ARGS__)
+extern void ft_raise(char *file, int line_num, VALUE etype, const char *fmt, ...);
+extern void weprintf(const char *fmt, ...);
+extern char *progname(void);
+extern void setprogname(const char *str);
+
+extern VALUE cQueryParseException;
+
+#define ERROR rb_eException
+#define IO_ERROR rb_eIOError
+#define ARG_ERROR rb_eArgError
+#define EOF_ERROR rb_eEOFError
+#define UNSUPPORTED_ERROR rb_eNotImpError
+#define STATE_ERROR rb_eException
+#define PARSE_ERROR cQueryParseException
+#define MEM_ERROR rb_eNoMemError
+
+typedef void * mutex_t;
+typedef void * thread_key_t;
+#define MUTEX_INITIALIZER NULL
+#define MUTEX_RECURSIVE_INITIALIZER NULL
+#define mutex_init(a, b)
+#define mutex_lock(a)
+#define mutex_trylock(a)
+#define mutex_unlock(a)
+#define mutex_destroy(a)
+#define thread_key_create(a, b)
+#define thread_key_delete(a)
+#define thread_setspecific(a, b)
+#define thread_getspecific(a) NULL
+#define thread_exit(a)
+
+#endif

data/ext/ind.c

@@ -0,0 +1,389 @@
+#include <string.h>
+#include "search.h"
+
+static const char *ID_STRING = "id";
+
+#define INDEX_CLOSE_READER(self) do {\
+  if (self->sea) {\
+    sea_close(self->sea);\
+    self->sea = NULL;\
+    self->ir = NULL;\
+  } else if (self->ir) {\
+    ir_close(self->ir);\
+    self->ir = NULL;\
+  }\
+} while (0)
+
+#define AUTOFLUSH_IR if (self->auto_flush) ir_commit(self->ir);\
+  else self->has_writes = true
+
+#define AUTOFLUSH_IW \
+  if (self->auto_flush) {\
+    iw_close(self->iw);\
+    self->iw = NULL;\
+  } else self->has_writes = true
+
+void index_auto_flush_ir(Index *self)
+{
+  AUTOFLUSH_IR;
+}
+
+void index_auto_flush_iw(Index *self)
+{
+  AUTOFLUSH_IW;
+}
+
+Index *index_create(Store *store, Analyzer *analyzer, HashSet *def_fields,
+    bool create)
+{
+  HashSet *all_fields = hs_str_create(&free);
+  Index *self = ALLOC(Index);
+  ZEROSET(self, Index, 1);
+  mutex_init(&self->mutex, NULL);
+  self->has_writes = false;
+  if (store) {
+    self->store = store;
+    self->close_store = false;
+  } else {
+    self->store = open_ram_store();
+    create = true;
+    self->close_store = true;
+  }
+  if (analyzer) {
+    self->analyzer = analyzer;
+    self->close_analyzer = false;
+  } else {
+    self->analyzer = standard_analyzer_create();
+    self->close_analyzer = true;
+  }
+  self->use_compound_file = true;
+
+  if (create) {
+    self->iw = iw_open(self->store, self->analyzer, create, false, false);
+    iw_close(self->iw);
+    self->iw = NULL;
+  }
+
+  /* options */
+  self->key = NULL;
+  self->id_field = (char *)ID_STRING;
+  self->def_field = (char *)ID_STRING;
+  self->auto_flush = false;
+
+  self->qp = qp_create(all_fields, def_fields, self->analyzer);
+  /* Index is a convenience class so set qp convenience options */
+  self->qp->allow_any_fields = true;
+  self->qp->clean_str = true;
+  self->qp->handle_parse_errors = true;
+
+  return self;
+}
+
+void index_destroy(Index *self)
+{
+  mutex_destroy(&self->mutex);
+  INDEX_CLOSE_READER(self);
+  if (self->iw) iw_close(self->iw);
+  if (self->close_store) self->store->close(self->store);
+  if (self->close_analyzer) a_destroy(self->analyzer);
+  if (self->qp) qp_destroy(self->qp);
+  if (self->id_field != ((char *)ID_STRING)) free(self->id_field);
+  if (self->def_field != ((char *)ID_STRING)) free(self->def_field);
+  if (self->key) hs_destroy_all(self->key);
+  free(self);
+}
+
+void index_flush(Index *self)
+{
+  if (self->ir) {
+    ir_commit(self->ir);
+  } else if (self->iw) {
+    iw_close(self->iw);
+    self->iw = NULL;
+  }
+  self->has_writes = false;
+}
+inline void ensure_writer_open(Index *self)
+{
+  if (!self->iw) {
+    INDEX_CLOSE_READER(self);
+    self->iw = iw_open(self->store, self->analyzer, false, false, false);
+    self->iw->use_compound_file = self->use_compound_file;
+  } else {
+    self->iw->analyzer = self->analyzer; /* in case it has changed */
+  }
+}
+
+inline void ensure_reader_open(Index *self)
+{
+  if (self->ir) {
+    if (!ir_is_latest(self->ir)) {
+      INDEX_CLOSE_READER(self);
+      self->ir = ir_open(self->store, false);
+    }
+  } else {
+    if (self->iw) {
+      iw_close(self->iw);
+      self->iw = NULL;
+    }
+    self->ir = ir_open(self->store, false);
+  }
+}
+
+inline void ensure_searcher_open(Index *self)
+{
+  ensure_reader_open(self);
+  if (!self->sea) {
+    self->sea = sea_create(self->ir);
+  }
+}
+
+int index_size(Index *self)
+{
+  int size;
+  mutex_lock(&self->store->ext_mutex);
+  ensure_reader_open(self);
+  size = self->ir->num_docs(self->ir);
+  mutex_unlock(&self->store->ext_mutex);
+  return size;
+}
+
+void index_optimize(Index *self)
+{
+  mutex_lock(&self->store->ext_mutex);
+  ensure_writer_open(self);
+  iw_optimize(self->iw);
+  AUTOFLUSH_IW;
+  mutex_unlock(&self->store->ext_mutex);
+}
+
+bool index_has_del(Index *self)
+{
+  bool has_del;
+  mutex_lock(&self->store->ext_mutex);
+  ensure_reader_open(self);
+  has_del = self->ir->has_deletions(self->ir);
+  mutex_unlock(&self->store->ext_mutex);
+  return has_del;
+}
+
+bool index_is_deleted(Index *self, int doc_num)
+{
+  bool is_del;
+  mutex_lock(&self->store->ext_mutex);
+  ensure_reader_open(self);
+  is_del = self->ir->is_deleted(self->ir, doc_num);
+  mutex_unlock(&self->store->ext_mutex);
+  return is_del;
+}
+
+static void inline index_add_doc_i(Index *self, Document *doc)
+{
+  /* If there is a key specified delete the document with the same key */
+  if (self->key) {
+    int i;
+    char *field;
+    DocField *df;
+    Query *q = bq_create(false);
+    TopDocs *td;
+    ensure_searcher_open(self);
+    for (i = 0; i < self->key->size; i++) {
+      field = self->key->elems[i];
+      df = doc_get_field(doc, field);
+      if (!df) continue;
+      bq_add_query(q, tq_create(term_create(field, df->data)), BC_MUST);
+    }
+    td = sea_search(self->sea, q, 0, 1, NULL, NULL);
+    if (td->total_hits > 1) {
+      td_destroy(td);
+      eprintf(ARG_ERROR, "Tried to use a key that was not unique");
+    } else if (td->total_hits == 1) {
+      ir_delete_doc(self->ir, td->hits[0]->doc);
+    }
+    q->destroy(q);
+    td_destroy(td);
+  }
+  ensure_writer_open(self);
+  iw_add_doc(self->iw, doc);
+  AUTOFLUSH_IW;
+}
+
+void index_add_doc_a(Index *self, Document *doc, Analyzer *analyzer)
+{
+  Analyzer *tmp_analyzer;
+  mutex_lock(&self->store->ext_mutex);
+  tmp_analyzer = self->analyzer;
+  self->analyzer = analyzer;
+  iw_add_doc(self->iw, doc);
+  index_add_doc_i(self, doc);
+  self->analyzer = tmp_analyzer;
+  mutex_unlock(&self->store->ext_mutex);
+}
+
+void index_add_doc(Index *self, Document *doc)
+{
+  mutex_lock(&self->store->ext_mutex);
+  index_add_doc_i(self, doc);
+  mutex_unlock(&self->store->ext_mutex);
+}
+
+void index_add_string(Index *self, char *str, Analyzer *analyzer)
+{
+  Document *doc = doc_create();
+  doc_add_field(doc, df_create(self->id_field, estrdup(str),
+        DF_STORE_YES, DF_INDEX_TOKENIZED, DF_TERM_VECTOR_NO));
+  if (analyzer) index_add_doc_a(self, doc, analyzer);
+  else index_add_doc(self, doc);
+  doc_destroy(doc);
+}
+
+void index_add_array(Index *self, Array *ary, Analyzer *analyzer)
+{
+  int i;
+  Document *doc = doc_create();
+  for (i = 0; i < ary->size; i++) {
+    doc_add_field(doc, df_create(self->id_field, estrdup(ary->elems[i]),
+        DF_STORE_YES, DF_INDEX_TOKENIZED, DF_TERM_VECTOR_NO));
+  }
+  if (analyzer) index_add_doc_a(self, doc, analyzer);
+  else index_add_doc(self, doc);
+  doc_destroy(doc);
+}
+
+Query *index_get_query(Index *self, char *qstr)
+{
+  int i;
+  HashSet *all_fields;
+  ensure_searcher_open(self);
+  all_fields = self->ir->get_field_names(self->ir, IR_ALL);
+  for (i = 0; i < all_fields->size; i++)
+    hs_add(self->qp->all_fields, estrdup(all_fields->elems[i]));
+  hs_destroy(all_fields);
+  return qp_parse(self->qp, qstr);
+}
+
+TopDocs *index_search_str(Index *self, char *qstr, int first_doc,
+    int num_docs, Filter *filter, Sort *sort)
+{
+  Query *query;
+  TopDocs *td;
+  query = index_get_query(self, qstr); /* will ensure_searcher is open */
+  td = sea_search(self->sea, query, first_doc, num_docs, filter, sort);
+  query->destroy(query);
+  return td;
+}
+
+Document *index_get_doc(Index *self, int doc_num)
+{
+  Document *doc;
+  ensure_reader_open(self);
+  doc = self->ir->get_doc(self->ir, doc_num);
+  return doc;
+}
+
+Document *index_get_doc_ts(Index *self, int doc_num)
+{
+  Document *doc;
+  mutex_lock(&self->store->ext_mutex);
+  doc = index_get_doc(self, doc_num);
+  mutex_unlock(&self->store->ext_mutex);
+  return doc;
+}
+
+int index_term_id(Index *self, Term *term)
+{
+  TermDocEnum *tde;
+  int doc_num = -1;
+  ensure_reader_open(self);
+  tde = ir_term_docs_for(self->ir, term);
+  if (tde->next(tde)) {
+    doc_num = tde->doc_num(tde);
+  }
+  tde->close(tde);
+  return doc_num;
+}
+
+Document *index_get_doc_term(Index *self, Term *term)
+{
+  Document *doc = NULL;
+  TermDocEnum *tde;
+  mutex_lock(&self->store->ext_mutex);
+  ensure_reader_open(self);
+  tde = ir_term_docs_for(self->ir, term);
+  if (tde->next(tde)) {
+    doc = index_get_doc(self, tde->doc_num(tde));
+    tde->close(tde);
+  }
+  mutex_unlock(&self->store->ext_mutex);
+  return doc;
+}
+
+Document *index_get_doc_id(Index *self, char *id)
+{
+  Term t;
+  t.field = self->id_field;
+  t.text = id;
+  return index_get_doc_term(self, &t);
+}
+
+void index_delete(Index *self, int doc_num)
+{
+  mutex_lock(&self->store->ext_mutex);
+  ensure_reader_open(self);
+  ir_delete_doc(self->ir, doc_num);
+  AUTOFLUSH_IR;
+  mutex_unlock(&self->store->ext_mutex);
+}
+
+void index_delete_term(Index *self, Term *term)
+{
+  TermDocEnum *tde;
+  mutex_lock(&self->store->ext_mutex);
+  ensure_reader_open(self);
+  tde = ir_term_docs_for(self->ir, term);
+  while (tde->next(tde)) {
+    ir_delete_doc(self->ir, tde->doc_num(tde));
+    AUTOFLUSH_IR;
+  }
+  tde->close(tde);
+  mutex_unlock(&self->store->ext_mutex);
+}
+
+void index_delete_id(Index *self, char *id)
+{
+  Term t;
+  t.field = self->id_field;
+  t.text = id;
+  index_delete_term(self, &t);
+}
+
+static void index_qdel_i(Searcher *sea, int doc_num, void *arg)
+{
+  ir_delete_doc(sea->ir, doc_num);
+}
+
+void index_delete_query(Index *self, Query *q, Filter *f)
+{
+  mutex_lock(&self->store->ext_mutex);
+  ensure_searcher_open(self);
+  sea_search_each(self->sea, q, f, &index_qdel_i, NULL);
+  AUTOFLUSH_IR;
+  mutex_unlock(&self->store->ext_mutex);
+}
+
+void index_delete_query_str(Index *self, char *qstr, Filter *f)
+{
+  Query *q = index_get_query(self, qstr);
+  index_delete_query(self, q, f);
+  q->destroy(q);
+}
+
+Explanation *index_explain(Index *self, Query *q, int doc_num)
+{
+  Explanation *expl;
+  mutex_lock(&self->store->ext_mutex);
+  ensure_searcher_open(self);
+  expl = sea_explain(self->sea, q, doc_num);
+  mutex_unlock(&self->store->ext_mutex);
+  return expl;
+}