ferret 0.3.2 → 0.9.0

data/ext/document.c

@@ -0,0 +1,336 @@
+#include <document.h>
+#include <string.h>
+
+/****************************************************************************
+ *
+ * DocField
+ *
+ ****************************************************************************/
+
+inline void df_set(DocField *df, const char *name,
+    char *data, int store, int index, int tv)
+{
+  if ((index == DF_INDEX_NO) && (store == DF_STORE_NO))
+    eprintf(ARG_ERROR,
+        "it doesn't make sense to have a field that is neither indexed nor stored");
+  if ((index == DF_INDEX_NO) && (tv != DF_TERM_VECTOR_NO))
+    eprintf(ARG_ERROR,
+        "cannot store term vector information for a field that is not indexed");
+  df->name = estrdup(name);
+  df->data = data;
+  df->blen = strlen(data);
+  df_set_store(df, store);
+  df_set_index(df, index);
+  df_set_term_vector(df, tv);
+  df->is_binary = false;
+  df->boost = 1.0;
+}
+
+DocField *df_create(const char *name, char *data, int store, int index, int tv)
+{
+  DocField *df = ALLOC(DocField);
+  df_set(df, name, data, store, index, tv);
+  return df;
+}
+
+DocField *df_clone(DocField *self)
+{
+  DocField *clone = ALLOC(DocField);
+  memcpy(clone, self, sizeof(DocField));
+  clone->name = estrdup(self->name);
+  clone->data = estrdup(self->data);
+  return clone;
+}
+
+void df_destroy(void *p)
+{
+  DocField *df = (DocField *)p;
+  free(df->name);
+  free(p);
+}
+
+void df_destroy_data(void *p)
+{
+  DocField *df = (DocField *)p;
+  free(df->data);
+  free(df->name);
+  free(p);
+}
+
+void df_set_store(DocField *df, int store)
+{
+  switch (store) {
+    case DF_STORE_YES:
+      df->is_stored = true;
+      df->is_compressed = false;
+      break;
+    case DF_STORE_NO:
+      df->is_stored = false;
+      df->is_compressed = false;
+      break;
+    case DF_STORE_COMPRESS:
+      df->is_stored = true;
+      df->is_compressed = true;
+      break;
+    default:
+      eprintf(ARG_ERROR, "Invalid value %d for store in document field", store);
+  }
+}
+
+void df_set_index(DocField *df, int index)
+{
+  df->omit_norms = false;
+  switch (index) {
+    case DF_INDEX_NO:
+      df->is_indexed = false;
+      df->is_tokenized = false;
+      break;
+    case DF_INDEX_TOKENIZED:
+      df->is_indexed = true;
+      df->is_tokenized = true;
+      break;
+    case DF_INDEX_UNTOKENIZED:
+      df->is_indexed = true;
+      df->is_tokenized = false;
+      break;
+    case DF_INDEX_NO_NORMS:
+      df->is_indexed = true;
+      df->is_tokenized = false;
+      df->omit_norms = true;
+      break;
+    default:
+      eprintf(ARG_ERROR, "Invalid value %d for index in document field", index);
+  }
+}
+
+void df_set_term_vector(DocField *df, int tv)
+{
+  switch (tv) {
+    case DF_TERM_VECTOR_NO:
+      df->store_tv = false;
+      df->store_offset = false;
+      df->store_pos = false;
+      break;
+    case DF_TERM_VECTOR_YES:
+      df->store_tv = true;
+      df->store_offset = false;
+      df->store_pos = false;
+      break;
+    case DF_TERM_VECTOR_WITH_OFFSETS:
+      df->store_tv = true;
+      df->store_offset = true;
+      df->store_pos = false;
+      break;
+    case DF_TERM_VECTOR_WITH_POSITIONS:
+      df->store_tv = true;
+      df->store_offset = false;
+      df->store_pos = true;
+      break;
+    case DF_TERM_VECTOR_WITH_POSITIONS_OFFSETS:
+      df->store_tv = true;
+      df->store_offset = true;
+      df->store_pos = true;
+      break;
+    default:
+      eprintf(ARG_ERROR,
+          "Invalid value %d for term_vector in document field", tv);
+  }
+}
+
+DocField *df_create_binary(char *name, char *data, int blen, int store)
+{
+  if (store == DF_STORE_NO) {
+    eprintf(ARG_ERROR, "It doesn't make sense not to store binary data\n");
+  }
+  DocField *df = df_create(name, data, store, DF_INDEX_NO, DF_TERM_VECTOR_NO);
+  df->is_binary = true;
+  df->blen = blen;
+  return df;
+}
+
+char *df_to_s(DocField *self)
+{
+  /* the length of the str is name.len + data.len + 119, add safety 10 */
+  char *str = ALLOC_N(char, strlen(self->name) + strlen(self->data) + 129);
+  char *str_ptr = str;
+
+  if (self->is_stored) {
+    sprintf(str, "stored/%s,", self->is_compressed ?  "compressed" : "uncompressed");
+    str_ptr = str + strlen(str);
+  }
+  sprintf(str_ptr, "%s%s%s%s%s%s%s<%s:%s>",
+      self->is_indexed ? "indexed," : "",
+      self->is_tokenized ? "tokenized," : "",
+      self->store_tv ? "store_term_vector," : "",
+      self->store_offset ? "store_offsets," : "",
+      self->store_pos ? "store_positions," : "",
+      self->omit_norms ? "omit_norms," : "",
+      self->is_binary ? "binary," : "",
+      self->name,
+      self->is_binary ? "=bin_data=" : self->data);
+
+  return str;
+}
+
+/****************************************************************************
+ *
+ * Document
+ *
+ ****************************************************************************/
+
+Document *doc_create()
+{
+  Document *doc = ALLOC(Document);
+  doc->fields = h_new_str(&free, &ary_destroy);
+  doc->fcnt = 0;
+  doc->dfcnt = 0;
+  doc->field_arr = NULL;
+  doc->df_arr = NULL;
+  doc->boost = 1.0;
+  doc->free_data = &df_destroy_data;
+  return doc;
+}
+
+Document *doc_create_keep_data()
+{
+  Document *doc = doc_create();
+  doc->free_data = df_destroy;
+  return doc;
+}
+
+void doc_destroy(void *p)
+{
+  Document *doc = (Document *)p;
+  free(doc->field_arr);
+  free(doc->df_arr);
+  h_destroy(doc->fields);
+  free(doc);
+}
+
+void doc_add_field(Document *doc, DocField *df)
+{
+  Array *field_ga = (Array *)h_get(doc->fields, df->name);
+  if (field_ga == NULL) {
+    field_ga = ary_create(1, doc->free_data);
+    h_set(doc->fields, estrdup(df->name), field_ga);
+    doc->fcnt++;
+    REALLOC_N(doc->field_arr, Array *, doc->fcnt);
+    doc->field_arr[doc->fcnt-1] = field_ga;
+  }
+  ary_append(field_ga, df);
+  doc->dfcnt++;
+  REALLOC_N(doc->df_arr, DocField *, doc->dfcnt);
+  doc->df_arr[doc->dfcnt-1] = df;
+}
+
+DocField *doc_get_field(Document *doc, const char *fname)
+{
+  Array *field_ga = (Array *)h_get(doc->fields, fname);
+  if (field_ga) {
+    return field_ga->elems[0];
+  } else {
+    return NULL;
+  }
+}
+
+Array *doc_get_fields(Document *doc, const char *fname)
+{
+  return (Array *)h_get(doc->fields, fname);
+}
+
+/**
+ * TODO:
+ * This is not exactly elegant or efficient but it works and is not going to
+ * be a performance problem. Still, it would be nice to make the code a little
+ * clearer.
+ */
+Array *doc_remove_fields(Document *doc, const char *fname)
+{
+  Array *field_ga = (Array *)h_rem(doc->fields, fname, true);
+  if (field_ga) {
+    int i, j;
+    doc->fcnt--;
+    for (i = 0; i < doc->fcnt; i++) {
+      if (field_ga == doc->field_arr[i]) {
+        memmove(&doc->field_arr[i],
+                &doc->field_arr[i+1],
+                sizeof(void *) * (doc->fcnt - i));
+        break;
+      }
+    }
+    for (i = 0, j = 0; i < doc->dfcnt && j < field_ga->size;) {
+      if (field_ga->elems[j] == doc->df_arr[i]) {
+        memmove(&doc->df_arr[i],
+                &doc->df_arr[i+1],
+                sizeof(void *) * (doc->dfcnt - i - 1));
+        j++;
+        doc->dfcnt--;
+      } else {
+        i++;
+      }
+    }
+    return field_ga;
+  } else {
+    return NULL;
+  }
+}
+
+DocField *doc_remove_field(Document *doc, const char *fname)
+{
+  DocField *df = NULL;
+  Array *dfs = (Array *)h_get(doc->fields, fname);
+  if (dfs) {
+    df = ary_remove(dfs, 0);
+    if (dfs->size == 0) {
+      Array *fields = doc_remove_fields(doc, fname);
+      ary_destroy(fields);
+    } else {
+      int i;
+      for (i = 0; i < doc->dfcnt; i++) {
+        if (df == doc->df_arr[i]) {
+          memmove(&doc->df_arr[i],
+                  &doc->df_arr[i+1],
+                  sizeof(void *) * (doc->dfcnt - i - 1));
+          doc->dfcnt--;
+          break;
+        }
+      }
+    }
+  }
+  return df;
+}
+
+bool doc_delete_fields(Document *doc, const char *fname)
+{
+  Array *field_ga = doc_remove_fields(doc, fname);
+  if (field_ga) {
+    ary_destroy(field_ga);
+    return true;
+  } else {
+    return false;
+  }
+  return h_del(doc->fields, fname);
+}
+
+char *doc_to_s(Document *doc)
+{
+  int i, len = 20;
+  char *str, *str_ptr;
+  char **df_strs = ALLOC_N(char *, doc->dfcnt);
+  for (i = 0; i < doc->dfcnt; i++) {
+    df_strs[i] = df_to_s(doc->df_arr[i]);
+    len += strlen(df_strs[i]) + 3;
+  }
+  str_ptr = str = ALLOC_N(char, len);
+  sprintf(str_ptr, "Document {\n");
+  str_ptr += strlen(str_ptr);
+  for (i = 0; i < doc->dfcnt; i++) {
+    sprintf(str_ptr, "  %s\n", df_strs[i]);
+    free(df_strs[i]);
+    str_ptr += strlen(str_ptr);
+  }
+  sprintf(str_ptr, "}");
+  free(df_strs);
+
+  return str;
+}

data/ext/document.h

@@ -0,0 +1,87 @@
+#ifndef FRT_DOCUMENT_H
+#define FRT_DOCUMENT_H
+
+#include <global.h>
+#include <hash.h>
+#include <array.h>
+
+/****************************************************************************
+ *
+ * DocField
+ *
+ ****************************************************************************/
+enum {
+  DF_STORE_YES = 0,
+  DF_STORE_NO = 1,
+  DF_STORE_COMPRESS = 2
+};
+
+enum {
+  DF_INDEX_UNTOKENIZED = 0,
+  DF_INDEX_TOKENIZED = 1,
+  DF_INDEX_NO = 2,
+  DF_INDEX_NO_NORMS = 3
+};
+
+enum {
+  DF_TERM_VECTOR_NO = 0,
+  DF_TERM_VECTOR_YES = 1,
+  DF_TERM_VECTOR_WITH_POSITIONS = 2,
+  DF_TERM_VECTOR_WITH_OFFSETS = 3,
+  DF_TERM_VECTOR_WITH_POSITIONS_OFFSETS = 4
+};
+
+typedef struct DocField {
+  char  *name;
+  char  *data;
+  int blen; // This is used for binary fields only to store the data length
+  float boost;
+  bool is_stored : 1;
+  bool is_compressed : 1;
+  bool is_indexed : 1;
+  bool is_tokenized : 1;
+  bool store_tv : 1;
+  bool store_pos : 1;
+  bool store_offset : 1;
+  bool omit_norms : 1;
+  bool is_binary : 1;
+} DocField;
+
+DocField *df_create(const char *name, char *data, int store, int index, int tv);
+DocField *df_clone(DocField *self);
+void df_set(DocField *df, const char *name, char *data, int store, int index, int tv);
+void df_destroy(void *p);
+void df_destroy_data(void *p);
+void df_set_store(DocField *df, int store);
+void df_set_index(DocField *df, int index);
+void df_set_term_vector(DocField *df, int tv);
+char *df_to_s(DocField *df);
+DocField *df_create_binary(char *name, char *data, int blen, int store);
+
+/****************************************************************************
+ *
+ * Document
+ *
+ ****************************************************************************/
+
+typedef struct Document {
+  HshTable *fields;
+  Array **field_arr;
+  int fcnt;
+  DocField **df_arr;
+  int dfcnt;
+  float boost;
+  void (*free_data)(void *p);
+} Document;
+
+Document *doc_create();
+Document *doc_create_keep_data();
+void doc_destroy(void *p);
+void doc_add_field(Document *doc, DocField *df);
+DocField *doc_get_field(Document *doc, const char *fname);
+Array *doc_get_fields(Document *doc, const char *fname);
+Array *doc_remove_fields(Document *doc, const char *fname);
+DocField *doc_remove_field(Document *doc, const char *fname);
+bool doc_delete_fields(Document *doc, const char *fname);
+char *doc_to_s(Document *doc);
+#endif

data/ext/ferret.c

@@ -1,73 +1,114 @@
 #include "ferret.h"
+#include "hash.h"
+
+/* Object Map */
+static HshTable *object_map;
 
 /* IDs */
 ID id_new;
-ID id_close;
-ID id_size;
-ID id_iv_size;
 
 /* Modules */
 VALUE mFerret;
-VALUE mStore;
-VALUE mIndex;
-VALUE mUtils;
 VALUE mAnalysis;
+VALUE mDocument;
+VALUE mIndex;
 VALUE mSearch;
+VALUE mStore;
 VALUE mStringHelper;
+VALUE mUtils;
+VALUE mSpans;
 
 /* Classes */
-VALUE cRAMDirectory;
-VALUE cIndexIn;
-VALUE cBufferedIndexIn;
-VALUE cFSIndexIn;
-VALUE cIndexOut;
-VALUE cBufferedIndexOut;
-VALUE cFSIndexOut;
-VALUE cRAMIndexOut;
-VALUE cRAMIndexIn;
-VALUE cTerm;
-VALUE cTermBuffer;
-VALUE cTermInfo;
-VALUE cToken;
-VALUE cPriorityQueue;
-VALUE cSegmentMergeQueue;
-VALUE cSegmentTermEnum;
-VALUE cTermEnum;
-VALUE cTermInfosReader;
-VALUE cSimilarity;
-VALUE cDefaultSimilarity;
+/*
+*/
+
+
+unsigned int
+object_hash(const void *key)
+{
+  return (unsigned int)key;
+}
+
+int
+object_eq(const void *key1, const void *key2)
+{
+  return key1 == key2;
+}
+
+VALUE
+object_get(void *key)
+{
+  VALUE val = (VALUE)h_get(object_map, key);
+  if (!val) val = Qnil;
+  return val;
+}
+
+//static int hash_cnt = 0;
+void
+//object_add(void *key, VALUE obj)
+object_add2(void *key, VALUE obj, const char *file, int line, const char *func)
+{
+  if (h_get(object_map, key))
+    printf("failed adding %d. %s:%d:%s\n", (int)key, file, line, func);
+  //printf("adding %d. now contains %d %s:%d:%s\n", (int)key, ++hash_cnt, file, line, func);
+  h_set(object_map, key, (void *)obj);
+}
+
+void
+//object_del(void *key)
+object_del2(void *key, const char *file, int line, const char *func)
+{
+  if (object_get(key) == Qnil) 
+    printf("failed deleting %d. %s:%d:%s\n", (int)key, file, line, func);
+  //printf("deleting %d. now contains %d, %s:%d:%s\n", (int)key, --hash_cnt, file, line, func);
+  h_del(object_map, key);
+}
+
+void
+frt_gc_mark(void *key)
+{
+  VALUE val = (VALUE)h_get(object_map, key);
+  if (val)
+    rb_gc_mark(val);
+}
+
+VALUE
+frt_data_alloc(VALUE klass)
+{
+  return Frt_Make_Struct(klass);
+}
+
+void
+frt_deref_free(void *p)
+{
+  object_del(p);
+}
 
 void
 Init_ferret_ext(void)
 {
+  /* initialize object map */
+  object_map = h_new(&object_hash, &object_eq, NULL, NULL);
+
   /* IDs */
-  id_new = rb_intern("new");
-  id_close = rb_intern("close");
-  id_size = rb_intern("size");
-  id_iv_size = rb_intern("@size");
+	id_new = rb_intern("new");
 
   /* Modules */
   mFerret = rb_define_module("Ferret");
-  mStore = rb_define_module_under(mFerret, "Store");
-  mIndex = rb_define_module_under(mFerret, "Index");
-  mUtils = rb_define_module_under(mFerret, "Utils");
   mAnalysis = rb_define_module_under(mFerret, "Analysis");
+  mDocument = rb_define_module_under(mFerret, "Document");
+  mIndex = rb_define_module_under(mFerret, "Index");
   mSearch = rb_define_module_under(mFerret, "Search");
-
-  /* Classes */
-  cTermEnum = rb_define_class_under(mIndex, "TermEnum", rb_cObject);
+  mStore = rb_define_module_under(mFerret, "Store");
+  mUtils = rb_define_module_under(mFerret, "Utils");
+  mSpans = rb_define_module_under(mSearch, "Spans");
 
   /* Inits */
-  Init_indexio();
   Init_term();
-  Init_term_buffer();
-  Init_term_info();
-  Init_term_infos_reader();
-  Init_token();
-  Init_priority_queue();
-  Init_segment_merge_queue();
-  Init_segment_term_enum();
-  Init_ram_directory();
-  Init_string_helper();
-  Init_similarity();
+  Init_analysis();
+  Init_doc();
+  Init_dir();
+  Init_index_io();
+  Init_search();
+  Init_qparser();
 }