ferret 0.3.2 → 0.9.0

data/ext/ferret.h

@@ -1,128 +1,62 @@
 #ifndef __FERRET_H_
 #define __FERRET_H_
 
-#include <ruby.h>
-
-#define BUFFER_SIZE 1024
-
-typedef unsigned char byte_t;
-
-typedef struct IndexBuffer {
-  long start;
-  int len;
-  int pos;
-  byte_t *buffer;
-} IndexBuffer;
-
-typedef struct Term {
-  VALUE field;
-  char *text;
-  int  tlen;
-} Term;
-
-typedef struct PriorityQueue {
-  VALUE *heap;
-  int len;
-  int size;
-} PriorityQueue;
-
-typedef struct TermInfo {
-  int doc_freq;
-  long freq_pointer;
-  long prox_pointer;
-  int skip_offset;
-} TermInfo;
-
-typedef struct RAMFile {
-  void  **buffers;
-  int   bufcnt;
-  VALUE mtime;
-  char  *name;
-  int   length;
-} RAMFile;
-
-typedef struct SegmentTermEnum {
-  VALUE input;
-  IndexBuffer *buf;
-  VALUE field_infos;
-  VALUE rtb_curr;
-  Term *tb_curr;
-  VALUE rtb_prev;
-  Term *tb_prev;
-  TermInfo *ti;
-  int is_index;
-  int size;
-  int position;
-  int index_pointer;
-  int index_interval;
-  int skip_interval;
-  int format;
-  int format_m1skip_interval;
-} SegmentTermEnum;
+#include "global.h"
+#include "document.h"
 
 /* IDs */
 extern ID id_new;
-extern ID id_close;
-extern ID id_size;
-extern ID id_iv_size;
 
 /* Modules */
 extern VALUE mFerret;
-extern VALUE mStore;
-extern VALUE mIndex;
-extern VALUE mUtils;
 extern VALUE mAnalysis;
+extern VALUE mDocument;
+extern VALUE mIndex;
 extern VALUE mSearch;
+extern VALUE mStore;
 extern VALUE mStringHelper;
+extern VALUE mUtils;
+extern VALUE mSpans;
 
 /* Classes */
-extern VALUE cRAMDirectory;
-extern VALUE cIndexIn;
-extern VALUE cBufferedIndexIn;
-extern VALUE cFSIndexIn;
-extern VALUE cIndexOut;
-extern VALUE cBufferedIndexOut;
-extern VALUE cFSIndexOut;
-extern VALUE cRAMIndexOut;
-extern VALUE cRAMIndexIn;
-extern VALUE cTerm;
-extern VALUE cTermBuffer;
-extern VALUE cTermInfo;
-extern VALUE cToken;
-extern VALUE cPriorityQueue;
-extern VALUE cSegmentMergeQueue;
-extern VALUE cTermEnum;
-extern VALUE cTermInfosReader;
-extern VALUE cSegmentTermEnum;
-extern VALUE cSimilarity;
-extern VALUE cDefaultSimilarity;
+extern VALUE cDirectory;
 
 /* Ferret Inits */
-extern void Init_indexio();
 extern void Init_term();
-extern void Init_term_info();
-extern void Init_term_infos_reader();
-extern void Init_term_buffer();
-extern void Init_priority_queue();
-extern void Init_token();
-extern void Init_segment_merge_queue();
-extern void Init_segment_term_enum();
-extern void Init_ram_directory();
-extern void Init_string_helper();
-extern void Init_similarity();
-
-/* External functions */
-extern int frt_hash(register char *p, register int len);
-extern unsigned long long frt_read_vint(VALUE self, IndexBuffer *my_buf);
-extern VALUE frt_indexin_read_long(VALUE self);
-extern VALUE frt_indexin_read_int(VALUE self);
-extern VALUE frt_indexin_seek(VALUE self, VALUE pos);
-extern VALUE frt_termbuffer_to_term(VALUE self);
-extern void frt_read_chars(VALUE self, char *buf, int offset, int len);
-extern void frt_write_bytes(VALUE self, byte_t *buf, int len);
-extern int frt_term_compare_to_int(VALUE self, VALUE rother);
-extern VALUE frt_termbuffer_init_copy(VALUE self, VALUE rother);
-extern VALUE frt_termbuffer_read(VALUE self, VALUE input, VALUE info);
-extern inline int frt_term_cmp(Term *t1, Term *t2);
+extern void Init_dir();
+extern void Init_analysis();
+extern void Init_doc();
+extern void Init_index_io();
+extern void Init_search();
+extern void Init_qparser();
+//extern void object_add(void *key, VALUE obj);
+#define object_add(key, obj) object_add2(key, obj,  __FILE__, __LINE__, __func__)
+extern void object_add2(void *key, VALUE obj, const char *file, int line, const char *func);
+//extern void object_del(void *key);
+#define object_del(key) object_del2(key,  __FILE__, __LINE__, __func__)
+extern void object_del2(void *key, const char *file, int line, const char *func);
+extern void frt_gc_mark(void *key);
+extern VALUE object_get(void *key);
+extern VALUE frt_data_alloc(VALUE klass);
+extern VALUE frt_get_doc(Document *doc);
+extern void frt_deref_free(void *p);
+
+
+#define Frt_Make_Struct(klass)\
+  rb_data_object_alloc(klass,NULL,(RUBY_DATA_FUNC)NULL,(RUBY_DATA_FUNC)NULL)
+
+#define Frt_Wrap_Struct(self,mmark,mfree,mdata)\
+  do {\
+    ((struct RData *)(self))->data = mdata;\
+    ((struct RData *)(self))->dmark = mmark;\
+    ((struct RData *)(self))->dfree = mfree;\
+  } while (0)
+
+#define Frt_Unwrap_Struct(self)\
+  do {\
+    ((struct RData *)(self))->data = NULL;\
+    ((struct RData *)(self))->dmark = NULL;\
+    ((struct RData *)(self))->dfree = NULL;\
+  } while (0)
 
 #endif

data/ext/field.c

@@ -0,0 +1,395 @@
+#include <index.h>
+#include <string.h>
+
+/****************************************************************************
+ *
+ * FieldInfo
+ *
+ ****************************************************************************/
+
+FieldInfo *fi_create(char *name, int number, bool is_indexed,
+    bool store_tv, bool store_pos, bool store_offset, bool omit_norms)
+{
+  FieldInfo *fi = ALLOC(FieldInfo);
+  fi->name = estrdup(name);
+  fi->number = number;
+  fi->is_indexed = is_indexed;
+  fi->store_tv = store_tv;
+  fi->store_offset = store_offset;
+  fi->store_pos = store_pos;
+  fi->omit_norms = omit_norms;
+  return fi;
+}
+
+void fi_destroy(void *p)
+{
+  FieldInfo *fi = (FieldInfo *)p;
+  free(fi->name);
+  free(fi);
+}
+
+/****************************************************************************
+ *
+ * FieldInfos
+ *
+ ****************************************************************************/
+
+FieldInfos *fis_create()
+{
+  FieldInfos *fis = ALLOC(FieldInfos);
+  fis->by_name = ht_create();
+  fis->by_number = NULL;
+  fis->fcnt = 0;
+  return fis;
+}
+
+FieldInfos *fis_open(Store *store, char *filename)
+{
+  FieldInfos *fis = fis_create();
+  InStream *is = store->open_input(store, filename);
+  fis_read(fis, is);
+  is_close(is);
+  return fis;
+}
+
+void fis_destroy(void *p)
+{
+  int i;
+  FieldInfos *fis = (FieldInfos *)p;
+  for (i = 0; i < fis->fcnt; i++) {
+    fi_destroy(fis->by_number[i]);
+  }
+  ht_destroy(fis->by_name);
+  free(fis->by_number);
+  free(fis);
+}
+
+FieldInfo *fis_add(FieldInfos *fis,
+    char *name,
+    bool is_indexed,
+    bool store_tv,
+    bool store_pos,
+    bool store_offset,
+    bool omit_norms)
+{
+  FieldInfo *fi = ht_get(fis->by_name, name);
+  if (fi == NULL) {
+    fi = fi_create(name, fis->fcnt, is_indexed, store_tv,
+        store_pos, store_offset, omit_norms);
+    fis->fcnt++;
+    REALLOC_N(fis->by_number, FieldInfo *, fis->fcnt);
+
+    fis->by_number[fi->number] = fi;
+    ht_set(fis->by_name, name, fi);
+  } else {
+    if (fi->is_indexed != is_indexed)
+      fi->is_indexed = true;             // once indexed, always index
+    if (fi->store_tv != store_tv)
+      fi->store_tv = true;               // once vector, always vector
+    if (fi->store_pos != store_pos)
+      fi->store_pos = true;              // once vector, always vector
+    if (fi->store_offset != store_offset)
+      fi->store_offset = true;           // once vector, always vector
+    if (fi->omit_norms != omit_norms)
+      fi->omit_norms = false;            // once kept, always keep
+  }
+  return fi;
+}
+
+void fis_add_fields(FieldInfos *fis,
+    HashSet *field_names,
+    bool is_indexed,
+    bool store_tv,
+    bool store_pos,
+    bool store_offset,
+    bool omit_norms)
+{
+  int i; 
+  for (i = 0; i < field_names->size; i++) {
+    fis_add(fis, field_names->elems[i], is_indexed, store_tv, store_pos,
+        store_offset, omit_norms);
+  }
+  hs_destroy(field_names);
+}
+
+bool fis_has_vectors(FieldInfos *fis)
+{
+  int i;
+  for (i = 0; i < fis->fcnt; i++) {
+    if (fis->by_number[i]->store_tv)
+      return true;
+  }
+  return false;
+}
+
+FieldInfo *fis_get_fi(FieldInfos *fis, char *name)
+{
+  return (FieldInfo *)ht_get(fis->by_name, name);
+}
+
+unsigned long long fis_get_number(FieldInfos *fis, char *name)
+{
+  FieldInfo *fi = (FieldInfo *)ht_get(fis->by_name, name);
+  if (fi == NULL)
+    return 0xFFFFFFFFull; // to be compatible with Jave version
+  else
+    return fi->number;
+}
+
+#define IS_INDEXED   0x01
+#define STORE_TV     0x02
+#define STORE_POS    0x04
+#define STORE_OFFSET 0x08
+#define OMIT_NORMS   0x10
+
+int fi_field_info_byte(FieldInfo *fi)
+{
+  int bits = 0x0;
+  if (fi->is_indexed)
+    bits |= IS_INDEXED;
+  if (fi->store_tv)
+    bits |= STORE_TV;
+  if (fi->store_pos)
+    bits |= STORE_POS;
+  if (fi->store_offset)
+    bits |= STORE_OFFSET;
+  if (fi->omit_norms)
+    bits |= OMIT_NORMS;
+  return bits;
+}
+
+void fis_write(FieldInfos *fis, Store *store, char *segment, char *ext)
+{
+  int i;
+  FieldInfo *fi;
+  char fname[SEGMENT_NAME_MAX_LENGTH];
+  strcpy(fname, segment);
+  strcat(fname, ext);
+  OutStream *os = store->create_output(store, fname);
+  os_write_vint(os, fis->fcnt);
+  for (i = 0; i < fis->fcnt; i++) {
+    fi = fis->by_number[i];
+    os_write_string(os, fi->name);
+    os_write_vint(os, fi_field_info_byte(fi));
+  }
+  os_close(os);
+}
+
+FieldInfos *fis_read(FieldInfos *fis, InStream *is)
+{
+  int i, size = is_read_vint(is); //read in the size
+  int bits, is_indexed, store_tv, store_pos, store_offset, omit_norms;
+  char *name;
+  for (i = 0; i < size; i++) {
+    name = is_read_string(is);
+    bits = is_read_byte(is);
+    is_indexed = (bits & IS_INDEXED) != 0;
+    store_tv = (bits & STORE_TV) != 0;
+    store_pos = (bits & STORE_POS) != 0;
+    store_offset = (bits & STORE_OFFSET) != 0;
+    omit_norms = (bits & OMIT_NORMS) != 0;
+    fis_add(fis, name, is_indexed, store_tv,
+        store_pos, store_offset, omit_norms);
+    free(name);
+  }
+  return fis;
+}
+
+FieldInfos *fis_add_doc(FieldInfos *fis, Document *doc)
+{
+  int i;
+  DocField *df;
+  for (i = 0; i < doc->dfcnt; i++) {
+    df = doc->df_arr[i];
+    fis_add(fis, df->name, df->is_indexed, df->store_tv,
+        df->store_pos, df->store_offset, df->omit_norms);
+  }
+  return fis;
+}
+
+/****************************************************************************
+ *
+ * FieldsWriter
+ *
+ ****************************************************************************/
+
+FieldsWriter *fw_open(Store *store, char *segment, FieldInfos *fis)
+{
+  char buf[SEGMENT_NAME_MAX_LENGTH];
+  int slen = strlen(segment);
+  strcpy(buf, segment);
+
+  FieldsWriter *fw = ALLOC(FieldsWriter);
+  fw->fis = fis;
+  strcpy(buf+slen, ".fdt");
+  fw->fields_out = store->create_output(store, buf);
+  strcpy(buf+slen, ".fdx");
+  fw->index_out = store->create_output(store, buf);
+  return fw;
+}
+
+void fw_close(FieldsWriter *fw)
+{
+  os_close(fw->fields_out);
+  os_close(fw->index_out);
+  free(fw);
+}
+
+void save_data(OutStream *fout, char *data, int dlen)
+{
+  os_write_vint(fout, dlen);
+  os_write_bytes(fout, (uchar *)data, dlen);
+}
+
+void fw_add_doc(FieldsWriter *fw, Document *doc)
+{
+  int i, bits;
+  OutStream *fout = fw->fields_out, *iout = fw->index_out;
+  os_write_long(iout, os_pos(fout));
+  DocField *df;
+  char *data;
+
+  int stored_count = 0;
+  for (i = 0; i < doc->dfcnt; i++) {
+    if (doc->df_arr[i]->is_stored)
+      stored_count++;
+  }
+  os_write_vint(fout, stored_count);
+  
+  for (i = 0; i < doc->dfcnt; i++) {
+    df = doc->df_arr[i];
+    if (df->is_stored) {
+      os_write_vint(fout, ((FieldInfo *)ht_get(fw->fis->by_name, df->name))->number);
+
+      bits = 0;
+      if (df->is_tokenized)
+        bits |= FIELD_IS_TOKENIZED;
+      if (df->is_binary)
+        bits |= FIELD_IS_BINARY;
+      if (df->is_compressed)
+        bits |= FIELD_IS_COMPRESSED;
+      os_write_byte(fout, bits);
+      
+      data = NULL;
+      if (df->is_compressed) {
+        // Not compressing just yet but we'll save it anyway
+        if (df->is_binary) {
+          save_data(fout, df->data, df->blen);
+        } else {
+          os_write_string(fout, df->data);
+        }
+      } else {
+        if (df->is_binary) {
+          save_data(fout, df->data, df->blen);
+        } else {
+          os_write_string(fout, df->data);
+        }
+      }
+    }
+  }
+}
+
+/****************************************************************************
+ *
+ * FieldsReader
+ *
+ ****************************************************************************/
+
+FieldsReader *fr_open(Store *store, char *segment, FieldInfos *fis)
+{
+  char buf[100];
+  int slen = strlen(segment);
+  strcpy(buf, segment);
+
+  FieldsReader *fr = ALLOC(FieldsReader);
+  fr->fis = fis;
+  strcpy(buf+slen, ".fdt");
+  fr->fields_in = store->open_input(store, buf);
+  strcpy(buf+slen, ".fdx");
+  InStream *iin = fr->index_in = store->open_input(store, buf);
+  fr->len = iin->length_internal(iin)/8;
+  return fr;
+}
+
+void fr_close(FieldsReader *fr)
+{
+  is_close(fr->fields_in);
+  is_close(fr->index_in);
+  free(fr);
+}
+
+Document *fr_get_doc(FieldsReader *fr, int doc_num)
+{
+  int i, bits, dlen;
+  char *data;
+  int store, index, stv;
+  int is_compressed, is_tokenized, is_binary;
+  Document *doc = doc_create();
+  InStream *iin = fr->index_in;
+  InStream *fin = fr->fields_in;
+  is_seek(iin, doc_num * 8);
+  int position = is_read_long(iin);
+  is_seek(fin, position);
+  int field_cnt = is_read_vint(fin);
+  int field_number;
+  FieldInfo *fi;
+
+  for (i = 0; i < field_cnt; i++) {
+    field_number = is_read_vint(fin);
+    fi = fr->fis->by_number[field_number];
+
+    bits = is_read_byte(fin);
+    
+    is_compressed = (bits & FIELD_IS_COMPRESSED) != 0;
+    is_tokenized = (bits & FIELD_IS_TOKENIZED) != 0;
+    is_binary = (bits & FIELD_IS_BINARY) != 0;
+    
+    if (is_binary) {
+      dlen = is_read_vint(fin);
+      data = ALLOC_N(char, dlen);
+      is_read_bytes(fin, (uchar *)data, 0, dlen);
+      if (is_compressed) {
+        doc_add_field(doc, df_create_binary(fi->name, data, dlen, DF_STORE_COMPRESS));
+      } else {
+        doc_add_field(doc, df_create_binary(fi->name, data, dlen, DF_STORE_YES));
+      }
+    } else {
+      store = DF_STORE_YES;
+      if (!fi->is_indexed) {
+        index = DF_INDEX_NO;
+      } else if (is_tokenized) {
+        index = DF_INDEX_TOKENIZED;
+      } else if (fi->omit_norms) {
+        index = DF_INDEX_NO_NORMS;
+      } else {
+        index = DF_INDEX_UNTOKENIZED;
+      }
+      data = NULL;
+      if (is_compressed) {
+        store = DF_STORE_COMPRESS;
+        dlen = is_read_vint(fin);
+        data = ALLOC_N(char, (dlen + 1));
+        data[dlen] = '\0';
+        is_read_bytes(fin, (uchar *)data, 0, dlen);
+      } else {
+        data = is_read_string(fin);
+      }
+      stv =  DF_TERM_VECTOR_NO;
+      if (fi->store_tv) { 
+        if (fi->store_pos && fi->store_offset) { 
+          stv =  DF_TERM_VECTOR_WITH_POSITIONS_OFFSETS;
+        } else if (fi->store_pos) { 
+          stv =  DF_TERM_VECTOR_WITH_POSITIONS;
+        } else if (fi->store_offset) { 
+          stv =  DF_TERM_VECTOR_WITH_OFFSETS;
+        } else { 
+          stv =  DF_TERM_VECTOR_YES;
+        }
+      }
+      doc_add_field(doc, df_create(fi->name, data, store, index, stv));
+    }
+  }
+
+  return doc;
+}
+