ferret 0.3.2 → 0.9.0

data/ext/store.c

@@ -0,0 +1,35 @@
+#include "store.h"
+
+void with_lock(Lock *lock, void (*func)(void *arg), void *arg)
+{
+  if (!lock->obtain(lock))
+    eprintf(IO_ERROR, "Could not obtain lock: <%s>", lock->name);
+  func(arg);
+  lock->release(lock);
+}
+
+void with_lock_name(Store *store, char *lock_name,
+    void (*func)(void *arg), void *arg)
+{
+  Lock *lock = store->open_lock(store, lock_name);
+  if (!lock->obtain(lock))
+    eprintf(IO_ERROR, "Could not obtain lock: <%s>", lock->name);
+  func(arg);
+  lock->release(lock);
+  store->close_lock(lock);
+}
+
+Store *store_create()
+{
+  Store *store = ALLOC(Store);
+  mutex_init(&store->mutex, NULL);
+  mutex_init(&store->ext_mutex, NULL);
+  return store;
+}
+
+void store_destroy(Store *store)
+{
+  mutex_destroy(&store->mutex);
+  mutex_destroy(&store->ext_mutex);
+  free(store);
+}

data/ext/store.h

@@ -0,0 +1,152 @@
+#include "global.h"
+#include "hash.h"
+
+#ifndef FRT_STORE_H
+#define FRT_STORE_H
+
+#define BUFFER_SIZE 1024
+#define LOCK_PREFIX "ferret-"
+
+#define VINT_MAX_LEN 10
+#define VINT_END BUFFER_SIZE - VINT_MAX_LEN
+
+typedef struct Buffer {
+  uchar buf[BUFFER_SIZE];
+  int start;
+  int pos;
+  int len;
+} Buffer;
+
+typedef struct OutStream {
+  Buffer buf;
+  void   *file;
+  int   pointer; // only used by RAMOut
+  void   (*flush_internal)(struct OutStream *os, uchar *buf, int len);
+  void   (*seek_internal)(struct OutStream *os, int pos);
+  void   (*close_internal)(struct OutStream *os);
+} OutStream;
+
+typedef struct CompoundInStream CompoundInStream;
+
+typedef struct InStream {
+  int    is_clone;
+  Buffer buf;
+  void   *file;
+  union {
+    int   pointer; // only used by RAMIn
+    char *path; // only used by FSIn
+    CompoundInStream *cis;
+  } d;
+  void   (*read_internal)(struct InStream *is, uchar *buf, int offset, int len);
+  void   (*seek_internal)(struct InStream *is, int pos);
+  void   (*close_internal)(struct InStream *is);
+  void   (*clone_internal)(struct InStream *is, struct InStream *new_index_i);
+  int    (*length_internal)(struct InStream *is);
+} InStream;
+
+struct CompoundInStream {
+  InStream *sub;
+  int offset;
+  int length;
+};
+
+#define is_length(mis) mis->length_internal(mis)
+
+typedef struct Store Store;
+typedef struct Lock Lock;
+struct Lock {
+  char *name;
+  Store *store;
+  int (*obtain)(Lock *lock);
+  int (*is_locked)(Lock *lock);
+  void (*release)(Lock *lock);
+};
+
+typedef struct CompoundStore {
+  Store *store;
+  const char *name;
+  HshTable *entries;
+  InStream *stream;
+} CompoundStore;
+
+struct Store {
+  int ref_cnt; /* for fs_store only */
+  mutex_t mutex;
+  mutex_t ext_mutex;
+  union {
+    char *path;          /* for fs_store only */
+    HshTable *ht;        /* for ram_store only */
+    CompoundStore *cmpd; /* for compound_store only */
+  } dir;
+  void (*touch)(Store *store, char *filename);
+  int  (*exists)(Store *store, char *filename);
+  int  (*remove)(Store *store, char *filename);
+  int  (*rename)(Store *store, char *from, char *to);
+  int  (*count)(Store *store);
+  void (*close)(Store *store);
+  void (*clear)(Store *store);
+  void (*clear_all)(Store *store);
+  void (*clear_locks)(Store *store);
+  int (*length)(Store *store, char *filename);
+  void (*each)(Store *store, void (*func)(char *fname, void *arg), void *arg);
+  OutStream *(*create_output)(Store *store, const char *filename);
+  InStream *(*open_input)(Store *store, const char *filename);
+  Lock *(*open_lock)(Store *store, char *lockname);
+  void (*close_lock)(Lock *lock);
+};
+
+#define store_close(mstore) mstore->close(mstore)
+
+Store *store_create();
+void store_destroy(Store *store);
+Store *open_fs_store(const char *pathname);
+Store *open_ram_store();
+Store *open_ram_store_and_copy(Store *store, bool close_dir);
+Store *open_cmpd_store(Store *sub, const char *filename);
+void ram_close(Store *store);
+Buffer *buf_create();
+void os_flush(OutStream *os);
+void os_close(OutStream *os);
+int os_pos(OutStream *os);
+void os_seek(OutStream *os, int new_pos);
+void os_write_byte(OutStream *os, uchar b);
+void os_write_bytes(OutStream *os, uchar *b, int len);
+uchar is_read_byte(InStream *is);
+int is_pos(InStream *is);
+uchar *is_read_bytes(InStream *is, uchar *b, int offset, int len);
+void is_seek(InStream *is, int pos);
+InStream *is_clone(InStream *is);
+void is_close(InStream *is);
+int is_read_int(InStream *is);
+long long is_read_long(InStream *is);
+unsigned int is_read_uint(InStream *is);
+unsigned long long is_read_ulong(InStream *is);
+unsigned long long is_read_vint(InStream *is);
+void is_read_chars(InStream *is, char* buffer, int off, int len) ;
+char *is_read_string(InStream *is);
+void os_write_int(OutStream *os, int l);
+void os_write_long(OutStream *os, long long l);
+void os_write_uint(OutStream *os, unsigned int l);
+void os_write_ulong(OutStream *os, unsigned long long l);
+void os_write_vint(OutStream *os, register unsigned long long i);
+void os_write_chars(OutStream *os, char *buf, int start, int length);
+void os_write_string(OutStream *os, char *str);
+OutStream *os_create();
+InStream *is_create();
+void buf_destroy(Buffer *buf);
+
+// RamStore functions
+int ramo_length(OutStream *os);
+void ramo_reset(OutStream *os);
+int rami_length(InStream *is);
+void ramo_write_to(OutStream *os, OutStream *other_o);
+OutStream *ram_create_buffer();
+void ram_destroy_buffer(OutStream *os);
+
+int file_is_lock(char *filename);
+
+void with_lock(Lock *lock, void (*func)(void *arg), void *arg);
+void with_lock_name(Store *store, char *lock_name,
+    void (*func)(void *arg), void *arg);
+
+#endif

data/ext/term.c

@@ -1,222 +1,783 @@
-#include "ferret.h"
+#include <index.h>
+#include <string.h>
+#include <helper.h>
+#include <hash.h>
 
+/****************************************************************************
+ *
+ * Term
+ *
+ ****************************************************************************/
+
+Term *term_clone(Term *term)
+{
+  Term *t = ALLOC(Term);
+
+  t->field = term->field;
+  t->text = estrdup(term->text);
+  return t;
+}
+
+Term *term_create(const char *field, char *text)
+{
+  Term *t = ALLOC(Term);
+
+  t->field = (char *)field;
+  t->text = estrdup(text);
+  return t;
+}
+
+void term_destroy(void *p)
+{
+  Term *t = (Term *)p;
+  free(t->text);
+  free(t);
+}
+
+int term_cmp(void *t1, void *t2)
+{
+  int res = strcmp(((Term *)t1)->field, ((Term *)t2)->field);
+  if (res != 0) {
+    return res;
+  } else {
+    return strcmp(((Term *)t1)->text, ((Term *)t2)->text);
+  }
+}
+
+int term_eq(const void *t1, const void *t2)
+{
+  return (strcmp(((Term *)t1)->text, ((Term *)t2)->text)) == 0 &&
+    (strcmp(((Term *)t1)->field, ((Term *)t2)->field) == 0);
+}
+
+unsigned int term_hash(const void *t)
+{
+  return str_hash(((Term *)t)->text) * str_hash(((Term *)t)->field);
+}
+
+char *term_to_s(Term *term)
+{
+  char *string = ALLOC_N(char, strlen(term->field) + strlen(term->text) + 2);
+  sprintf(string, "%s:%s", term->field, term->text);
+  return string;
+}
 
 /****************************************************************************
  *
- * Term Methods
+ * TermBuffer
  *
  ****************************************************************************/
 
-void
-frt_term_free(void *p)
+void tb_reset(TermBuffer *tb)
+{
+  tb->field = (char *)EMPTY_STRING;
+  tb->text[0] = '\0';
+}
+
+TermBuffer *tb_create()
+{
+  TermBuffer *tb = ALLOC(TermBuffer);
+  tb->field = (char *)EMPTY_STRING;
+  tb->text[0] = '\0';
+  return tb;
+}
+
+void tb_destroy(void *p)
 {
-  Term *term = (Term *)p;
-  free(term->text);
   free(p);
 }
 
-void
-frt_term_mark(void *p)
+TermBuffer *tb_set_term(TermBuffer *tb, Term *t)
+{
+  tb->field = t->field;
+  strcpy(tb->text, t->text);
+  return tb;
+}
+
+Term *tb_get_term(TermBuffer *tb)
 {
-  Term *term = (Term *)p;
-  rb_gc_mark(term->field);
+  return term_create(tb->field, tb->text);
 }
 
-static VALUE
-frt_term_alloc(VALUE klass)
+int tb_cmp(TermBuffer *tb1, TermBuffer *tb2)
 {
-  Term *term = ALLOC(Term);
-  MEMZERO(term, Term, 1);
-  term->field = Qnil;
-  return Data_Wrap_Struct(klass, frt_term_mark, frt_term_free, term);
+  int res = strcmp(tb1->field, tb2->field);
+  if (res != 0) {
+    return res;
+  } else {
+    return strcmp(tb1->text, tb2->text);
+  }
 }
 
-#define GET_TERM Term *term; Data_Get_Struct(self, Term, term)
-VALUE 
-frt_term_set(VALUE self, VALUE rfield, VALUE rtext)
+int tb_term_cmp(TermBuffer *tb, Term *t)
 {
-  int tlen;
-  GET_TERM;
+  int res = strcmp(tb->field, t->field);
+  if (res != 0) {
+    return res;
+  } else {
+    return strcmp(tb->text, t->text);
+  }
+}
 
-  tlen = RSTRING(rtext)->len;
-  term->field = rfield;
-  REALLOC_N(term->text, char, tlen + 1);
-  MEMCPY(term->text, RSTRING(rtext)->ptr, char, tlen);
-  term->tlen = tlen;
-  
-  return Qnil;
+TermBuffer *tb_cpy(TermBuffer *tb1, TermBuffer *tb2)
+{
+  tb1->field = tb2->field;
+  strcpy(tb1->text, tb2->text);
+  return tb1;
 }
 
-static VALUE 
-frt_term_init(VALUE self, VALUE rfield, VALUE rtext)
+TermBuffer *tb_read(TermBuffer *tb, InStream *is, FieldInfos *fis)
 {
-  frt_term_set(self, rfield, rtext);
-  return self;
+  int start = is_read_vint(is);
+  int length = is_read_vint(is);
+  int total_length = start + length;
+  is_read_bytes(is, (uchar *)tb->text, start, length);
+  tb->text[total_length] = '\0';
+  int fnum = is_read_vint(is);
+  if (fnum < 0)
+    tb->field = (char *)EMPTY_STRING;
+  else
+    tb->field = fis->by_number[fnum]->name;
+  return tb;
 }
 
-static VALUE 
-frt_term_get_text(VALUE self)
+/****************************************************************************
+ *
+ * TermInfo
+ *
+ ****************************************************************************/
+
+TermInfo *ti_create(int doc_freq, int freq_pointer, int prox_pointer, int skip_offset)
 {
-  GET_TERM;
-  return rb_str_new(term->text, term->tlen);
+  TermInfo *ti = ALLOC(TermInfo);
+  ti->doc_freq = doc_freq;
+  ti->freq_pointer = freq_pointer;
+  ti->prox_pointer = prox_pointer;
+  ti->skip_offset = skip_offset;
+  return ti;
 }
 
-static VALUE 
-frt_term_set_text(VALUE self, VALUE rtext)
+TermInfo *ti_set(TermInfo *ti, int doc_freq, int freq_pointer, int prox_pointer, int skip_offset)
 {
-  int tlen;
-  char *text;
-  GET_TERM;
-  tlen = RSTRING(rtext)->len;
-  text = RSTRING(rtext)->ptr;
+  ti->doc_freq = doc_freq;
+  ti->freq_pointer = freq_pointer;
+  ti->prox_pointer = prox_pointer;
+  ti->skip_offset = skip_offset;
+  return ti;
+}
 
-  REALLOC_N(term->text, char, tlen + 1);
-    
-  MEMCPY(term->text, text, char, tlen);
-  term->tlen = tlen;
-    
-  return Qnil;
+void ti_destroy(void *p)
+{
+  free(p);
 }
 
-static VALUE 
-frt_term_get_field(VALUE self)
+TermInfo *ti_cpy(TermInfo *ti, TermInfo *other)
 {
-  GET_TERM;
-  return term->field;
+  memcpy(ti, other, sizeof(TermInfo));
+  return ti;
 }
 
-static VALUE 
-frt_term_set_field(VALUE self, VALUE rfield)
+TermInfo *ti_clone(TermInfo *other)
 {
-  GET_TERM;
-  term->field = rfield;
-  return Qnil;
+  return ti_create(other->doc_freq,
+      other->freq_pointer, other->prox_pointer, other->skip_offset);
 }
 
-VALUE 
-frt_term_to_s(VALUE self)
+int ti_eq(TermInfo *ti, TermInfo *other)
 {
-  int tlen, flen;
-  char delim[] = ":";
-  char *res;
-  GET_TERM;
-  tlen = term->tlen;
-  flen = RSTRING(term->field)->len;
-  res = alloca(flen + tlen + 1);    
+  return (memcmp(ti, other, sizeof(TermInfo)) == 0);
+}
+
+/****************************************************************************
+ *
+ * TermEnum
+ *
+ ****************************************************************************/
+
+TermEnum *te_create()
+{
+  TermEnum *te = ALLOC(TermEnum);
+  te->tb_curr = tb_create();
+  te->tb_prev = tb_create();
+  te->ti_curr = ti_create(0, 0, 0, 0);
+  return te;
+}
+
+void te_destroy(void *p)
+{
+  TermEnum *te = (TermEnum *)p;
+  tb_destroy(te->tb_curr);
+  tb_destroy(te->tb_prev);
+  ti_destroy(te->ti_curr);
+  free(p);
+}
+
+Term *te_get_term(TermEnum *te)
+{
+  return tb_get_term(te->tb_curr);
+}
+
+TermInfo *te_get_ti(TermEnum *te)
+{
+  TermInfo *ti = te->ti_curr;
+  return ti_create(ti->doc_freq, ti->freq_pointer, ti->prox_pointer, ti->skip_offset);
+}
+
+TermBuffer *te_skip_to(TermEnum *te, Term *t)
+{
+  TermBuffer *tb_curr;
+  if (tb_term_cmp(te->tb_curr, t) == 0)
+    return te->tb_curr;
+
+  while (((tb_curr = te->next(te)) != NULL) &&
+      (tb_term_cmp(tb_curr, t) < 0)) {
+  }
+  return tb_curr;
+}
+
+/****************************************************************************
+ *
+ * SegmentTermEnum
+ *
+ ****************************************************************************/
+
+#define GET_STE SegmentTermEnum *ste = (SegmentTermEnum *)te->data;
+
+TermBuffer *ste_next(TermEnum *te)
+{
+  GET_STE;
+  InStream *is = ste->is;
+  ste->pos++;
+  if (ste->pos > ste->size - 1) {
+    tb_reset(te->tb_curr);
+    return NULL;
+  }
+
+  tb_cpy(te->tb_prev, te->tb_curr);
+  tb_read(te->tb_curr, is, ste->fis);
   
-  MEMCPY(res, StringValuePtr(term->field), char, flen);
-  MEMCPY(res + flen, delim, char, 1);
-  MEMCPY(res + flen + 1, term->text, char, tlen);
-  return rb_str_new(res, tlen + flen + 1 );
+  TermInfo *ti = te->ti_curr;
+  ti->doc_freq = is_read_vint(is);         // read doc freq
+  ti->freq_pointer += is_read_vint(is);    // read freq pointer
+  ti->prox_pointer += is_read_vint(is);    // read prox pointer
+  
+  if (ste->format == -1) {
+    // just read skip_offset in order to increment  file pointer
+    // value is never used since skip_to is switched off
+    if (!ste->is_index) {
+      if (ti->doc_freq > ste->format_m1skip_interval)
+        ti->skip_offset = is_read_vint(is);
+    }
+  } else {
+    if (ti->doc_freq >= ste->skip_interval)
+      ti->skip_offset = is_read_vint(is);
+  }
+  
+  if (ste->is_index)
+    ste->index_pointer += is_read_vint(is); // read index pointer
+
+  return te->tb_curr;
 }
 
-inline int
-frt_term_cmp(Term *t1, Term *t2)
+TermEnum *ste_clone(TermEnum *other_te);
+TermEnum *ste_allocate()
 {
-  int comp, size, my_len, o_len;
+  TermEnum *te = te_create();
+  te->next = &ste_next;
+  te->close = &ste_close;
+  te->clone = &ste_clone;
+  SegmentTermEnum *ste =
+    ALLOC(SegmentTermEnum);
+  te->data = ste;
+  return te;
+}
+
+TermEnum *ste_clone(TermEnum *other_te)
+{
+  SegmentTermEnum *other_ste = (SegmentTermEnum *)other_te->data;
+  TermEnum *te = ste_allocate();
+  SegmentTermEnum *ste = (SegmentTermEnum *)te->data;
+  memcpy(ste, other_ste, sizeof(SegmentTermEnum));
+  ste->is = is_clone(other_ste->is);
+  tb_cpy(te->tb_curr, other_te->tb_curr);
+  tb_cpy(te->tb_prev, other_te->tb_prev);
+  ti_cpy(te->ti_curr, other_te->ti_curr);
+  return te;
+}
+
+void ste_close(TermEnum *te)
+{
+  GET_STE;
+  is_close(ste->is);
+  free(ste);
+  te->data = NULL;
+  te_destroy(te);
+}
+
+TermEnum *ste_create(InStream *is, FieldInfos *fis, int is_index)
+{
+  TermEnum *te = ste_allocate();
+  GET_STE;
+  ste->fis = fis;
+  ste->is_index = is_index;
+  ste->is = is;
+  ste->pos = -1;
+  ste->index_pointer = 0;
+  ste->format_m1skip_interval = -1;
+
+  int first_int = is_read_int(is);
+
+  if (first_int >= 0) {
+    // original-format file, without explicit format version number
+    ste->format = 0;
+    ste->size = first_int;
+
+    // back-compatible settings
+    ste->index_interval = 128;
+    ste->skip_interval = INT_MAX; // switch off skip_to optimization
+
+  } else {
+    // check that it is a format we can understand
+    if (first_int < TERM_INFO_FORMAT)
+      eprintf(ERROR, "Unknown format version:%d", first_int);
+
+    // we have a format version number
+    ste->format = first_int;
+
+
+    ste->size = is_read_long(is); // read the size
     
-  my_len = RSTRING(t1->field)->len;
-  o_len = RSTRING(t2->field)->len;
-  size = my_len >= o_len ? o_len : my_len;
-  comp = memcmp(RSTRING(t1->field)->ptr, RSTRING(t2->field)->ptr, size);
-  if (comp == 0) {
-    if (my_len == o_len) {
-      my_len = t1->tlen;
-      o_len = t2->tlen;
-      size = my_len >= o_len ? o_len : my_len;
-      comp = memcmp(t1->text, t2->text, size);
-      if(comp == 0 && my_len != o_len)
-        comp = my_len > o_len ? 1 : -1;
+    if (ste->format == -1) {
+      if (!ste->is_index) {
+        ste->index_interval = is_read_int(is);
+        ste->format_m1skip_interval = is_read_int(is);
+      }
+      // switch off skip_to optimization for file format prior to
+      // 1.4rc2 in order to avoid a bug in skip_to implementation
+      // of these versions
+      ste->skip_interval = INT_MAX;
     } else {
-      comp = my_len > o_len ? 1 : -1;
+      ste->index_interval = is_read_int(is);
+      ste->skip_interval = is_read_int(is);
     }
   }
-  return comp;
+  return te;
+}
+
+void ste_seek(TermEnum *te, int pointer, int pos, Term *t, TermInfo *ti)
+{
+  GET_STE;
+  is_seek(ste->is, pointer);
+  ste->pos = pos;
+  tb_set_term(te->tb_curr, t);
+  tb_reset(te->tb_prev);
+  ti_cpy(te->ti_curr, ti);
+}
+
+TermInfo *ste_scan_for_term_info(TermEnum *te, Term *t)
+{
+  te_skip_to(te, t);
+
+  if (tb_term_cmp(te->tb_curr, t) == 0) {
+    return te_get_ti(te);
+  } else {
+    return NULL;
+  }
 }
 
-int
-frt_term_compare_to_int(VALUE self, VALUE rother)
+Term *ste_scan_for_term(TermEnum *te, int pos)
 {
-  Term *other;
-  GET_TERM;
-  Data_Get_Struct(rother, Term, other);
-  return frt_term_cmp(term, other);
+  GET_STE;
+  while (ste->pos < pos) {
+    if (ste_next(te) == NULL)
+      return NULL;
+  }
+
+  return te_get_term(te);
 }
 
-VALUE
-frt_term_lt(VALUE self, VALUE rother)
+/****************************************************************************
+ *
+ * MultiTermEnum
+ *
+ ****************************************************************************/
+
+#define GET_MTE MultiTermEnum *mte = (MultiTermEnum *)te->data;
+
+TermBuffer *mte_next(TermEnum *te)
 {
-  return frt_term_compare_to_int(self, rother) < 0 ? Qtrue : Qfalse;
+  GET_MTE;
+  SegmentMergeInfo *top = (SegmentMergeInfo *)pq_top(mte->smi_queue);
+
+  if (top == NULL) {
+    tb_reset(te->tb_curr);
+    return false;
+  }
+
+  tb_cpy(te->tb_prev, te->tb_curr);
+  tb_cpy(te->tb_curr, top->tb);
+
+  te->ti_curr->doc_freq = 0;
+
+  while ((top != NULL) && (tb_cmp(te->tb_curr, top->tb) == 0)) {
+    pq_pop(mte->smi_queue);
+    te->ti_curr->doc_freq += top->te->ti_curr->doc_freq;    // increment freq
+    if (smi_next(top)) {
+      pq_push(mte->smi_queue, top);          // restore queue
+    } else {
+      smi_destroy(top);          // done with a segment
+    }
+    top = (SegmentMergeInfo *)pq_top(mte->smi_queue);
+  }
+  return te->tb_curr;
 }
 
-VALUE
-frt_term_gt(VALUE self, VALUE rother)
+void mte_close(TermEnum *te)
 {
-  return frt_term_compare_to_int(self, rother) > 0 ? Qtrue : Qfalse;
+  GET_MTE;
+  pq_clear(mte->smi_queue);
+  pq_destroy(mte->smi_queue);
+  free(mte);
+  te_destroy(te);
 }
 
-VALUE
-frt_term_le(VALUE self, VALUE rother)
+TermEnum *mte_clone(TermEnum *te)
 {
-  return frt_term_compare_to_int(self, rother) <= 0 ? Qtrue : Qfalse;
+  eprintf(ERROR, "MultiTermEnum does not support cloning");
+  return NULL;
 }
 
-VALUE
-frt_term_ge(VALUE self, VALUE rother)
+TermEnum *mte_create(IndexReader **readers, int *starts, int rcnt, Term *t)
 {
-  return frt_term_compare_to_int(self, rother) >= 0 ? Qtrue : Qfalse;
+  int i;
+  TermEnum *te = te_create();
+  te->next = &mte_next;
+  te->clone = &mte_clone;
+  te->close = &mte_close;
+
+  MultiTermEnum *mte = ALLOC(MultiTermEnum);
+  te->data = mte;
+
+  IndexReader *reader;
+  TermEnum *sub_te;
+
+  mte->smi_queue = pq_create(rcnt, &smi_lt);
+  mte->smi_queue->free_elem = &smi_destroy;
+
+  for (i = 0; i < rcnt; i++) {
+    reader = readers[i];
+
+    if (t != NULL) {
+      sub_te = reader->terms_from(reader, t);
+    } else {
+      sub_te = reader->terms(reader);
+    }
+
+    SegmentMergeInfo *smi = smi_create(starts[i], sub_te, reader);
+    if (((t == NULL) && smi_next(smi)) ||
+        (sub_te->tb_curr->field != (char *)EMPTY_STRING)) {
+      pq_push(mte->smi_queue, smi);          // initialize queue
+    } else {
+      smi_destroy(smi);
+    }
+  }
+
+  if ((t != NULL) && (mte->smi_queue->count > 0)) {
+    mte_next(te);
+  }
+
+  return te;
 }
 
-VALUE
-frt_term_eq(VALUE self, VALUE rother)
+/****************************************************************************
+ *
+ * TermInfosWriter
+ *
+ ****************************************************************************/
+
+const Term EmptyTerm = {"", ""};
+
+TermInfosWriter *tiw_open_internal(Store *store,
+    char *segment,
+    FieldInfos *fis,
+    int interval,
+    int is_index)
 {
-  if (rother == Qnil)
-    return Qfalse;
-  return frt_term_compare_to_int(self, rother) == 0 ? Qtrue : Qfalse;
+  TermInfosWriter *tiw = ALLOC(TermInfosWriter);
+  tiw->index_interval = interval;
+  tiw->skip_interval = 16;
+  tiw->last_index_pointer = 0;
+  tiw->last_term = (Term *)&EmptyTerm;
+  tiw->last_term_info = ti_create(0,0,0,0);
+  tiw->size = 0;
+  tiw->is_index = is_index;
+  tiw->fis = fis;
+  tiw->curr_field = NULL;
+  tiw->curr_field_num = -1;
+
+  char fname[SEGMENT_NAME_MAX_LENGTH];
+  strcpy(fname, segment);
+  strcat(fname, (is_index ? ".tii" : ".tis"));
+  OutStream *os = tiw->os = store->create_output(store, fname);
+  os_write_int(os, TERM_INFO_FORMAT);                     // write format
+  os_write_long(os, 0);                         // leave space for size
+  os_write_int(os, tiw->index_interval);        // write index_interval
+  os_write_int(os, tiw->skip_interval);         // write skip_interval
+  if (!is_index) {
+    tiw->other = tiw_open_internal(store, segment, fis, interval, true);
+    tiw->other->other = tiw;
+  }
+  return tiw;
 }
 
+TermInfosWriter *tiw_open(Store *store, char *segment, FieldInfos *fis, int interval)
+{
+  return tiw_open_internal(store, segment, fis, interval, false);
+}
 
-static VALUE
-frt_term_compare_to(VALUE self, VALUE other)
+void tiw_write_term(TermInfosWriter *tiw, OutStream *os, Term *t)
 {
-  return INT2FIX(frt_term_compare_to_int(self, other));
+  //printf("%s, %s\n", tiw->last_term->text, t->text);
+  int start = hlp_string_diff(tiw->last_term->text, t->text);
+  int length = strlen(t->text) - start;
+
+  os_write_vint(os, start);                   // write shared prefix length
+  os_write_vint(os, length);                  // write delta length
+  os_write_chars(os, t->text, start, length); // write delta chars
+  if (tiw->curr_field != t->field) {
+    tiw->curr_field = t->field;
+    tiw->curr_field_num = fis_get_number(tiw->fis, t->field);
+  }
+  os_write_vint(os, tiw->curr_field_num);
+  tiw->last_term = t;
 }
 
-static VALUE
-frt_term_hash(VALUE self)
+void tiw_add(TermInfosWriter *tiw, Term *t, TermInfo *ti)
 {
-  GET_TERM;
-  return INT2FIX(frt_hash(term->text, term->tlen) +
-      frt_hash(RSTRING(term->field)->ptr, RSTRING(term->field)->len));
+  if (tiw->is_index && term_cmp(tiw->last_term, t) > 0) {
+    eprintf(STATE_ERROR,
+        "term out of order %s < %s", t->text, tiw->last_term->text);
+  }
+  if (ti->freq_pointer < tiw->last_term_info->freq_pointer) {
+    eprintf(STATE_ERROR, "freq pointer out of order"); 
+  }
+  if (ti->prox_pointer < tiw->last_term_info->prox_pointer) {
+    eprintf(STATE_ERROR, "prox pointer out of order");
+  }
+
+  if (!tiw->is_index && (tiw->size % tiw->index_interval) == 0)
+    tiw_add(tiw->other, tiw->last_term, tiw->last_term_info); // add an index term
+
+  tiw_write_term(tiw, tiw->os, t);                          // write term
+  os_write_vint(tiw->os, ti->doc_freq);                   // write doc freq
+  os_write_vint(tiw->os, ti->freq_pointer - tiw->last_term_info->freq_pointer);
+  os_write_vint(tiw->os, ti->prox_pointer - tiw->last_term_info->prox_pointer);
+  if (ti->doc_freq >= tiw->skip_interval) 
+    os_write_vint(tiw->os, ti->skip_offset);
+
+  if (tiw->is_index) {
+    OutStream *other_os = tiw->other->os;
+    int other_pos = os_pos(other_os);
+    os_write_vint(tiw->os, other_pos - tiw->last_index_pointer);
+    tiw->last_index_pointer = other_pos; // write pointer
+  }
+
+  ti_cpy(tiw->last_term_info, ti);
+  tiw->size++;
+}
+
+void tiw_close(TermInfosWriter *tiw)
+{
+  OutStream *os = tiw->os;
+  os_seek(os, 4); // write @size after format
+  os_write_long(os, tiw->size);
+  os_close(os);
+
+  if (!tiw->is_index)
+    tiw_close(tiw->other);
+
+  ti_destroy(tiw->last_term_info);
+  free(tiw);
 }
 
 /****************************************************************************
  *
- * Init Function
+ * TermInfosReader
  *
  ****************************************************************************/
 
-void
-Init_term(void)
-{
-  /* Term */
-  cTerm = rb_define_class_under(mIndex, "Term", rb_cObject);
-  rb_define_alloc_func(cTerm, frt_term_alloc);
-  rb_include_module(cTerm, rb_mComparable);
-
-  rb_define_method(cTerm, "initialize", frt_term_init, 2);
-  rb_define_method(cTerm, "set!", frt_term_set, 2);
-  rb_define_method(cTerm, "to_s", frt_term_to_s, 0);
-  rb_define_method(cTerm, "<=>", frt_term_compare_to, 1);
-  rb_define_method(cTerm, "<", frt_term_lt, 1);
-  rb_define_method(cTerm, ">", frt_term_gt, 1);
-  rb_define_method(cTerm, "<=", frt_term_le, 1);
-  rb_define_method(cTerm, ">=", frt_term_ge, 1);
-  rb_define_method(cTerm, "eql?", frt_term_eq, 1);
-  rb_define_method(cTerm, "==", frt_term_eq, 1);
-  rb_define_method(cTerm, "text", frt_term_get_text, 0);
-  rb_define_method(cTerm, "text=", frt_term_set_text, 1);
-  rb_define_method(cTerm, "field", frt_term_get_field, 0);
-  rb_define_method(cTerm, "field=", frt_term_set_field, 1);
-  rb_define_method(cTerm, "hash", frt_term_hash, 0);
+void tir_close(TermInfosReader *tir)
+{
+  int i;
+  if (tir->index_terms != NULL) {
+    for (i = 0; i < tir->index_size; i++) {
+      term_destroy(tir->index_terms[i]);
+      ti_destroy(tir->index_term_infos[i]);
+    }
+    free(tir->index_terms);
+    free(tir->index_term_infos);
+    free(tir->index_pointers);
+  }
+  if (tir->orig_te) tir->orig_te->close(tir->orig_te);
+  thread_key_delete(tir->thread_te);
+  ary_destroy(tir->te_bucket);
+  if (tir->index_te) tir->index_te->close(tir->index_te);
+  mutex_destroy(&tir->mutex);
+  free(tir);
+}
+
+TermInfosReader *tir_open(Store *store, char *segment, FieldInfos *fis)
+{
+  TermInfosReader *tir = ALLOC(TermInfosReader);
+  char fname[SEGMENT_NAME_MAX_LENGTH];
+  mutex_init(&tir->mutex, NULL);
+  strcpy(fname, segment);
+  strcpy(fname + strlen(segment), ".tis");
+  InStream *is = store->open_input(store, fname);
+  tir->orig_te = ste_create(is, fis, false);
+  thread_key_create(&tir->thread_te, NULL);
+  tir->te_bucket = ary_create(1, (destroy_func_t)tir->orig_te->close);
+
+  SegmentTermEnum *ste = tir->orig_te->data;
+  tir->size = ste->size;
+  tir->skip_interval = ste->skip_interval;
+
+  strcpy(fname + strlen(segment), ".tii");
+  is = store->open_input(store, fname);
+  tir->index_te = ste_create(is, fis, true);
+  tir->index_terms = NULL;
+  tir->index_term_infos = NULL;
+  tir->index_pointers = NULL;
+  return tir;
+}
+
+void tir_ensure_index_is_read(TermInfosReader *tir)
+{
+  mutex_lock(&tir->mutex);
+  if (tir->index_terms == NULL) {
+    int index_size = ((SegmentTermEnum *)tir->index_te->data)->size;
+    tir->index_size = index_size;
+
+    tir->index_terms = ALLOC_N(Term *, index_size);
+    tir->index_term_infos = ALLOC_N(TermInfo *, index_size);
+    tir->index_pointers = ALLOC_N(int, index_size);
+
+    int i = 0;
+    TermEnum *index_te = tir->index_te;
+    SegmentTermEnum *ste = index_te->data;
+
+    while (ste_next(index_te) != NULL) {
+      tir->index_terms[i] = te_get_term(index_te);
+      tir->index_term_infos[i] = te_get_ti(index_te);
+      tir->index_pointers[i] = ste->index_pointer;
+      i++;
+    }
+
+    index_te->close(index_te);
+    tir->index_te = NULL;
+  }
+  mutex_unlock(&tir->mutex);
+}
+
+static inline TermEnum *tir_enum(TermInfosReader *tir)
+{
+  TermEnum *te;
+  if ((te = thread_getspecific(tir->thread_te)) == NULL) {
+    te = tir->orig_te->clone(tir->orig_te);
+    ary_append(tir->te_bucket, te);
+    thread_setspecific(tir->thread_te, te);
+  }
+  return te;
 }
+
+void tir_seek_enum(TermInfosReader *tir, int ind_offset)
+{
+  TermEnum *te = tir_enum(tir);
+  SegmentTermEnum *ste = (SegmentTermEnum *)te->data;
+  ste_seek(te, tir->index_pointers[ind_offset],
+      (ind_offset * ste->index_interval) - 1,
+      tir->index_terms[ind_offset],
+      tir->index_term_infos[ind_offset]);
+}
+
+int tir_get_index_offset(TermInfosReader *tir, Term *t)
+{
+  int lo = 0;            // binary search tir->index_terms[]
+  int hi = tir->index_size - 1;
+  int mid, delta;
+  Term **index_terms = tir->index_terms;
+
+  while (hi >= lo) {
+    mid = (lo + hi) >> 1;
+    delta = term_cmp(t, index_terms[mid]);
+    if (delta < 0) {
+      hi = mid - 1;
+    } else if (delta > 0) {
+      lo = mid + 1;
+    } else {
+      return mid;
+    }
+  }
+  return hi;
+}
+
+TermInfo *tir_get_ti(TermInfosReader *tir, Term *t)
+{
+  if (tir->size == 0)
+    return NULL;
+
+  tir_ensure_index_is_read(tir);
+
+  // optimize sequential access: first try scanning cached enum w/o seeking
+  TermEnum *te = tir_enum(tir);
+  SegmentTermEnum *ste = (SegmentTermEnum *)te->data;
+  if (ste->pos < ste->size && tb_term_cmp(te->tb_curr, t) <= 0) {
+    SegmentTermEnum *ste = (SegmentTermEnum *)te->data;
+    int enum_offset = (int)(ste->pos / ste->index_interval) + 1;
+    if (tir->index_size == enum_offset ||
+        term_cmp(t, tir->index_terms[enum_offset]) < 0) { // but before end of block
+      return ste_scan_for_term_info(te, t);       // no need to seek
+    }
+  }
+
+  // random-access: must seek
+  tir_seek_enum(tir, tir_get_index_offset(tir, t));
+  return ste_scan_for_term_info(te, t);
+}
+
+Term *tir_get_term(TermInfosReader *tir, int pos)
+{ 
+  if (tir->size == 0)
+    return NULL;
+
+  TermEnum *te = tir_enum(tir);
+  SegmentTermEnum *ste = (SegmentTermEnum *)te->data;
+  if (pos >= ste->pos &&
+      pos < (ste->pos + ste->index_interval)) {
+    return ste_scan_for_term(te, pos);      // can avoid seek
+  }
+
+  tir_seek_enum(tir, (int)(pos / ste->index_interval)); // must seek
+  return ste_scan_for_term(te, pos);
+}
+
+int tir_get_term_pos(TermInfosReader *tir, Term *t)
+{
+  if (tir->size == 0)
+    return -1;
+
+  tir_ensure_index_is_read(tir);
+
+  int ind_offset = tir_get_index_offset(tir, t);
+  tir_seek_enum(tir, ind_offset);
+
+  TermEnum *te = tir_enum(tir);
+  while ((tb_term_cmp(te->tb_curr, t) < 0) && (ste_next(te) != NULL))
+    ;
+
+  if (tb_term_cmp(te->tb_curr, t) == 0)
+    return ((SegmentTermEnum *)te->data)->pos;
+  else
+    return -1;
+}
+