ferret 0.9.1 → 0.9.2

data/ext/analysis.h

@@ -40,13 +40,14 @@ struct TokenStream {
   Token *(*next)(TokenStream *ts);
   void (*reset)(TokenStream *ts, char *text);
   void (*clone_i)(TokenStream *orig_ts, TokenStream *new_ts);
-  void (*destroy)(void *p);
+  void (*destroy)(TokenStream *ts);
   TokenStream *sub_ts; /* used by filters */
-  bool destroy_sub : 1;
+  int ref_cnt;
 };
 
 #define ts_next(mts) mts->next(mts)
-#define ts_destroy(mts) mts->destroy(mts)
+
+void ts_deref(void *p);
 
 TokenStream *whitespace_tokenizer_create();
 TokenStream *mb_whitespace_tokenizer_create(bool lowercase);
@@ -93,13 +94,19 @@ typedef struct Analyzer {
   void *data;
   TokenStream *current_ts;
   TokenStream *(*get_ts)(struct Analyzer *a, char *field, char *text);
-  void (*destroy)(void *p);
+  void (*destroy)(struct Analyzer *a);
+  int ref_cnt;
 } Analyzer;
 
-#define a_destroy(ma) ma->destroy(ma)
+void a_deref(void *p);
+
 #define a_get_ts(ma, field, text) ma->get_ts(ma, field, text)
 #define a_get_new_ts(ma, field, text) ts_clone(ma->get_ts(ma, field, text))
 
+Analyzer *analyzer_create(void *data, TokenStream *ts,
+    void (*destroy)(Analyzer *),
+    TokenStream *(*get_ts)(Analyzer *a, char *field, char *text));
+void a_standard_destroy(Analyzer *a);
 Analyzer *whitespace_analyzer_create(bool lowercase);
 Analyzer *mb_whitespace_analyzer_create(bool lowercase);
 
@@ -118,7 +125,12 @@ Analyzer *mb_standard_analyzer_create_with_words(
 Analyzer *mb_standard_analyzer_create_with_words_len(
     const char **words, int len, bool lowercase);
 
-Analyzer *per_field_analyzer_create(Analyzer *def, bool destroy_subs);
+typedef struct PerFieldAnalyzer {
+  HshTable *dict;
+  Analyzer *def;
+} PerFieldAnalyzer;
+
+Analyzer *per_field_analyzer_create(Analyzer *def);
 void pfa_add_field(Analyzer *self, char *field, Analyzer *analyzer);
 
 #endif

data/ext/array.c

@@ -18,9 +18,8 @@ Array *ary_create(int allocate, void (*free_elem)(void *p))
   return ary;
 }
 
-void ary_destroy(void *p)
+void ary_destroy(Array *ary)
 {
-  Array *ary = (Array *)p;
   int i;
   if (ary->free_elem) {
     for (i = 0; i < ary->size; i++) {

data/ext/array.h

@@ -9,7 +9,7 @@ typedef struct Array {
 } Array;
 
 Array *ary_create(int size, void (*free_elem)(void *p));
-void ary_destroy(void *p);
+void ary_destroy(Array *ary);
 void ary_set(Array *ary, int index, void *value);
 void ary_append(Array *ary, void *value);
 void *ary_get(Array *ary, int index);

data/ext/bitvector.c

@@ -20,15 +20,15 @@ BitVector *bv_create()
   return bv_create_size(BV_INIT_CAPA);
 }
 
-void bv_destroy(void *p)
+void bv_destroy(BitVector *bv)
 {
-  BitVector *bv = (BitVector *)p;
   free(bv->bits);
   free(bv);
 }
 
 void bv_set(BitVector *bv, int bit)
 {
+  uchar *byte_p;
   int byte = bit>>3;
   uchar bitmask = 1<<(bit&7);
   if (bv->size <= byte) {
@@ -41,7 +41,7 @@ void bv_set(BitVector *bv, int bit)
       bv->capa = capa;
     }
   }
-  uchar *byte_p = &(bv->bits[byte]);
+  byte_p = &(bv->bits[byte]);
   if ((bitmask & *byte_p) == 0) {
     bv->count++;
     *byte_p |= bitmask;
@@ -63,10 +63,14 @@ void bv_clear(BitVector *bv)
 
 void bv_unset(BitVector *bv, int bit)
 {
+  uchar *byte_p;
+  uchar bitmask;
   int byte = bit>>3;
+  
   if (byte >= bv->size) return;
-  uchar *byte_p = &(bv->bits[byte]);
-  uchar bitmask = 1<<(bit&7);
+  
+  byte_p = &(bv->bits[byte]);
+  bitmask = 1<<(bit&7);
   if ((bitmask & *byte_p) > 0) {
     bv->count--;
     *byte_p &= ~bitmask;
@@ -115,7 +119,7 @@ BitVector *bv_read(Store *store, char *name)
 {
   BitVector *bv = ALLOC(BitVector);
   InStream *is = store->open_input(store, name);
-  bv->capa = bv->size = is_read_vint(is);
+  bv->capa = bv->size = (int)is_read_vint(is);
   bv->bits = ALLOC_N(uchar, bv->capa);
   is_read_bytes(is, bv->bits, 0, bv->size);
   is_close(is);

data/ext/bitvector.h

@@ -15,7 +15,7 @@ typedef struct BitVector {
 
 BitVector *bv_create();;
 BitVector *bv_create_size(int size);
-void bv_destroy(void *bv);
+void bv_destroy(BitVector *bv);
 void bv_set(BitVector *bv, int bit);
 int bv_get(BitVector *bv, int bit);
 void bv_clear(BitVector *bv);
@@ -24,6 +24,6 @@ void bv_write(BitVector *bv, Store *store, char *name);
 BitVector *bv_read(Store *store, char *name);
 void bv_scan_reset(BitVector *bv);
 int bv_scan_next(BitVector *bv);
-int bv_scan_next_from(BitVector *bv, int from);
+int bv_scan_next_from(BitVector *bv, register const int from);
 
 #endif

data/ext/compound_io.c

@@ -1,4 +1,5 @@
 #include "index.h" 
+
 static char * const ALREADY_CLOSED_MSG = "Already closed";
 static char * const STREAM_CLOSED_MSG = "Stream closed";
 static char * const MISSING_FILE_MSG = "No sub-file found";
@@ -8,6 +9,8 @@ static char * const FILE_OFFSET_MSG =  "Difference in the output file offsets"
         " does not match the original file length";
 static char * const NO_FILES_TO_MERGE_MSG = "No Files to merge into the compound file";
 
+extern void store_destroy(Store *store);
+
 /****************************************************************************
  *
  * CompoundStore
@@ -26,10 +29,11 @@ void cmpd_touch(Store *store, char *filename)
 
 int cmpd_exists(Store *store, char *filename)
 {
-  if (h_get(store->dir.cmpd->entries, filename) != NULL)
+  if (h_get(store->dir.cmpd->entries, filename) != NULL) {
     return true;
-  else
+  } else {
     return false;
+  }
 }
 
 /**
@@ -63,9 +67,8 @@ void cmpd_clear(Store *store)
   RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
 }
 
-void cmpd_close(Store *store)
+void cmpd_close_i(Store *store)
 {
-  mutex_lock(&store->mutex);
   CompoundStore *cmpd = store->dir.cmpd;
   if (cmpd->stream == NULL)
     RAISE(IO_ERROR, ALREADY_CLOSED_MSG);
@@ -143,6 +146,7 @@ InStream *cmpd_create_input(InStream *sub_is, int offset, int length)
 
 InStream *cmpd_open_input(Store *store, const char *filename)
 {
+  FileEntry *entry;
   CompoundStore *cmpd = store->dir.cmpd;
   InStream *is;
 
@@ -152,7 +156,7 @@ InStream *cmpd_open_input(Store *store, const char *filename)
     RAISE(IO_ERROR, STREAM_CLOSED_MSG);
   }
 
-  FileEntry *entry = (FileEntry *)h_get(cmpd->entries, filename);
+  entry = (FileEntry *)h_get(cmpd->entries, filename);
   if (entry == NULL) {
     mutex_unlock(&store->mutex);
     RAISE(IO_ERROR, MISSING_FILE_MSG);
@@ -183,6 +187,9 @@ void cmpd_close_lock(Lock *lock)
 
 Store *open_cmpd_store(Store *store, const char *name)
 {
+  int count, i, offset;
+  char *fname;
+  FileEntry *entry;
   Store * volatile new_store = NULL;
   CompoundStore * volatile cmpd = NULL;
   InStream * volatile is = NULL;
@@ -193,16 +200,14 @@ Store *open_cmpd_store(Store *store, const char *name)
 
     cmpd->store   = store;
     cmpd->name    = name;
-    cmpd->entries = h_new_str(&efree, &efree);
+    cmpd->entries = h_new_str(&free, &free);
     is = cmpd->stream = store->open_input(store, cmpd->name);
 
     // read the directory and init files
-    int count = is_read_vint(is);
-    FileEntry *entry = NULL;
-    int i, offset;
-    char *fname;
+    count = (int)is_read_vint(is);
+    entry = NULL;
     for (i = 0; i < count; i++) {
-      offset = is_read_long(is);
+      offset = (int)is_read_long(is);
       fname = is_read_string(is);
 
       if (entry != NULL) {
@@ -231,9 +236,9 @@ Store *open_cmpd_store(Store *store, const char *name)
   new_store->remove   = &cmpd_remove;
   new_store->rename   = &cmpd_rename;
   new_store->count    = &cmpd_count;
-  new_store->close    = &cmpd_close;
   new_store->clear    = &cmpd_clear;
   new_store->length   = &cmpd_length;
+  new_store->close_i  = &cmpd_close_i;
   new_store->create_output = &cmpd_create_output;
   new_store->open_input    = &cmpd_open_input;
   new_store->open_lock     = &cmpd_open_lock;
@@ -260,19 +265,13 @@ WFileEntry *wfe_create(char *name)
   return wfe;
 }
 
-void wfe_destroy(void *p)
-{
-  WFileEntry *wfe = (WFileEntry *)p;
-  efree(wfe);
-}
-
 CompoundWriter *open_cw(Store *store, char *name)
 {
   CompoundWriter *cw = ALLOC(CompoundWriter);
   cw->store = store;
   cw->name = name;
   cw->ids = hs_str_create(NULL);
-  cw->file_entries = ary_create(1, &wfe_destroy);
+  cw->file_entries = ary_create(1, &free);
   cw->merged = false;
   return cw;
 }
@@ -290,14 +289,16 @@ void cw_add_file(CompoundWriter *cw, char *id)
 void cw_copy_file(CompoundWriter *cw, WFileEntry *src, OutStream *os)
 {
   int start_ptr = os_pos(os);
+  int end_ptr;
   int remainder, length, len;
-
+  uchar buffer[BUFFER_SIZE];
+  
   InStream *is = cw->store->open_input(cw->store, src->name);
   
   TRY
     remainder = length = is_length(is);
 
-    uchar buffer[BUFFER_SIZE];
+
     while (remainder > 0) {
       len = MIN(remainder, BUFFER_SIZE);
       is_read_bytes(is, buffer, 0, len);
@@ -310,9 +311,9 @@ void cw_copy_file(CompoundWriter *cw, WFileEntry *src, OutStream *os)
       RAISE(IO_ERROR, REMAINDER_ERROR_MSG);
 
     // Verify that the output length diff is equal to original file
-    int end_ptr = os_pos(os);
-    int diff = end_ptr - start_ptr;
-    if (diff != length)
+    end_ptr = os_pos(os);
+    len = end_ptr - start_ptr;
+    if (len != length)
       RAISE(IO_ERROR, FILE_OFFSET_MSG);
 
   XFINALLY
@@ -322,13 +323,16 @@ void cw_copy_file(CompoundWriter *cw, WFileEntry *src, OutStream *os)
 
 void cw_close(CompoundWriter *cw)
 {
+  OutStream * volatile os = NULL;
+  int i;
+  WFileEntry *wfe;
+
   if (cw->merged) RAISE(STATE_ERROR, ALREADY_MERGED_MSG);
   if (cw->ids->size <= 0)
     RAISE(STATE_ERROR, NO_FILES_TO_MERGE_MSG);
 
   cw->merged = true;
 
-  OutStream * volatile os = NULL;
   TRY
     os = cw->store->create_output(cw->store, cw->name);
     os_write_vint(os, cw->file_entries->size);
@@ -336,8 +340,7 @@ void cw_close(CompoundWriter *cw)
     /* Write the directory with all offsets at 0.
      * Remember the positions of directory entries so that we can adjust the
      * offsets later */
-    int i;
-    WFileEntry *wfe; 
+
     for (i = 0; i < cw->file_entries->size; i++) {
       wfe = (WFileEntry *)cw->file_entries->elems[i];
       wfe->dir_offset = os_pos(os);

data/ext/document.c

@@ -1,4 +1,4 @@
-#include <document.h>
+#include "document.h"
 #include <string.h>
 
 /****************************************************************************
@@ -26,7 +26,7 @@ inline void df_set(DocField *df, const char *name,
     RAISE(ARG_ERROR, INDEX_NO_TV_YES_MSG);
   df->name = estrdup(name);
   df->data = data;
-  df->blen = strlen(data);
+  df->blen = (int)strlen(data);
   df_set_store(df, store);
   df_set_index(df, index);
   df_set_term_vector(df, tv);
@@ -53,19 +53,17 @@ DocField *df_clone(DocField *self)
   return clone;
 }
 
-void df_destroy(void *p)
+void df_destroy(DocField *df)
 {
-  DocField *df = (DocField *)p;
   free(df->name);
-  free(p);
+  free(df);
 }
 
-void df_destroy_data(void *p)
+void df_destroy_data(DocField *df)
 {
-  DocField *df = (DocField *)p;
   free(df->data);
   free(df->name);
-  free(p);
+  free(df);
 }
 
 /*
@@ -161,10 +159,13 @@ void df_set_term_vector(DocField *df, int tv)
  */
 DocField *df_create_binary(char *name, char *data, int blen, int store)
 {
+  DocField *df;
+  
   if (store == DF_STORE_NO) {
     RAISE(ARG_ERROR, BIN_FIELD_STORE_NO_MSG);
   }
-  DocField *df = df_create(name, data, store, DF_INDEX_NO, DF_TERM_VECTOR_NO);
+  
+  df = df_create(name, data, store, DF_INDEX_NO, DF_TERM_VECTOR_NO);
   df->is_binary = true;
   df->blen = blen;
   return df;
@@ -203,27 +204,26 @@ char *df_to_s(DocField *self)
 Document *doc_create()
 {
   Document *doc = ALLOC(Document);
-  doc->fields = h_new_str(&free, &ary_destroy);
+  doc->fields = h_new_str(&free, (free_ft)&ary_destroy);
   doc->fcnt = 0;
   doc->dfcnt = 0;
   doc->field_arr = NULL;
   doc->df_arr = NULL;
   doc->boost = 1.0;
-  doc->free_data = &df_destroy_data;
+  doc->free_data = (free_ft)&df_destroy_data;
   return doc;
 }
 
 Document *doc_create_keep_data()
 {
   Document *doc = doc_create();
-  doc->free_data = df_destroy;
+  doc->free_data = (free_ft)&df_destroy;
   return doc;
 }
 
-void doc_destroy(void *p)
+void doc_destroy(Document *doc)
 {
   int i;
-  Document *doc = (Document *)p;
   if (doc->free_data) {
     for (i = 0; i < doc->dfcnt; i++) {
       doc->free_data(doc->df_arr[i]);
@@ -349,7 +349,7 @@ char *doc_to_s(Document *doc)
   char **df_strs = ALLOC_N(char *, doc->dfcnt);
   for (i = 0; i < doc->dfcnt; i++) {
     df_strs[i] = df_to_s(doc->df_arr[i]);
-    len += strlen(df_strs[i]) + 3;
+    len += (int)strlen(df_strs[i]) + 3;
   }
   str_ptr = str = ALLOC_N(char, len);
   sprintf(str_ptr, "Document {\n");

data/ext/document.h

@@ -34,7 +34,7 @@ enum {
 typedef struct DocField {
   char  *name;
   char  *data;
-  int blen; // This is used for binary fields only to store the data length
+  int blen; /* used for binary fields to store data length */
   float boost;
   bool is_stored : 1;
   bool is_compressed : 1;
@@ -50,8 +50,8 @@ typedef struct DocField {
 DocField *df_create(const char *name, char *data, int store, int index, int tv);
 DocField *df_clone(DocField *self);
 void df_set(DocField *df, const char *name, char *data, int store, int index, int tv);
-void df_destroy(void *p);
-void df_destroy_data(void *p);
+void df_destroy(DocField *df);
+void df_destroy_data(DocField *df);
 void df_set_store(DocField *df, int store);
 void df_set_index(DocField *df, int index);
 void df_set_term_vector(DocField *df, int tv);
@@ -71,12 +71,12 @@ typedef struct Document {
   DocField **df_arr;
   int dfcnt;
   float boost;
-  void (*free_data)(void *p);
+  free_ft free_data;
 } Document;
 
 Document *doc_create();
 Document *doc_create_keep_data();
-void doc_destroy(void *p);
+void doc_destroy(Document *doc);
 void doc_add_field(Document *doc, DocField *df);
 DocField *doc_get_field(Document *doc, const char *fname);
 Array *doc_get_fields(Document *doc, const char *fname);

data/ext/except.c

@@ -1,6 +1,8 @@
 #include "global.h"
 #include "except.h"
 
+bool except_show_pos = true;
+
 char * const UNSUPPORTED_ERROR_MSG = "Unsupported operation";
 char * const EOF_ERROR_MSG = "Read past end of file";