ferret 0.11.6 → 0.11.8.4

data/ext/symbol.c

@@ -0,0 +1,10 @@
+#include <stdlib.h>
+#include "symbol.h"
+
+
+FrtSymbol frt_intern_and_free(char *str)
+{
+    FrtSymbol sym = (FrtSymbol)rb_intern(str);
+    free(str);
+    return sym;
+}

data/ext/symbol.h

@@ -0,0 +1,23 @@
+#ifndef FRT_SYMBOL_H
+#define FRT_SYMBOL_H
+
+#include <ruby.h>
+
+typedef void *FrtSymbol;
+
+VALUE rb_id2str(ID id);
+#define frt_symbol_init()
+#define frt_intern(str) (FrtSymbol)rb_intern(str)
+extern FrtSymbol frt_intern_and_free(char *str);
+
+#define FRT_I frt_intern
+#define FRT_InF frt_intern_and_free
+#define FRT_S(_sym) rb_id2name((ID)_sym)
+
+#define FSYM2SYM(_sym) ID2SYM((ID)_sym)
+#define SYM2FSYM(_sym) (FrtSymbol)SYM2ID(_sym)
+
+#define frt_sym_hash(sym) frt_str_hash(rb_id2name((ID)sym))
+#define frt_sym_len(sym) strlen(rb_id2name((ID)sym))
+
+#endif

data/ext/term_vectors.c

@@ -2,6 +2,8 @@
 #include "index.h"
 #include "array.h"
 #include "helper.h"
+#include "symbol.h"
+#include "internal.h"
 
 /****************************************************************************
  *
@@ -18,12 +20,11 @@ void tv_destroy(TermVector *tv)
         free(tv->terms[i].positions);
     }
     free(tv->offsets);
-    free(tv->field);
     free(tv->terms);
     free(tv);
 }
 
-int tv_get_tv_term_index(TermVector *tv, const char *term)
+int tv_scan_to_term_index(TermVector *tv, const char *term)
 {
     int lo = 0;                 /* search starts array */
     int hi = tv->term_cnt - 1;  /* for 1st element < n, return its index */
@@ -45,18 +46,24 @@ int tv_get_tv_term_index(TermVector *tv, const char *term)
             return mid;
         }
     }
-    if (hi >= 0 && strcmp(term, tv->terms[hi].text) == 0) {
-        return hi;
+    return lo;
+}
+
+int tv_get_term_index(TermVector *tv, const char *term)
+{
+    int index = tv_scan_to_term_index(tv, term);
+    if (index < tv->term_cnt && (0 == strcmp(term, tv->terms[index].text))) {
+        /* found term */
+        return index;
     }
     else {
         return -1;
     }
-    return hi;
 }
 
-extern TVTerm *tv_get_tv_term(TermVector *tv, const char *term)
+TVTerm *tv_get_tv_term(TermVector *tv, const char *term)
 {
-    int index = tv_get_tv_term_index(tv, term);
+    int index = tv_get_term_index(tv, term);
     if (index >= 0) {
         return &(tv->terms[index]);
     }
@@ -64,289 +71,3 @@ extern TVTerm *tv_get_tv_term(TermVector *tv, const char *term)
         return NULL;
     }
 }
-
-/****************************************************************************
- *
- * TermVectorsReader
- *
- ****************************************************************************/
-
-TermVectorsReader *tvr_open(Store *store,
-                            const char *segment,
-                            FieldInfos *fis)
-{
-    TermVectorsReader *tvr = ALLOC(TermVectorsReader);
-    char file_name[SEGMENT_NAME_MAX_LENGTH];
-
-    tvr->fis = fis;
-    sprintf(file_name, "%s.tvx", segment);
-    tvr->tvx_in = store->open_input(store, file_name);
-    tvr->size = is_length(tvr->tvx_in) / 12;
-
-    sprintf(file_name, "%s.tvd", segment);
-    tvr->tvd_in = store->open_input(store, file_name);
-    return tvr;
-}
-
-TermVectorsReader *tvr_clone(TermVectorsReader *orig)
-{
-    TermVectorsReader *tvr = ALLOC(TermVectorsReader);
-
-    memcpy(tvr, orig, sizeof(TermVectorsReader));
-    tvr->tvx_in = is_clone(orig->tvx_in);
-    tvr->tvd_in = is_clone(orig->tvd_in);
-    
-    return tvr;
-}
-
-void tvr_close(TermVectorsReader *tvr)
-{
-    is_close(tvr->tvx_in);
-    is_close(tvr->tvd_in);
-    free(tvr);
-}
-
-TermVector *tvr_read_term_vector(TermVectorsReader *tvr, int field_num)
-{
-    TermVector *tv = ALLOC_AND_ZERO(TermVector);
-    InStream *tvd_in = tvr->tvd_in;
-    FieldInfo *fi = tvr->fis->fields[field_num];
-    const int num_terms = is_read_vint(tvd_in);
-    
-    tv->field_num = field_num;
-    tv->field = estrdup(fi->name);
-
-    if (num_terms > 0) {
-        int i, j, delta_start, delta_len, total_len, freq;
-        int store_positions = fi_store_positions(fi);
-        int store_offsets = fi_store_offsets(fi);
-        uchar buffer[MAX_WORD_SIZE];
-        TVTerm *term;
-
-        tv->term_cnt = num_terms;
-        tv->terms = ALLOC_AND_ZERO_N(TVTerm, num_terms);
-
-        for (i = 0; i < num_terms; i++) {
-            term = &(tv->terms[i]);
-            /* read delta encoded term */
-            delta_start = is_read_vint(tvd_in);
-            delta_len = is_read_vint(tvd_in);
-            total_len = delta_start + delta_len;
-            is_read_bytes(tvd_in, buffer + delta_start, delta_len);
-            buffer[total_len++] = '\0';
-            term->text = memcpy(ALLOC_N(char, total_len), buffer, total_len);
-
-            /* read freq */
-            freq = term->freq = is_read_vint(tvd_in);
-
-            /* read positions if necessary */
-            if (store_positions) {
-                int *positions = term->positions = ALLOC_N(int, freq);
-                int pos = 0;
-                for (j = 0; j < freq; j++) {
-                    positions[j] = pos += is_read_vint(tvd_in);
-                }
-            }
-
-            /* read offsets if necessary */
-        }
-        if (store_offsets) {
-            int num_positions = tv->offset_cnt = is_read_vint(tvd_in);
-            Offset *offsets = tv->offsets = ALLOC_N(Offset, num_positions);
-            int offset = 0;
-            for (i = 0; i < num_positions; i++) {
-                offsets[i].start = offset += is_read_vint(tvd_in);
-                offsets[i].end = offset += is_read_vint(tvd_in);
-            }
-        }
-    }
-    return tv;
-}
-
-HashTable *tvr_get_tv(TermVectorsReader *tvr, int doc_num)
-{
-    HashTable *term_vectors = h_new_str((free_ft)NULL, (free_ft)&tv_destroy);
-    int i;
-    InStream *tvx_in = tvr->tvx_in;
-    InStream *tvd_in = tvr->tvd_in;
-    off_t data_ptr, field_index_ptr;
-    int field_cnt;
-    int *field_nums;
-
-    if (doc_num >= 0 && doc_num < tvr->size) {
-        is_seek(tvx_in, 12 * doc_num);
-
-        data_ptr = (off_t)is_read_u64(tvx_in);
-        field_index_ptr = data_ptr + (off_t)is_read_u32(tvx_in);
-
-        /* scan fields to get position of field_num's term vector */
-        is_seek(tvd_in, field_index_ptr);
-
-        field_cnt = is_read_vint(tvd_in);
-        field_nums = ALLOC_N(int, field_cnt);
-
-        for (i = 0; i < field_cnt; i++) {
-            field_nums[i] = is_read_vint(tvd_in);
-            is_read_vint(tvd_in); /* skip space, we don't need it */
-        }
-        is_seek(tvd_in, data_ptr);
-
-        for (i = 0; i < field_cnt; i++) {
-            TermVector *tv = tvr_read_term_vector(tvr, field_nums[i]);
-            h_set(term_vectors, tv->field, tv);
-        }
-        free(field_nums);
-    }
-    return term_vectors;
-}
-
-TermVector *tvr_get_field_tv(TermVectorsReader *tvr,
-                             int doc_num,
-                             int field_num)
-{
-    int i;
-    InStream *tvx_in = tvr->tvx_in;
-    InStream *tvd_in = tvr->tvd_in;
-    off_t data_ptr, field_index_ptr;
-    int field_cnt;
-    int offset = 0;
-    TermVector *tv = NULL;
-
-    if (doc_num >= 0 && doc_num < tvr->size) {
-        is_seek(tvx_in, 12 * doc_num);
-
-        data_ptr = (off_t)is_read_u64(tvx_in);
-        field_index_ptr = data_ptr + (off_t)is_read_u32(tvx_in);
-
-        /* scan fields to get position of field_num's term vector */
-        is_seek(tvd_in, field_index_ptr);
-
-        field_cnt = is_read_vint(tvd_in);
-        for (i = 0; i < field_cnt; i++) {
-            if ((int)is_read_vint(tvd_in) == field_num) {
-                break;
-            }
-            offset += is_read_vint(tvd_in); /* space taken by field */
-        }
-        if (i < field_cnt) {
-            /* field was found */
-            is_seek(tvd_in, data_ptr + offset);
-            tv = tvr_read_term_vector(tvr, field_num);
-        }
-    }
-    return tv;
-}
-
-/****************************************************************************
- *
- * TermVectorsWriter
- *
- ****************************************************************************/
-
-TermVectorsWriter *tvw_open(Store *store, const char *segment, FieldInfos *fis)
-{
-    TermVectorsWriter *tvw = ALLOC(TermVectorsWriter);
-    char file_name[SEGMENT_NAME_MAX_LENGTH];
-    tvw->fis = fis;
-    tvw->fields = ary_new_type_capa(TVField, TV_FIELD_INIT_CAPA);
-
-    snprintf(file_name, SEGMENT_NAME_MAX_LENGTH, "%s.tvx", segment);
-    tvw->tvx_out = store->new_output(store, file_name);
-
-    snprintf(file_name, SEGMENT_NAME_MAX_LENGTH, "%s.tvd", segment);
-    tvw->tvd_out = store->new_output(store, file_name);
-
-    return tvw;
-}
-
-void tvw_close(TermVectorsWriter *tvw)
-{
-    os_close(tvw->tvx_out);
-    os_close(tvw->tvd_out);
-    ary_free(tvw->fields);
-    free(tvw);
-}
-
-void tvw_open_doc(TermVectorsWriter *tvw)
-{
-    ary_size(tvw->fields) = 0;
-    tvw->tvd_ptr = os_pos(tvw->tvd_out);
-    os_write_u64(tvw->tvx_out, tvw->tvd_ptr);
-}
-
-void tvw_close_doc(TermVectorsWriter *tvw)
-{
-    int i;
-    OutStream *tvd_out = tvw->tvd_out;
-    os_write_u32(tvw->tvx_out, (f_u32)(os_pos(tvw->tvd_out) - tvw->tvd_ptr));
-    os_write_vint(tvd_out, ary_size(tvw->fields));
-    for (i = 0; i < ary_size(tvw->fields); i++) {
-        os_write_vint(tvd_out, tvw->fields[i].field_num);
-        os_write_vint(tvd_out, tvw->fields[i].size);
-    }
-}
-
-void tvw_add_postings(TermVectorsWriter *tvw,
-                      int field_num,
-                      PostingList **plists,
-                      int posting_count,
-                      Offset *offsets,
-                      int offset_count)
-{
-    int i, delta_start, delta_length;
-    const char *last_term = EMPTY_STRING;
-    off_t tvd_start_pos = os_pos(tvw->tvd_out);
-    OutStream *tvd_out = tvw->tvd_out;
-    PostingList *plist;
-    Posting *posting;
-    Occurence *occ;
-    FieldInfo *fi = tvw->fis->fields[field_num];
-    int store_positions = fi_store_positions(fi);
-
-    ary_grow(tvw->fields);
-    ary_last(tvw->fields).field_num = field_num;
-
-    os_write_vint(tvd_out, posting_count);
-    for (i = 0; i < posting_count; i++) {
-        plist = plists[i];
-        posting = plist->last;
-        delta_start = hlp_string_diff(last_term, plist->term);
-        delta_length = plist->term_len - delta_start;
-
-        os_write_vint(tvd_out, delta_start);  /* write shared prefix length */
-        os_write_vint(tvd_out, delta_length); /* write delta length */
-        /* write delta chars */
-        os_write_bytes(tvd_out,
-                       (uchar *)(plist->term + delta_start),
-                       delta_length);
-        os_write_vint(tvd_out, posting->freq);
-        last_term = plist->term;
-
-        if (store_positions) {
-            /* use delta encoding for positions */
-            int last_pos = 0;
-            for (occ = posting->first_occ; occ; occ = occ->next) {
-                os_write_vint(tvd_out, occ->pos - last_pos);
-                last_pos = occ->pos;
-            }
-        }
-
-    }
-
-    if (fi_store_offsets(fi)) {
-        /* use delta encoding for offsets */
-        int last_end = 0;
-        os_write_vint(tvd_out, offset_count);  /* write shared prefix length */
-        for (i = 0; i < offset_count; i++) {
-            int start = offsets[i].start;
-            int end = offsets[i].end;
-            os_write_vint(tvd_out, start - last_end);
-            os_write_vint(tvd_out, end - start);
-            last_end = end;
-        }
-    }
-
-    ary_last(tvw->fields).size = os_pos(tvd_out) - tvd_start_pos;
-}
-
-

data/ext/threading.h

@@ -4,28 +4,28 @@
 #include "hash.h"
 #define UNTHREADED 1
 
-typedef void * mutex_t;
-typedef struct HashTable *thread_key_t;
-typedef int thread_once_t;
-#define MUTEX_INITIALIZER NULL
-#define MUTEX_RECURSIVE_INITIALIZER NULL
-#define THREAD_ONCE_INIT 1;
-#define mutex_init(a, b)
-#define mutex_lock(a)
-#define mutex_trylock(a)
-#define mutex_unlock(a)
-#define mutex_destroy(a)
-#define thread_key_create(a, b) frt_thread_key_create(a, b)
-#define thread_key_delete(a) frt_thread_key_delete(a)
-#define thread_setspecific(a, b) frt_thread_setspecific(a, b)
-#define thread_getspecific(a) frt_thread_getspecific(a)
-#define thread_exit(a)
-#define thread_once(a, b) frt_thread_once(a, b)
+typedef void * frt_mutex_t;
+typedef struct FrtHash *frt_thread_key_t;
+typedef int frt_thread_once_t;
+#define FRT_MUTEX_INITIALIZER NULL
+#define FRT_MUTEX_RECURSIVE_INITIALIZER NULL
+#define FRT_THREAD_ONCE_INIT 1;
+#define frt_mutex_init(a, b)
+#define frt_mutex_lock(a)
+#define frt_mutex_trylock(a)
+#define frt_mutex_unlock(a)
+#define frt_mutex_destroy(a)
+#define frt_thread_key_create(a, b) frb_thread_key_create(a, b)
+#define frt_thread_key_delete(a) frb_thread_key_delete(a)
+#define frt_thread_setspecific(a, b) frb_thread_setspecific(a, b)
+#define frt_thread_getspecific(a) frb_thread_getspecific(a)
+#define frt_thread_exit(a)
+#define frt_thread_once(a, b) frb_thread_once(a, b)
 
-void frt_thread_once(int *once_control, void (*init_routine)(void));
-void frt_thread_key_create(thread_key_t *key, void (*destr_function)(void *));
-void frt_thread_key_delete(thread_key_t key);
-void frt_thread_setspecific(thread_key_t key, const void *pointer);
-void *frt_thread_getspecific(thread_key_t key);
+void frb_thread_once(int *once_control, void (*init_routine)(void));
+void frb_thread_key_create(frt_thread_key_t *key, frt_free_ft destroy);
+void frb_thread_key_delete(frt_thread_key_t key);
+void frb_thread_setspecific(frt_thread_key_t key, const void *pointer);
+void *frb_thread_getspecific(frt_thread_key_t key);
 
 #endif

data/ext/win32.h

@@ -1,8 +1,11 @@
-#include "global.h"
-
 #ifndef FRT_WIN32_H
 #define FRT_WIN32_H
 
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "global.h"
 #include <io.h>
 
 struct dirent
@@ -19,8 +22,8 @@ typedef struct DIR
 
 DIR *opendir(const char *dirname)
 {
-    DIR *d = ALLOC_AND_ZERO(DIR);
-    char dirname_buf[MAX_FILE_PATH];
+    DIR *d = FRT_ALLOC_AND_ZERO(DIR);
+    char dirname_buf[FRT_MAX_FILE_PATH];
     long ff_res;
     sprintf(dirname_buf, "%s\\*", dirname);
     ff_res = _findfirst(dirname_buf, &d->find_data);
@@ -51,4 +54,9 @@ void closedir(DIR *d)
     _findclose(d->handle);
     free(d);
 }
+
+#ifdef __cplusplus
+} // extern "C"
+#endif
+
 #endif