ferret 0.9.6 → 0.10.0

data/CHANGELOG

@@ -1,9 +0,0 @@
-20060316:
-  * changed Token#term_text to Token#text
-  * changed Token#position_increment to Term#pos_inc
-  * changed order of args to Token.new. Now Term.new(text, start_offset,
-    end_offset, pos_inc=1, type="text"). NOTE: type does nothing.
-  * changed TermVectorOffsetInfo#start_offset to TermVectorOffsetInfo#start
-  * changed TermVectorOffsetInfo#end_offset to TermVectorOffsetInfo#end
-  * added :id_field option to Index::Index class.
-

data/ext/field.c

@@ -1,408 +0,0 @@
-#include "index.h"
-#include <string.h>
-
-/****************************************************************************
- *
- * FieldInfo
- *
- ****************************************************************************/
-
-FieldInfo *fi_create(char *name, int number, bool is_indexed,
-    bool store_tv, bool store_pos, bool store_offset, bool omit_norms)
-{
-  FieldInfo *fi = ALLOC(FieldInfo);
-  fi->name = estrdup(name);
-  fi->number = number;
-  fi->is_indexed = is_indexed;
-  fi->store_tv = store_tv;
-  fi->store_offset = store_offset;
-  fi->store_pos = store_pos;
-  fi->omit_norms = omit_norms;
-  return fi;
-}
-
-void fi_destroy(FieldInfo *fi)
-{
-  free(fi->name);
-  free(fi);
-}
-
-/****************************************************************************
- *
- * FieldInfos
- *
- ****************************************************************************/
-
-FieldInfos *fis_create()
-{
-  FieldInfos *fis = ALLOC(FieldInfos);
-  fis->by_name = ht_create();
-  fis->by_number = NULL;
-  fis->fcnt = 0;
-  return fis;
-}
-
-FieldInfos *fis_open(Store *store, char *filename)
-{
-  FieldInfos *fis = fis_create();
-  InStream *is = store->open_input(store, filename);
-  TRY
-    fis_read(fis, is);
-  XFINALLY
-    is_close(is);
-  XENDTRY
-  return fis;
-}
-
-void fis_destroy(FieldInfos *fis)
-{
-  int i;
-  for (i = 0; i < fis->fcnt; i++) {
-    fi_destroy(fis->by_number[i]);
-  }
-  ht_destroy(fis->by_name);
-  free(fis->by_number);
-  free(fis);
-}
-
-FieldInfo *fis_add(FieldInfos *fis,
-    char *name,
-    bool is_indexed,
-    bool store_tv,
-    bool store_pos,
-    bool store_offset,
-    bool omit_norms)
-{
-  FieldInfo *fi = ht_get(fis->by_name, name);
-  if (fi == NULL) {
-    fi = fi_create(name, fis->fcnt, is_indexed, store_tv,
-        store_pos, store_offset, omit_norms);
-    fis->fcnt++;
-    REALLOC_N(fis->by_number, FieldInfo *, fis->fcnt);
-
-    fis->by_number[fi->number] = fi;
-    ht_set(fis->by_name, name, fi);
-  } else {
-    if (fi->is_indexed != is_indexed)
-      fi->is_indexed = true;             // once indexed, always index
-    if (fi->store_tv != store_tv)
-      fi->store_tv = true;               // once vector, always vector
-    if (fi->store_pos != store_pos)
-      fi->store_pos = true;              // once vector, always vector
-    if (fi->store_offset != store_offset)
-      fi->store_offset = true;           // once vector, always vector
-    if (fi->omit_norms != omit_norms)
-      fi->omit_norms = false;            // once kept, always keep
-  }
-  return fi;
-}
-
-void fis_add_fields(FieldInfos *fis,
-    HashSet *field_names,
-    bool is_indexed,
-    bool store_tv,
-    bool store_pos,
-    bool store_offset,
-    bool omit_norms)
-{
-  int i; 
-  for (i = 0; i < field_names->size; i++) {
-    fis_add(fis, field_names->elems[i], is_indexed, store_tv, store_pos,
-        store_offset, omit_norms);
-  }
-  hs_destroy(field_names);
-}
-
-bool fis_has_vectors(FieldInfos *fis)
-{
-  int i;
-  for (i = 0; i < fis->fcnt; i++) {
-    if (fis->by_number[i]->store_tv)
-      return true;
-  }
-  return false;
-}
-
-FieldInfo *fis_get_fi(FieldInfos *fis, char *name)
-{
-  return (FieldInfo *)ht_get(fis->by_name, name);
-}
-
-ullong fis_get_number(FieldInfos *fis, char *name)
-{
-  FieldInfo *fi = (FieldInfo *)ht_get(fis->by_name, name);
-  if (fi == NULL)
-    return NOT_A_FIELD;
-  else
-    return fi->number;
-}
-
-#define IS_INDEXED   0x01
-#define STORE_TV     0x02
-#define STORE_POS    0x04
-#define STORE_OFFSET 0x08
-#define OMIT_NORMS   0x10
-
-int fi_field_info_byte(FieldInfo *fi)
-{
-  int bits = 0x0;
-  if (fi->is_indexed)
-    bits |= IS_INDEXED;
-  if (fi->store_tv)
-    bits |= STORE_TV;
-  if (fi->store_pos)
-    bits |= STORE_POS;
-  if (fi->store_offset)
-    bits |= STORE_OFFSET;
-  if (fi->omit_norms)
-    bits |= OMIT_NORMS;
-  return bits;
-}
-
-void fis_write(FieldInfos *fis, Store *store, char *segment, char *ext)
-{
-  int i;
-  FieldInfo *fi;
-  char fname[SEGMENT_NAME_MAX_LENGTH];
-  OutStream *os;
-  
-  strcpy(fname, segment);
-  strcat(fname, ext);
-  os = store->create_output(store, fname);
-  TRY
-    os_write_vint(os, fis->fcnt);
-    for (i = 0; i < fis->fcnt; i++) {
-      fi = fis->by_number[i];
-      os_write_string(os, fi->name);
-      os_write_vint(os, fi_field_info_byte(fi));
-    }
-  XFINALLY
-    os_close(os);
-  XENDTRY
-}
-
-FieldInfos *fis_read(FieldInfos *fis, InStream *is)
-{
-  int i, size = (int)is_read_vint(is); /* read in the size */
-  int bits, is_indexed, store_tv, store_pos, store_offset, omit_norms;
-  char *name;
-  for (i = 0; i < size; i++) {
-    name = is_read_string(is);
-    bits = is_read_byte(is);
-    is_indexed = (bits & IS_INDEXED) != 0;
-    store_tv = (bits & STORE_TV) != 0;
-    store_pos = (bits & STORE_POS) != 0;
-    store_offset = (bits & STORE_OFFSET) != 0;
-    omit_norms = (bits & OMIT_NORMS) != 0;
-    fis_add(fis, name, is_indexed, store_tv,
-        store_pos, store_offset, omit_norms);
-    free(name);
-  }
-  return fis;
-}
-
-FieldInfos *fis_add_doc(FieldInfos *fis, Document *doc)
-{
-  int i;
-  DocField *df;
-  for (i = 0; i < doc->dfcnt; i++) {
-    df = doc->df_arr[i];
-    fis_add(fis, df->name, df->is_indexed, df->store_tv,
-        df->store_pos, df->store_offset, df->omit_norms);
-  }
-  return fis;
-}
-
-/****************************************************************************
- *
- * FieldsReader
- *
- ****************************************************************************/
-
-FieldsReader *fr_open(Store *store, char *segment, FieldInfos *fis)
-{
-  FieldsReader *fr = ALLOC(FieldsReader);
-  InStream *iin;
-  char buf[100];
-  int slen = (int)strlen(segment);
-  strcpy(buf, segment);
-
-  fr->fis = fis;
-  strcpy(buf+slen, ".fdt");
-  fr->fields_in = store->open_input(store, buf);
-  strcpy(buf+slen, ".fdx");
-  iin = fr->index_in = store->open_input(store, buf);
-  fr->len = iin->length_internal(iin)/8;
-  return fr;
-}
-
-void fr_close(FieldsReader *fr)
-{
-  is_close(fr->fields_in);
-  is_close(fr->index_in);
-  free(fr);
-}
-
-Document *fr_get_doc(FieldsReader *fr, int doc_num)
-{
-  int i, bits, dlen;
-  FieldInfo *fi;
-  char *data;
-  int store, index, stv;
-  int is_compressed, is_tokenized, is_binary;
-  int position, field_cnt, field_number;
-  Document *doc = doc_create();
-  InStream *iin = fr->index_in;
-  InStream *fin = fr->fields_in;
-  
-  is_seek(iin, doc_num * 8);
-  position = (int)is_read_long(iin);
-  is_seek(fin, position);
-  field_cnt = (int)is_read_vint(fin);
-  
-  for (i = 0; i < field_cnt; i++) {
-    field_number = (int)is_read_vint(fin);
-    fi = fr->fis->by_number[field_number];
-
-    bits = is_read_byte(fin);
-    
-    is_compressed = (bits & FIELD_IS_COMPRESSED) != 0;
-    is_tokenized = (bits & FIELD_IS_TOKENIZED) != 0;
-    is_binary = (bits & FIELD_IS_BINARY) != 0;
-    
-    if (is_binary) {
-      dlen = (int)is_read_vint(fin);
-      data = ALLOC_N(char, dlen);
-      is_read_bytes(fin, (uchar *)data, 0, dlen);
-      if (is_compressed) {
-        doc_add_field(doc, df_create_binary(fi->name, data, dlen, DF_STORE_COMPRESS));
-      } else {
-        doc_add_field(doc, df_create_binary(fi->name, data, dlen, DF_STORE_YES));
-      }
-    } else {
-      store = DF_STORE_YES;
-      if (!fi->is_indexed) {
-        index = DF_INDEX_NO;
-      } else if (is_tokenized) {
-        index = DF_INDEX_TOKENIZED;
-      } else if (fi->omit_norms) {
-        index = DF_INDEX_NO_NORMS;
-      } else {
-        index = DF_INDEX_UNTOKENIZED;
-      }
-      data = NULL;
-      if (is_compressed) {
-        store = DF_STORE_COMPRESS;
-        dlen = (int)is_read_vint(fin);
-        data = ALLOC_N(char, (dlen + 1));
-        data[dlen] = '\0';
-        is_read_bytes(fin, (uchar *)data, 0, dlen);
-      } else {
-        data = is_read_string(fin);
-      }
-      stv =  DF_TERM_VECTOR_NO;
-      if (fi->store_tv) { 
-        if (fi->store_pos && fi->store_offset) { 
-          stv =  DF_TERM_VECTOR_WITH_POSITIONS_OFFSETS;
-        } else if (fi->store_pos) { 
-          stv =  DF_TERM_VECTOR_WITH_POSITIONS;
-        } else if (fi->store_offset) { 
-          stv =  DF_TERM_VECTOR_WITH_OFFSETS;
-        } else { 
-          stv =  DF_TERM_VECTOR_YES;
-        }
-      }
-      doc_add_field(doc, df_create(fi->name, data, store, index, stv));
-    }
-  }
-
-  return doc;
-}
-
-/****************************************************************************
- *
- * FieldsWriter
- *
- ****************************************************************************/
-
-FieldsWriter *fw_open(Store *store, char *segment, FieldInfos *fis)
-{
-  FieldsWriter *fw = ALLOC(FieldsWriter);
-  char buf[SEGMENT_NAME_MAX_LENGTH];
-  int slen = (int)strlen(segment);
-
-  strcpy(buf, segment);
-
-  fw->fis = fis;
-  strcpy(buf+slen, ".fdt");
-  fw->fields_out = store->create_output(store, buf);
-  strcpy(buf+slen, ".fdx");
-  fw->index_out = store->create_output(store, buf);
-  return fw;
-}
-
-void fw_close(FieldsWriter *fw)
-{
-  os_close(fw->fields_out);
-  os_close(fw->index_out);
-  free(fw);
-}
-
-void save_data(OutStream *fout, char *data, int dlen)
-{
-  os_write_vint(fout, dlen);
-  os_write_bytes(fout, (uchar *)data, dlen);
-}
-
-void fw_add_doc(FieldsWriter *fw, Document *doc)
-{
-  int i, bits;
-  DocField *df;
-  char *data;
-  int stored_count = 0;
-  OutStream *fout = fw->fields_out, *iout = fw->index_out;
-
-  os_write_long(iout, os_pos(fout));
-
-  for (i = 0; i < doc->dfcnt; i++) {
-    if (doc->df_arr[i]->is_stored)
-      stored_count++;
-  }
-  os_write_vint(fout, stored_count);
-  
-  for (i = 0; i < doc->dfcnt; i++) {
-    df = doc->df_arr[i];
-    if (df->is_stored) {
-      os_write_vint(fout, ((FieldInfo *)ht_get(fw->fis->by_name, df->name))->number);
-
-      bits = 0;
-      if (df->is_tokenized) {
-        bits |= FIELD_IS_TOKENIZED;
-      }
-      if (df->is_binary) {
-        bits |= FIELD_IS_BINARY;
-      }
-      if (df->is_compressed) {
-        bits |= FIELD_IS_COMPRESSED;
-      }
-      os_write_byte(fout, bits);
-      
-      data = NULL;
-      if (df->is_compressed) {
-        /* Not compressing just yet but we'll save it anyway */
-        if (df->is_binary) {
-          save_data(fout, df->data, df->blen);
-        } else {
-          os_write_string(fout, df->data);
-        }
-      } else {
-        if (df->is_binary) {
-          save_data(fout, df->data, df->blen);
-        } else {
-          os_write_string(fout, df->data);
-        }
-      }
-    }
-  }
-}
-

data/ext/frtio.h

@@ -1,13 +0,0 @@
-#ifndef FRT_IO_H
-#define FRT_IO_H
-
-extern char *join_path(char *buf, const char *base, const char *filename);
-extern bool exists(char *path);
-extern int fcount(char *path);
-extern void dir_each(char *path, void (*func)(char *fname, void *arg), void *arg);
-extern void fs_clear_locks(Store *store);
-extern void fs_clear(Store *store);
-extern void fs_clear_all(Store *store);
-
-#endif
-

data/ext/inc/except.h

@@ -1,90 +0,0 @@
-#ifndef FRT_EXCEPT_H
-#define FRT_EXCEPT_H
-
-#include <setjmp.h>
-#include <ruby.h>
-
-#define BODY 0
-#define FINALLY -1
-#define EXCEPTION 1
-#define IO_ERROR 2
-#define ARG_ERROR 3
-#define EOF_ERROR 4
-#define UNSUPPORTED_ERROR 5
-#define STATE_ERROR 6
-#define PARSE_ERROR 7
-#define MEM_ERROR 8
-
-typedef struct xcontext_t {
-  jmp_buf jbuf;
-  struct xcontext_t *next;
-  char *msg;
-  volatile int excode;
-  int handled : 1;
-  int in_finally : 1;
-} xcontext_t;
-
-RUBY_EXTERN int rb_thread_critical;
-extern xcontext_t *xtop_context;
-
-#define TRY\
-  do {\
-    xcontext_t xcontext;\
-    rb_thread_critical = Qtrue;\
-    xcontext.next = xtop_context;\
-    xtop_context = &xcontext;\
-    xcontext.handled = true;\
-    xcontext.in_finally = false;\
-    switch (setjmp(xcontext.jbuf)) {\
-      case BODY:
-
-
-#define XENDTRY\
-    }\
-    xtop_context = xcontext.next;\
-    if (!xcontext.handled) {\
-      RAISE(xcontext.excode, xcontext.msg);\
-    }\
-    rb_thread_critical = 0;\
-  } while (0);
-
-#define ENDTRY\
-    }\
-    if (!xcontext.in_finally) {\
-      xtop_context = xcontext.next;\
-      if (!xcontext.handled) {\
-        RAISE(xcontext.excode, xcontext.msg);\
-      }\
-      xcontext.in_finally = 1;\
-      longjmp(xcontext.jbuf, FINALLY);\
-    }\
-    rb_thread_critical = 0;\
-  } while (0);
-
-#define XFINALLY default: xcontext.in_finally = 1;
-
-#define XCATCHALL break; default: xcontext.in_finally = 1;
-
-//fprintf(stderr,"Error occured in %s, %d: %s\n", __FILE__, __LINE__, __func__);
-#define RAISE(xexcode, xmsg) \
-  do {\
-    if (!xtop_context) {\
-      eprintf(EXCEPTION_CODE, "Error: exception %d not handled: %s", xexcode, xmsg);\
-    } else if (!xtop_context->in_finally) {\
-      xtop_context->msg = xmsg;\
-      xtop_context->excode = xexcode;\
-      xtop_context->handled = false;\
-      longjmp(xtop_context->jbuf, xexcode);\
-    } else if (xtop_context->handled) {\
-      xtop_context->msg = xmsg;\
-      xtop_context->excode = xexcode;\
-      xtop_context->handled = false;\
-    }\
-  } while (0)
-
-#define HANDLED() xcontext.handled = 1 /* true */
-
-extern char * const UNSUPPORTED_ERROR_MSG;
-extern char * const EOF_ERROR_MSG;
-
-#endif