ferret 0.9.0 → 0.9.1

data/ext/analysis.h

@@ -30,27 +30,58 @@ int tk_cmp(Token *tk1, Token *tk2);
  *
  ****************************************************************************/
 
+
 typedef struct TokenStream TokenStream;
 struct TokenStream {
   void *data;
   char *text;
-  int pos;
+  char *t; /* ptr used to scan text */
   Token *token;
   Token *(*next)(TokenStream *ts);
   void (*reset)(TokenStream *ts, char *text);
+  void (*clone_i)(TokenStream *orig_ts, TokenStream *new_ts);
   void (*destroy)(void *p);
-  TokenStream *sub_ts; // used by filters
+  TokenStream *sub_ts; /* used by filters */
+  bool destroy_sub : 1;
 };
 
 #define ts_next(mts) mts->next(mts)
 #define ts_destroy(mts) mts->destroy(mts)
 
 TokenStream *whitespace_tokenizer_create();
+TokenStream *mb_whitespace_tokenizer_create(bool lowercase);
+
 TokenStream *letter_tokenizer_create();
+TokenStream *mb_letter_tokenizer_create(bool lowercase);
+
 TokenStream *standard_tokenizer_create();
+TokenStream *mb_standard_tokenizer_create();
+
 TokenStream *lowercase_filter_create(TokenStream *ts);
-TokenStream *stop_filter_create_with_words(TokenStream *ts, char **words, int len);
+TokenStream *mb_lowercase_filter_create(TokenStream *ts);
+
+extern const char *ENGLISH_STOP_WORDS[];
+extern const char *FULL_ENGLISH_STOP_WORDS[];
+extern const char *EXTENDED_ENGLISH_STOP_WORDS[];
+extern const char *FULL_FRENCH_STOP_WORDS[];
+extern const char *FULL_SPANISH_STOP_WORDS[];
+extern const char *FULL_PORTUGUESE_STOP_WORDS[];
+extern const char *FULL_ITALIAN_STOP_WORDS[];
+extern const char *FULL_GERMAN_STOP_WORDS[];
+extern const char *FULL_DUTCH_STOP_WORDS[];
+extern const char *FULL_SWEDISH_STOP_WORDS[];
+extern const char *FULL_NORWEGIAN_STOP_WORDS[];
+extern const char *FULL_DANISH_STOP_WORDS[];
+extern const char *FULL_RUSSIAN_STOP_WORDS[];
+extern const char *FULL_FINNISH_STOP_WORDS[];
+
+TokenStream *stop_filter_create_with_words_len(TokenStream *ts,
+    const char **words, int len);
+TokenStream *stop_filter_create_with_words(TokenStream *ts, const char **words);
 TokenStream *stop_filter_create(TokenStream *ts);
+TokenStream *stem_filter_create(TokenStream *ts, const char * algorithm,
+    const char * charenc);
+TokenStream *ts_clone(TokenStream *orig_ts);
 
 /****************************************************************************
  *
@@ -67,10 +98,27 @@ typedef struct Analyzer {
 
 #define a_destroy(ma) ma->destroy(ma)
 #define a_get_ts(ma, field, text) ma->get_ts(ma, field, text)
+#define a_get_new_ts(ma, field, text) ts_clone(ma->get_ts(ma, field, text))
+
+Analyzer *whitespace_analyzer_create(bool lowercase);
+Analyzer *mb_whitespace_analyzer_create(bool lowercase);
+
+Analyzer *letter_analyzer_create(bool lowercase);
+Analyzer *mb_letter_analyzer_create(bool lowercase);
+
+Analyzer *standard_analyzer_create(bool lowercase);
+Analyzer *mb_standard_analyzer_create(bool lowercase);
+
+Analyzer *standard_analyzer_create_with_words(
+    const char **words, bool lowercase);
+Analyzer *standard_analyzer_create_with_words_len(
+    const char **words, int len, bool lowercase);
+Analyzer *mb_standard_analyzer_create_with_words(
+    const char **words, bool lowercase);
+Analyzer *mb_standard_analyzer_create_with_words_len(
+    const char **words, int len, bool lowercase);
 
-Analyzer *whitespace_analyzer_create();
-Analyzer *letter_analyzer_create();
-Analyzer *standard_analyzer_create();
-Analyzer *standard_analyzer_create_with_words(char **words, int len);
+Analyzer *per_field_analyzer_create(Analyzer *def, bool destroy_subs);
+void pfa_add_field(Analyzer *self, char *field, Analyzer *analyzer);
 
 #endif

data/ext/api.c

@@ -0,0 +1,69 @@
+
+#include <stdlib.h> /* for calloc, free */
+#include "header.h"
+
+extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size)
+{
+    struct SN_env * z = (struct SN_env *) calloc(1, sizeof(struct SN_env));
+    if (z == NULL) return NULL;
+    z->p = create_s();
+    if (z->p == NULL) goto error;
+    if (S_size)
+    {
+        int i;
+        z->S = (symbol * *) calloc(S_size, sizeof(symbol *));
+        if (z->S == NULL) goto error;
+
+        for (i = 0; i < S_size; i++)
+        {
+            z->S[i] = create_s();
+            if (z->S[i] == NULL) goto error;
+        }
+        z->S_size = S_size;
+    }
+
+    if (I_size)
+    {
+        z->I = (int *) calloc(I_size, sizeof(int));
+        if (z->I == NULL) goto error;
+        z->I_size = I_size;
+    }
+
+    if (B_size)
+    {
+        z->B = (symbol *) calloc(B_size, sizeof(symbol));
+        if (z->B == NULL) goto error;
+        z->B_size = B_size;
+    }
+
+    return z;
+error:
+    SN_close_env(z);
+    return NULL;
+}
+
+extern void SN_close_env(struct SN_env * z)
+{
+    if (z == NULL) return;
+    if (z->S_size)
+    {
+        int i;
+        for (i = 0; i < z->S_size; i++)
+        {
+            lose_s(z->S[i]);
+        }
+        free(z->S);
+    }
+    if (z->I_size) free(z->I);
+    if (z->B_size) free(z->B);
+    if (z->p) lose_s(z->p);
+    free(z);
+}
+
+extern int SN_set_current(struct SN_env * z, int size, const symbol * s)
+{
+    int err = replace_s(z, 0, z->l, size, s, NULL);
+    z->c = 0;
+    return err;
+}
+

data/ext/api.h

@@ -0,0 +1,27 @@
+
+typedef unsigned char symbol;
+
+/* Or replace 'char' above with 'short' for 16 bit characters.
+
+   More precisely, replace 'char' with whatever type guarantees the
+   character width you need. Note however that sizeof(symbol) should divide
+   HEAD, defined in header.h as 2*sizeof(int), without remainder, otherwise
+   there is an alignment problem. In the unlikely event of a problem here,
+   consult Martin Porter.
+
+*/
+
+struct SN_env {
+    symbol * p;
+    int c; int a; int l; int lb; int bra; int ket;
+    int S_size; int I_size; int B_size;
+    symbol * * S;
+    int * I;
+    symbol * B;
+};
+
+extern struct SN_env * SN_create_env(int S_size, int I_size, int B_size);
+extern void SN_close_env(struct SN_env * z);
+
+extern int SN_set_current(struct SN_env * z, int size, const symbol * s);
+

data/ext/array.c

@@ -22,9 +22,11 @@ void ary_destroy(void *p)
 {
   Array *ary = (Array *)p;
   int i;
-  for (i = 0; i < ary->size; i++) {
-    if (ary->free_elem != NULL && ary->elems[i] != NULL)
-      ary->free_elem(ary->elems[i]);
+  if (ary->free_elem) {
+    for (i = 0; i < ary->size; i++) {
+      if (ary->elems[i])
+        ary->free_elem(ary->elems[i]);
+    }
   }
   free(ary->elems);
   free(ary);
@@ -41,7 +43,7 @@ void ary_set(Array *ary, int index, void *value)
   if (index >= ary->size)
     ary->size = index + 1;
 
-  if (ary->free_elem != NULL && ary->elems[index] != NULL)
+  if (ary->free_elem && ary->elems[index])
     ary->free_elem(ary->elems[index]);
 
   ary->elems[index] = value;
@@ -63,7 +65,8 @@ void ary_delete(Array *ary, int index)
 {
   if (index >= ary->size)
     return;
-  ary->free_elem(ary->elems[index]);
+  if (ary->free_elem && ary->elems[index])
+    ary->free_elem(ary->elems[index]);
   ary->elems[index] = NULL;
   if (index == ary->size - 1)
     ary->size--;

data/ext/compound_io.c

@@ -1,4 +1,12 @@
-#include "index.h"
+#include "index.h" 
+static char * const ALREADY_CLOSED_MSG = "Already closed";
+static char * const STREAM_CLOSED_MSG = "Stream closed";
+static char * const MISSING_FILE_MSG = "No sub-file found";
+static char * const ALREADY_MERGED_MSG = "Already merged";
+static char * const REMAINDER_ERROR_MSG = "Non-zero remainder length after copying";
+static char * const FILE_OFFSET_MSG =  "Difference in the output file offsets"
+        " does not match the original file length";
+static char * const NO_FILES_TO_MERGE_MSG = "No Files to merge into the compound file";
 
 /****************************************************************************
  *
@@ -24,15 +32,21 @@ int cmpd_exists(Store *store, char *filename)
     return false;
 }
 
+/**
+ * @throws UNSUPPORTED_ERROR
+ */
 int cmpd_remove(Store *store, char *filename)
 {
-  eprintf(UNSUPPORTED_ERROR, "Unsupported operation");
+  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
   return 0;
 }
 
+/**
+ * @throws UNSUPPORTED_ERROR
+ */
 int cmpd_rename(Store *store, char *from, char *to)
 {
-  eprintf(UNSUPPORTED_ERROR, "Unsupported operation");
+  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
   return 0;
 }
 
@@ -41,9 +55,12 @@ int cmpd_count(Store *store)
   return store->dir.cmpd->entries->used;
 }
 
+/**
+ * @throws UNSUPPORTED_ERROR
+ */
 void cmpd_clear(Store *store)
 {
-  eprintf(UNSUPPORTED_ERROR, "Unsupported operation");
+  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
 }
 
 void cmpd_close(Store *store)
@@ -51,7 +68,7 @@ void cmpd_close(Store *store)
   mutex_lock(&store->mutex);
   CompoundStore *cmpd = store->dir.cmpd;
   if (cmpd->stream == NULL)
-    eprintf(IO_ERROR, "Already closed");
+    RAISE(IO_ERROR, ALREADY_CLOSED_MSG);
 
   h_destroy(cmpd->entries);
 
@@ -92,12 +109,15 @@ int cmpdi_length_internal(InStream *is)
   return (is->d.cis->length);
 }
 
+/*
+ * raises: EOF_ERROR
+ */
 void cmpdi_read_internal(InStream *is, uchar *b, int offset, int len)
 {
   CompoundInStream *cis = is->d.cis;
   int start = is_pos(is);
   if ((start + len) > cis->length)
-    eprintf(EOF_ERROR, "read past EOF");
+    RAISE(EOF_ERROR, EOF_ERROR_MSG);
   is_seek(cis->sub, cis->offset + start);
   is_read_bytes(cis->sub, b, offset, len);
 }
@@ -129,13 +149,13 @@ InStream *cmpd_open_input(Store *store, const char *filename)
   mutex_lock(&store->mutex);
   if (cmpd->stream == NULL) {
     mutex_unlock(&store->mutex);
-    eprintf(IO_ERROR, "Stream closed");
+    RAISE(IO_ERROR, STREAM_CLOSED_MSG);
   }
 
   FileEntry *entry = (FileEntry *)h_get(cmpd->entries, filename);
   if (entry == NULL) {
     mutex_unlock(&store->mutex);
-    eprintf(IO_ERROR, "No sub-file with id <%s> found", filename);
+    RAISE(IO_ERROR, MISSING_FILE_MSG);
   }
 
   is = cmpd_create_input(cmpd->stream, entry->offset, entry->length);
@@ -146,53 +166,64 @@ InStream *cmpd_open_input(Store *store, const char *filename)
 
 OutStream *cmpd_create_output(Store *store, const char *filename)
 {
-  eprintf(UNSUPPORTED_ERROR, "Unsupported operation");
+  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
   return NULL;
 }
 
 Lock *cmpd_open_lock(Store *store, char *lockname)
 {
-  eprintf(UNSUPPORTED_ERROR, "Unsupported operation");
+  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
   return NULL;
 }
 
 void cmpd_close_lock(Lock *lock)
 {
-  eprintf(UNSUPPORTED_ERROR, "Unsupported operation");
+  RAISE(UNSUPPORTED_ERROR, UNSUPPORTED_ERROR_MSG);
 }
 
 Store *open_cmpd_store(Store *store, const char *name)
 {
-  CompoundStore *cmpd = ALLOC(CompoundStore);
-  Store *new_store = store_create();
-
-  cmpd->store   = store;
-  cmpd->name    = name;
-  cmpd->entries = h_new_str(&efree, &efree);
-  InStream *is = cmpd->stream = store->open_input(store, cmpd->name);
-
-  // read the directory and init files
-  int count = is_read_vint(is);
-  FileEntry *entry = NULL;
-  int i, offset;
-  char *fname;
-  for (i = 0; i < count; i++) {
-    offset = is_read_long(is);
-    fname = is_read_string(is);
-
-    if (entry != NULL) {
-      // set length of the previous entry
-      entry->length = offset - entry->offset;
+  Store * volatile new_store = NULL;
+  CompoundStore * volatile cmpd = NULL;
+  InStream * volatile is = NULL;
+
+  TRY
+    new_store = store_create();
+    cmpd = ALLOC(CompoundStore);
+
+    cmpd->store   = store;
+    cmpd->name    = name;
+    cmpd->entries = h_new_str(&efree, &efree);
+    is = cmpd->stream = store->open_input(store, cmpd->name);
+
+    // read the directory and init files
+    int count = is_read_vint(is);
+    FileEntry *entry = NULL;
+    int i, offset;
+    char *fname;
+    for (i = 0; i < count; i++) {
+      offset = is_read_long(is);
+      fname = is_read_string(is);
+
+      if (entry != NULL) {
+        // set length of the previous entry
+        entry->length = offset - entry->offset;
+      }
+
+      entry = ALLOC(FileEntry);
+      entry->offset = offset;
+      h_set(cmpd->entries, fname, entry);
     }
 
-    entry = ALLOC(FileEntry);
-    entry->offset = offset;
-    h_set(cmpd->entries, fname, entry);
-  }
+    // set the length of the final entry
+    if (entry != NULL)
+      entry->length = is_length(is) - entry->offset;
+  XCATCHALL
+    free(new_store);
+    free(cmpd);
+    if (is) is_close(is);
+  XENDTRY
 
-  // set the length of the final entry
-  if (entry != NULL)
-    entry->length = is_length(is) - entry->offset;
 
   new_store->dir.cmpd = cmpd;
   new_store->touch    = &cmpd_touch;
@@ -248,9 +279,9 @@ CompoundWriter *open_cw(Store *store, char *name)
 
 void cw_add_file(CompoundWriter *cw, char *id)
 {
-  if (cw->merged) eprintf(STATE_ERROR, "Already merged");
+  if (cw->merged) RAISE(STATE_ERROR, ALREADY_MERGED_MSG);
   if (hs_add(cw->ids, id) != HASH_KEY_DOES_NOT_EXIST)
-    eprintf(STATE_ERROR, "Already merged");
+    RAISE(STATE_ERROR, ALREADY_MERGED_MSG);
 
   hs_add(cw->ids, id);
   ary_append(cw->file_entries, wfe_create(id));
@@ -258,78 +289,83 @@ void cw_add_file(CompoundWriter *cw, char *id)
 
 void cw_copy_file(CompoundWriter *cw, WFileEntry *src, OutStream *os)
 {
-
   int start_ptr = os_pos(os);
+  int remainder, length, len;
 
   InStream *is = cw->store->open_input(cw->store, src->name);
-  int remainder, length, len;
-  remainder = length = is_length(is);
-
-  uchar buffer[BUFFER_SIZE];
-  while (remainder > 0) {
-    len = MIN(remainder, BUFFER_SIZE);
-    is_read_bytes(is, buffer, 0, len);
-    os_write_bytes(os, buffer, len);
-    remainder -= len;
-  }
+  
+  TRY
+    remainder = length = is_length(is);
+
+    uchar buffer[BUFFER_SIZE];
+    while (remainder > 0) {
+      len = MIN(remainder, BUFFER_SIZE);
+      is_read_bytes(is, buffer, 0, len);
+      os_write_bytes(os, buffer, len);
+      remainder -= len;
+    }
 
-  // Verify that remainder is 0
-  if (remainder != 0)
-    eprintf(IO_ERROR, "Non-zero remainder length after copying: %ld "
-        "(id:%s, length: %ld, buffer size: %ld\n", remainder,
-        src->name, length, BUFFER_SIZE);
+    // Verify that remainder is 0
+    if (remainder != 0)
+      RAISE(IO_ERROR, REMAINDER_ERROR_MSG);
 
-  // Verify that the output length diff is equal to original file
-  int end_ptr = os_pos(os);
-  int diff = end_ptr - start_ptr;
-  if (diff != length)
-    eprintf(IO_ERROR, "Difference in the output file offsets %ld "
-        " does not match the original file length ", diff, length);
+    // Verify that the output length diff is equal to original file
+    int end_ptr = os_pos(os);
+    int diff = end_ptr - start_ptr;
+    if (diff != length)
+      RAISE(IO_ERROR, FILE_OFFSET_MSG);
 
-  is_close(is);
+  XFINALLY
+    is_close(is);
+  XENDTRY
 }
 
 void cw_close(CompoundWriter *cw)
 {
-  if (cw->merged) eprintf(STATE_ERROR, "Already merged");
+  if (cw->merged) RAISE(STATE_ERROR, ALREADY_MERGED_MSG);
   if (cw->ids->size <= 0)
-    eprintf(STATE_ERROR, "No Files to merge into the compound file");
+    RAISE(STATE_ERROR, NO_FILES_TO_MERGE_MSG);
 
   cw->merged = true;
 
-  OutStream *os = cw->store->create_output(cw->store, cw->name);
-  os_write_vint(os, cw->file_entries->size);
-
-  /* Write the directory with all offsets at 0.
-   * Remember the positions of directory entries so that we can adjust the
-   * offsets later */
-  int i;
-  WFileEntry *wfe; 
-  for (i = 0; i < cw->file_entries->size; i++) {
-    wfe = (WFileEntry *)cw->file_entries->elems[i];
-    wfe->dir_offset = os_pos(os);
-    os_write_long(os, 0);  // for now
-    os_write_string(os, wfe->name);
-  }
+  OutStream * volatile os = NULL;
+  TRY
+    os = cw->store->create_output(cw->store, cw->name);
+    os_write_vint(os, cw->file_entries->size);
+
+    /* Write the directory with all offsets at 0.
+     * Remember the positions of directory entries so that we can adjust the
+     * offsets later */
+    int i;
+    WFileEntry *wfe; 
+    for (i = 0; i < cw->file_entries->size; i++) {
+      wfe = (WFileEntry *)cw->file_entries->elems[i];
+      wfe->dir_offset = os_pos(os);
+      os_write_long(os, 0);  // for now
+      os_write_string(os, wfe->name);
+    }
 
-  /* Open the files and copy their data into the stream.  Remember the
-   * locations of each file's data section. */
-  for (i = 0; i < cw->file_entries->size; i++) {
-    wfe = (WFileEntry *)cw->file_entries->elems[i];
-    wfe->data_offset = os_pos(os);
-    cw_copy_file(cw, wfe, os);
-  }
+    /* Open the files and copy their data into the stream.  Remember the
+     * locations of each file's data section. */
+    for (i = 0; i < cw->file_entries->size; i++) {
+      wfe = (WFileEntry *)cw->file_entries->elems[i];
+      wfe->data_offset = os_pos(os);
+      cw_copy_file(cw, wfe, os);
+    }
 
-  /* Write the data offsets into the directory of the compound stream */
-  for (i = 0; i < cw->file_entries->size; i++) {
-    wfe = (WFileEntry *)cw->file_entries->elems[i];
-    os_seek(os, wfe->dir_offset);
-    os_write_long(os, wfe->data_offset);
-  }
+    /* Write the data offsets into the directory of the compound stream */
+    for (i = 0; i < cw->file_entries->size; i++) {
+      wfe = (WFileEntry *)cw->file_entries->elems[i];
+      os_seek(os, wfe->dir_offset);
+      os_write_long(os, wfe->data_offset);
+    }
 
-  os_close(os);
-  hs_destroy(cw->ids);
-  ary_destroy(cw->file_entries);
-  free(cw);
+  XFINALLY
+    if (os) os_close(os);
+    hs_destroy(cw->ids);
+    ary_destroy(cw->file_entries);
+    free(cw);
+    break;
+  XENDTRY
 }