ferret 0.10.13 → 0.10.14

data/Rakefile

@@ -41,7 +41,7 @@ SRC = (FileList["ext/*.[ch]"] + EXT_SRC_DEST).uniq
 
 CLEAN.include(FileList['**/*.o', '**/*.obj', 'InstalledFiles',
                        '.config', 'ext/cferret.c'])
-CLOBBER.include(FileList['**/*.so'], 'ext/Makefile', 'ext/mem_pool.*', 'ext/defines.h', EXT_SRC_DEST)
+CLOBBER.include(FileList['**/*.so'], 'ext/Makefile', EXT_SRC_DEST)
 POLISH = Rake::FileList.new.include(FileList['**/*.so'], 'ext/Makefile')
 
 desc "Clean specifically for the release."
@@ -115,7 +115,10 @@ EXT_SRC.each do |fn|
 end if File.exists?("../c")
 
 desc "Build the extension"
-task :ext => ["ext/#{EXT}"] + SRC
+task :ext => ["ext/#{EXT}"] + SRC do
+  rm_f 'ext/mem_pool.*'
+  rm_f 'ext/defines.h'
+end
 
 file "ext/#{EXT}" => ["ext/Makefile"] do
   cp "ext/inc/lang.h", "ext/lang.h"

data/ext/index.c

@@ -91,7 +91,7 @@ HashTable *co_hash_create()
  *
  ****************************************************************************/
 
- __inline void fi_set_store(FieldInfo *fi, int store)
+__inline void fi_set_store(FieldInfo *fi, int store)
 {
     switch (store) {
         case STORE_NO:

data/ext/q_phrase.c

@@ -898,7 +898,12 @@ static char *phq_to_s(Query *self, const char *field)
     char *buffer;
 
     if (phq->pos_cnt == 0) {
-        return NULL;
+        if (strcmp(field, phq->field) != 0) {
+            return strfmt("%s:\"\"", phq->field);
+        }
+        else {
+            return estrdup("\"\"");
+        }
     }
 
     /* sort the phrase positions by position */

data/ext/q_span.c

@@ -439,6 +439,208 @@ static SpanEnum *spante_new(Query *query, IndexReader *ir)
     return self;
 }
 
+/*****************************************************************************
+ * SpanMultiTermEnum
+ *****************************************************************************/
+
+/* * TermPosEnumWrapper * */
+#define TPE_READ_SIZE 16
+
+typedef struct TermPosEnumWrapper
+{
+    const char  *term;
+    TermDocEnum *tpe;
+    int          doc;
+    int          pos;
+} TermPosEnumWrapper;
+
+static bool tpew_less_than(const TermPosEnumWrapper *tpew1,
+                           const TermPosEnumWrapper *tpew2)
+{
+    return (tpew1->doc < tpew2->doc)
+        || (tpew1->doc == tpew2->doc && tpew1->pos < tpew2->pos);
+}
+
+static bool tpew_next(TermPosEnumWrapper *self)
+{
+    TermDocEnum *tpe = self->tpe;
+    if (0 > (self->pos = tpe->next_position(tpe))) {
+        if (!tpe->next(tpe)) return false;
+        self->doc = tpe->doc_num(tpe);
+        self->pos = tpe->next_position(tpe);
+    }
+    return true;
+}
+
+static bool tpew_skip_to(TermPosEnumWrapper *self, int doc_num)
+{
+    TermDocEnum *tpe = self->tpe;
+
+    if (tpe->skip_to(tpe, doc_num)) {
+        self->doc = tpe->doc_num(tpe);
+        self->pos = tpe->next_position(tpe);
+        return true;
+    }
+    else {
+        return false;
+    }
+}
+
+static void tpew_destroy(TermPosEnumWrapper *self)
+{
+    self->tpe->close(self->tpe);
+    free(self);
+}
+
+static TermPosEnumWrapper *tpew_new(const char *term, TermDocEnum *tpe)
+{
+    TermPosEnumWrapper *self = ALLOC_AND_ZERO(TermPosEnumWrapper);
+    self->term = term;
+    self->tpe = tpe;
+    self->doc = -1;
+    self->pos = -1;
+    return self;
+}
+#define SpMTEn(span_enum) ((SpanMultiTermEnum *)(span_enum))
+#define SpMTQ(query) ((SpanMultiTermQuery *)(query))
+
+typedef struct SpanMultiTermEnum
+{
+    SpanEnum             super;
+    PriorityQueue       *tpew_pq;
+    TermPosEnumWrapper **tpews;
+    int                  tpew_cnt;
+    int                  pos;
+    int                  doc;
+} SpanMultiTermEnum;
+
+static bool spanmte_next(SpanEnum *self)
+{
+    int curr_doc, curr_pos;
+    TermPosEnumWrapper *tpew;
+    SpanMultiTermEnum *mte = SpMTEn(self);
+    PriorityQueue *tpew_pq = mte->tpew_pq;
+    if (tpew_pq == NULL) {
+        TermPosEnumWrapper **tpews = mte->tpews;
+        int i;
+        tpew_pq = pq_new(mte->tpew_cnt, (lt_ft)tpew_less_than, (free_ft)NULL);
+        for (i = mte->tpew_cnt - 1; i >= 0; i--) {
+            if (tpew_next(tpews[i])) {
+                pq_push(tpew_pq, tpews[i]);
+            }
+        }
+        mte->tpew_pq = tpew_pq;
+    }
+    
+    tpew = (TermPosEnumWrapper *)pq_top(tpew_pq);
+    if (tpew == NULL) {
+        return false;
+    }
+
+    mte->doc = curr_doc = tpew->doc;
+    mte->pos = curr_pos = tpew->pos;
+
+    do {
+        if (tpew_next(tpew)) {
+            pq_down(tpew_pq);
+        }
+        else {
+            pq_pop(tpew_pq);
+        }
+    } while (((tpew = (TermPosEnumWrapper *)pq_top(tpew_pq)) != NULL)
+             && tpew->doc == curr_doc && tpew->pos == curr_pos);
+    return true;
+}
+
+static bool spanmte_skip_to(SpanEnum *self, int target)
+{
+    SpanMultiTermEnum *mte = SpMTEn(self);
+    PriorityQueue *tpew_pq = mte->tpew_pq;
+    TermPosEnumWrapper *tpew;
+    if (tpew_pq == NULL) {
+        TermPosEnumWrapper **tpews = mte->tpews;
+        int i;
+        tpew_pq = pq_new(mte->tpew_cnt, (lt_ft)tpew_less_than, (free_ft)NULL);
+        for (i = mte->tpew_cnt - 1; i >= 0; i--) {
+            tpew_skip_to(tpews[i], target);
+            pq_push(tpew_pq, tpews[i]);
+        }
+        mte->tpew_pq = tpew_pq;
+    }
+    if (tpew_pq->size == 0) {
+        mte->doc = -1;
+        return false;
+    }
+    while ((tpew = (TermPosEnumWrapper *)pq_top(tpew_pq)) != NULL
+           && (target > tpew->doc)) {
+        if (tpew_skip_to(tpew, target)) {
+            pq_down(tpew_pq);
+        }
+        else {
+            pq_pop(tpew_pq);
+        }
+    }
+    return spanmte_next(self);
+}
+
+static int spanmte_doc(SpanEnum *self)
+{
+    return SpMTEn(self)->doc;
+}
+
+static int spanmte_start(SpanEnum *self)
+{
+    return SpMTEn(self)->pos;
+}
+
+static int spanmte_end(SpanEnum *self)
+{
+    return SpMTEn(self)->pos + 1;
+}
+
+static void spanmte_destroy(SpanEnum *self)
+{
+    SpanMultiTermEnum *mte = SpMTEn(self);
+    int i;
+    if (mte->tpew_pq) pq_destroy(mte->tpew_pq);
+    for (i = 0; i < mte->tpew_cnt; i++) {
+        tpew_destroy(mte->tpews[i]);
+    }
+    free(mte->tpews);
+    free(self);
+}
+
+static SpanEnum *spanmte_new(Query *query, IndexReader *ir)
+{
+    char *field = SpQ(query)->field;
+    SpanEnum *self = (SpanEnum *)emalloc(sizeof(SpanMultiTermEnum));
+    SpanMultiTermEnum *smte = SpMTEn(self);
+    SpanMultiTermQuery *smtq = SpMTQ(query);
+    int i;
+
+
+    smte->tpews = ALLOC_N(TermPosEnumWrapper *, smtq->term_cnt);
+    for (i = 0; i < smtq->term_cnt; i++) {
+        char *term = smtq->terms[i];
+        smte->tpews[i] = tpew_new(term, ir_term_positions_for(ir, field, term));
+    }
+    smte->tpew_cnt          = smtq->term_cnt;
+    smte->tpew_pq           = NULL;
+    smte->pos               = -1;
+    smte->doc               = -1;
+
+    self->query             = query;
+    self->next              = &spanmte_next;
+    self->skip_to           = &spanmte_skip_to;
+    self->doc               = &spanmte_doc;
+    self->start             = &spanmte_start;
+    self->end               = &spanmte_end;
+    self->destroy           = &spanmte_destroy;
+    self->to_s              = &spante_to_s;
+
+    return self;
+}
+
 
 /*****************************************************************************
  * SpanFirstEnum
@@ -1385,6 +1587,132 @@ Query *spantq_new(const char *field, const char *term)
     return self;
 }
 
+/*****************************************************************************
+ * SpanMultiTermQuery
+ *****************************************************************************/
+
+static char *spanmtq_to_s(Query *self, const char *field)
+{
+    char *terms = NULL, *p;
+    int len = 2, i;
+    SpanMultiTermQuery *smtq = SpMTQ(self);
+    for (i = 0; i < smtq->term_cnt; i++) {
+        len += strlen(smtq->terms[i]) + 2;
+    }
+    p = terms = ALLOC_N(char, len);
+    *(p++) = '[';
+    for (i = 0; i < smtq->term_cnt; i++) {
+        strcpy(p, smtq->terms[i]);
+        p += strlen(smtq->terms[i]);
+        *(p++) = ',';
+    }
+    if (p > terms) p--;
+    *(p++) = ']';
+    *p = '\0';
+
+    if (field == SpQ(self)->field) {
+        p = strfmt("span_terms(%s)", terms);
+    }
+    else {
+        p = strfmt("span_terms(%s:%s)", SpQ(self)->field, terms);
+    }
+    free(terms);
+    return p;
+}
+
+static void spanmtq_destroy_i(Query *self)
+{
+    SpanMultiTermQuery *smtq = SpMTQ(self);
+    int i;
+    for (i = 0; i < smtq->term_cnt; i++) {
+        free(smtq->terms[i]);
+    }
+    free(smtq->terms);
+    free(SpQ(self)->field);
+    spanq_destroy_i(self);
+}
+
+static void spanmtq_extract_terms(Query *self, HashSet *terms)
+{
+    SpanMultiTermQuery *smtq = SpMTQ(self);
+    int i;
+    for (i = 0; i < smtq->term_cnt; i++) {
+        hs_add(terms, term_new(SpQ(self)->field, smtq->terms[i]));
+    }
+}
+
+static HashSet *spanmtq_get_terms(Query *self)
+{
+    HashSet *terms = hs_new_str(&free);
+    SpanMultiTermQuery *smtq = SpMTQ(self);
+    int i;
+    for (i = 0; i < smtq->term_cnt; i++) {
+        hs_add(terms, estrdup(smtq->terms[i]));
+    }
+    return terms;
+}
+
+static unsigned long spanmtq_hash(Query *self)
+{
+    unsigned long hash = spanq_hash(self);
+    SpanMultiTermQuery *smtq = SpMTQ(self);
+    int i;
+    for (i = 0; i < smtq->term_cnt; i++) {
+        hash ^= str_hash(smtq->terms[i]);
+    }
+    return hash;
+}
+
+static int spanmtq_eq(Query *self, Query *o)
+{
+    SpanMultiTermQuery *smtq = SpMTQ(self);
+    SpanMultiTermQuery *smtqo = SpMTQ(o);
+    int i;
+    if (!spanq_eq(self, o)) return false;
+    if (smtq->term_cnt != smtqo->term_cnt) return false;
+    for (i = 0; i < smtq->term_cnt; i++) {
+        if (strcmp(smtq->terms[i], smtqo->terms[i]) != 0) return false;
+    }
+    return true;;
+}
+
+Query *spanmtq_new_conf(const char *field, int max_terms)
+{
+    Query *self             = q_new(SpanMultiTermQuery);
+
+    SpMTQ(self)->terms      = ALLOC_N(char *, max_terms);
+    SpMTQ(self)->term_cnt   = 0;
+    SpMTQ(self)->term_capa  = max_terms;
+
+    SpQ(self)->field        = estrdup(field);
+    SpQ(self)->get_spans    = &spanmte_new;
+    SpQ(self)->get_terms    = &spanmtq_get_terms;
+
+    self->type              = SPAN_MULTI_TERM_QUERY;
+    self->extract_terms     = &spanmtq_extract_terms;
+    self->to_s              = &spanmtq_to_s;
+    self->hash              = &spanmtq_hash;
+    self->eq                = &spanmtq_eq;
+    self->destroy_i         = &spanmtq_destroy_i;
+    self->create_weight_i   = &spanw_new;
+    self->get_matchv_i      = &spanq_get_matchv_i;
+
+    return self;
+}
+
+Query *spanmtq_new(const char *field)
+{
+    return spanmtq_new_conf(field, SPAN_MULTI_TERM_QUERY_CAPA);
+}
+
+void spanmtq_add_term(Query *self, const char *term)
+{
+    SpanMultiTermQuery *smtq = SpMTQ(self);
+    if (smtq->term_cnt < smtq->term_capa) {
+        smtq->terms[smtq->term_cnt++] = estrdup(term);
+    }
+}
+
 /*****************************************************************************
  *
  * SpanFirstQuery
@@ -1427,6 +1755,7 @@ static Query *spanfq_rewrite(Query *self, IndexReader *ir)
 static void spanfq_destroy_i(Query *self)
 {
     q_deref(SpFQ(self)->match);
+    free(SpQ(self)->field);
     spanq_destroy_i(self);
 }
 
@@ -1451,7 +1780,7 @@ Query *spanfq_new_nr(Query *match, int end)
     SpFQ(self)->match       = match;
     SpFQ(self)->end         = end;
 
-    SpQ(self)->field        = SpQ(match)->field;
+    SpQ(self)->field        = estrdup(SpQ(match)->field);
     SpQ(self)->get_spans    = &spanfe_new;
     SpQ(self)->get_terms    = &spanfq_get_terms;
 
@@ -1569,6 +1898,7 @@ static void spanoq_destroy_i(Query *self)
         q_deref(clause);
     }
     free(soq->clauses);
+    free(SpQ(self)->field);
 
     spanq_destroy_i(self);
 }
@@ -1612,7 +1942,7 @@ Query *spanoq_new()
     SpOQ(self)->clauses     = ALLOC_N(Query *, CLAUSE_INIT_CAPA);
     SpOQ(self)->c_capa      = CLAUSE_INIT_CAPA;
 
-    SpQ(self)->field        = (char *)EMPTY_STRING;
+    SpQ(self)->field        = estrdup((char *)EMPTY_STRING);
     SpQ(self)->get_spans    = &spanoq_get_spans;
     SpQ(self)->get_terms    = &spanoq_get_terms;
 
@@ -1637,7 +1967,8 @@ Query *spanoq_add_clause_nr(Query *self, Query *clause)
               "SpanQuery.", q_get_query_name(clause->type));
     }
     if (curr_index == 0) {
-        SpQ(self)->field = SpQ(clause)->field;
+        free(SpQ(self)->field);
+        SpQ(self)->field = estrdup(SpQ(clause)->field);
     }
     else if (strcmp(SpQ(self)->field, SpQ(clause)->field) != 0) {
         RAISE(ARG_ERROR, "All clauses in a SpanQuery must have the same field. "
@@ -1752,6 +2083,7 @@ static void spannq_destroy(Query *self)
         q_deref(clause);
     }
     free(snq->clauses);
+    free(SpQ(self)->field);
 
     spanq_destroy_i(self);
 }
@@ -1804,7 +2136,7 @@ Query *spannq_new(int slop, bool in_order)
 
     SpQ(self)->get_spans    = &spannq_get_spans;
     SpQ(self)->get_terms    = &spannq_get_terms;
-    SpQ(self)->field        = (char *)EMPTY_STRING;
+    SpQ(self)->field        = estrdup((char *)EMPTY_STRING);
 
     self->type              = SPAN_NEAR_QUERY;
     self->rewrite           = &spannq_rewrite;
@@ -1827,7 +2159,8 @@ Query *spannq_add_clause_nr(Query *self, Query *clause)
               "SpanQuery.", q_get_query_name(clause->type));
     }
     if (curr_index == 0) {
-        SpQ(self)->field = SpQ(clause)->field;
+        free(SpQ(self)->field);
+        SpQ(self)->field = estrdup(SpQ(clause)->field);
     }
     else if (strcmp(SpQ(self)->field, SpQ(clause)->field) != 0) {
         RAISE(ARG_ERROR, "All clauses in a SpanQuery must have the same field. "
@@ -1904,6 +2237,8 @@ static void spanxq_destroy(Query *self)
     q_deref(sxq->inc);
     q_deref(sxq->exc);
 
+    free(SpQ(self)->field);
+
     spanq_destroy_i(self);
 }
 
@@ -1937,7 +2272,7 @@ Query *spanxq_new_nr(Query *inc, Query *exc)
     SpXQ(self)->inc         = inc;
     SpXQ(self)->exc         = exc;
 
-    SpQ(self)->field        = SpQ(inc)->field;
+    SpQ(self)->field        = estrdup(SpQ(inc)->field);
     SpQ(self)->get_spans    = &spanxe_new;
     SpQ(self)->get_terms    = &spanxq_get_terms;
 
@@ -1961,3 +2296,106 @@ Query *spanxq_new(Query *inc, Query *exc)
     return spanxq_new_nr(inc, exc);
 }
 
+
+/*****************************************************************************
+ *
+ * Rewritables
+ *
+ *****************************************************************************/
+
+/*****************************************************************************
+ *
+ * SpanPrefixQuery
+ *
+ *****************************************************************************/
+
+#define SpPfxQ(query) ((SpanPrefixQuery *)(query))
+
+static char *spanprq_to_s(Query *self, const char *current_field) 
+{
+    char *buffer, *bptr;
+    const char *prefix = SpPfxQ(self)->prefix;
+    const char *field = SpQ(self)->field;
+    size_t plen = strlen(prefix);
+    size_t flen = strlen(field);
+
+    bptr = buffer = ALLOC_N(char, plen + flen + 35);
+
+    if (strcmp(field, current_field) != 0) {
+        sprintf(bptr, "%s:", field);
+        bptr += flen + 1;
+    }
+
+    sprintf(bptr, "%s*", prefix);
+    bptr += plen + 1;
+    if (self->boost != 1.0) {
+        *bptr = '^';
+        dbl_to_s(++bptr, self->boost);
+    }
+
+    return buffer;
+}
+
+static Query *spanprq_rewrite(Query *self, IndexReader *ir)
+{
+    const char *field = SpQ(self)->field;
+    const int field_num = fis_get_field_num(ir->fis, field);
+    Query *volatile q = spanmtq_new_conf(field, SPAN_PREFIX_QUERY_MAX_TERMS);
+    q->boost = self->boost;        /* set the boost */
+
+    if (field_num >= 0) {
+        const char *prefix = SpPfxQ(self)->prefix;
+        TermEnum *te = ir->terms_from(ir, field_num, prefix);
+        const char *term = te->curr_term;
+        size_t prefix_len = strlen(prefix);
+
+        TRY
+            do { 
+                if (strncmp(term, prefix, prefix_len) != 0) {
+                    break;
+                }
+                spanmtq_add_term(q, term);       /* found a match */
+            } while (te->next(te));
+        XFINALLY
+            te->close(te);
+        XENDTRY
+    }
+
+    return q;
+}
+
+static void spanprq_destroy(Query *self)
+{
+    free(SpQ(self)->field);
+    free(SpPfxQ(self)->prefix);
+    spanq_destroy_i(self);
+}
+
+static unsigned long spanprq_hash(Query *self)
+{
+    return str_hash(SpQ(self)->field) ^ str_hash(SpPfxQ(self)->prefix);
+}
+
+static int spanprq_eq(Query *self, Query *o)
+{
+    return (strcmp(SpPfxQ(self)->prefix, SpPfxQ(o)->prefix) == 0) 
+        && (strcmp(SpQ(self)->field,  SpQ(o)->field) == 0);
+}
+
+Query *spanprq_new(const char *field, const char *prefix)
+{
+    Query *self = q_new(SpanPrefixQuery);
+
+    SpQ(self)->field        = estrdup(field);
+    SpPfxQ(self)->prefix    = estrdup(prefix);
+
+    self->type              = SPAN_PREFIX_QUERY;
+    self->rewrite           = &spanprq_rewrite;
+    self->to_s              = &spanprq_to_s;
+    self->hash              = &spanprq_hash;
+    self->eq                = &spanprq_eq;
+    self->destroy_i         = &spanprq_destroy;
+    self->create_weight_i   = &q_create_weight_unsup;
+
+    return self;
+}