ferret 0.9.6 → 0.10.0

data/ext/r_qparser.c

@@ -4,14 +4,16 @@
 static VALUE cQueryParser;
 VALUE cQueryParseException;
 
-VALUE rhandle_parse_errors_key;
-VALUE rallow_any_fields_key;
-VALUE rwild_lower_key;
-VALUE roccur_default_key;
-VALUE rdefault_slop_key;
-VALUE rclean_str_key;
-VALUE rfields_key;
-extern VALUE ranalyzer_key;
+extern VALUE sym_analyzer;
+static VALUE sym_wild_card_downcase;
+static VALUE sym_all_fields;
+static VALUE sym_default_field;
+static VALUE sym_validate_fields;
+static VALUE sym_or_default;
+static VALUE sym_default_slop;
+static VALUE sym_handle_parse_errors;
+static VALUE sym_clean_string;
+static VALUE sym_max_clauses;
 
 extern VALUE frt_get_analyzer(Analyzer *a);
 extern VALUE frt_get_q(Query *q);
@@ -26,163 +28,231 @@ extern Analyzer *frt_get_cwrapped_analyzer(VALUE ranalyzer);
 static void
 frt_qp_free(void *p)
 {
-  QParser *qp = (QParser *)p;
-  object_del(p);
-  qp_destroy(qp);
+    object_del(p);
+    qp_destroy((QParser *)p);
 }
 
 static void
 frt_qp_mark(void *p)
 {
-  QParser *qp = (QParser *)p;
-  frt_gc_mark(qp->analyzer);
+    frt_gc_mark(((QParser *)p)->analyzer);
 }
 
-HashSet *
+static HashSet *
 frt_get_fields(VALUE rfields)
 {
-  VALUE rval;
-  HashSet *fields = hs_str_create(&free);
-  char *s, *p, *str;
+    VALUE rval;
+    HashSet *fields = hs_new_str(&free);
+    char *s, *p, *str;
 
-  if (TYPE(rfields) == T_ARRAY) {
-    int i;
-    for (i = 0; i < RARRAY(rfields)->len; i++) {
-      rval = rb_obj_as_string(RARRAY(rfields)->ptr[i]);
-      hs_add(fields, estrdup(RSTRING(rval)->ptr));
-    }
-  } else {
-    rval = rb_obj_as_string(rfields);
-    if (strcmp("*", RSTRING(rval)->ptr) == 0) {
-      hs_destroy(fields);
-      fields = NULL;
+    if (TYPE(rfields) == T_ARRAY) {
+        int i;
+        for (i = 0; i < RARRAY(rfields)->len; i++) {
+            rval = rb_obj_as_string(RARRAY(rfields)->ptr[i]);
+            hs_add(fields, estrdup(RSTRING(rval)->ptr));
+        }
     } else {
-      s = str = estrdup(RSTRING(rval)->ptr);
-      while ((p = strchr(s, '|')) != '\0') {
-        *p = '\0';
-        hs_add(fields, estrdup(s));
-        s = p + 1;
-      }
-      hs_add(fields, estrdup(s));
-      free(str);
+        rval = rb_obj_as_string(rfields);
+        if (strcmp("*", RSTRING(rval)->ptr) == 0) {
+            hs_destroy(fields);
+            fields = NULL;
+        } else {
+            s = str = estrdup(RSTRING(rval)->ptr);
+            while ((p = strchr(s, '|')) != '\0') {
+                *p = '\0';
+                hs_add(fields, estrdup(s));
+                s = p + 1;
+            }
+            hs_add(fields, estrdup(s));
+            free(str);
+        }
     }
-  }
-  return fields;
+    return fields;
 }
 
+/* 
+ *  call-seq:
+ *     QueryParser.new(options = {}) -> QueryParser
+ *
+ *  Create a new QueryParser. The QueryParser is used to convert string
+ *  queries into Query objects. The options are;
+ *
+ *  === Options
+ *
+ *  :default_field::        Default: "*" (all fields). The default field to
+ *                          search when no field is specified in the search
+ *                          string. It can also be an array of fields.
+ *  :analyzer::             Default: StandardAnalyzer. Analyzer used by the
+ *                          query parser to parse query terms
+ *  :wild_card_downcase::   Default: true. Specifies whether wild-card queries
+ *                          should be downcased or not since they are not
+ *                          passed through the parser
+ *  :all_fields::           Default: []. Lets the query parser know what
+ *                          fields are available for searching, particularly
+ *                          when the "*" is specified as the search field
+ *  :validate_fields::      Default: false. Set to true if you want an
+ *                          exception to be raised if there is an attempt to
+ *                          search a non-existent field
+ *  :or_default::           Default: true. Use "OR" as the default boolean
+ *                          operator
+ *  :default_slop::         Default: 0. Default slop to use in PhraseQuery
+ *  :handle_parser_errors:: Default: true. QueryParser will quietly handle all
+ *                          parsing errors internally. If you'd like to handle
+ *                          them yourself, set this parameter to false.
+ *  :clean_string::         Default: true. QueryParser will do a quick
+ *                          once-over the query string make sure that quotes
+ *                          and brackets match up and special characters are
+ *                          escaped
+ *  :max_clauses::          Default: 512. the maximum number of clauses
+ *                          allowed in boolean queries and the maximum number
+ *                          of terms allowed in multi, prefix, wild-card or
+ *                          fuzzy queries when those queries are generated by
+ *                          rewriting other queries
+ */                   
 static VALUE
 frt_qp_init(int argc, VALUE *argv, VALUE self)
 {
-  VALUE rdef_field, roptions;
-  VALUE rval;
-  Analyzer *analyzer = NULL;
-
-  HashSet *all_fields = hs_str_create(&free);
-  HashSet *def_fields = NULL;
-  QParser *qp;
-
-  rb_scan_args(argc, argv, "02", &rdef_field, &roptions);
-  /* process default_field */
-  if (argc > 0) {
-    def_fields = frt_get_fields(rdef_field);
-  }
-
-  if (argc == 2) {
-    if (Qnil != (rval = rb_hash_aref(roptions, ranalyzer_key))) {
-      analyzer = frt_get_cwrapped_analyzer(rval);
-    }
-    if (Qnil != (rval = rb_hash_aref(roptions, rfields_key))) {
-      all_fields = frt_get_fields(rval);
-    }
-  }
-  if (all_fields == NULL) {
-    all_fields = hs_str_create(&free);
-  }
-
-  if (!analyzer) {
-    analyzer = mb_standard_analyzer_create(true);
-  }
-
-  qp = qp_create(all_fields, def_fields, analyzer);
-  qp->allow_any_fields = true;
-  qp->clean_str = true;
-  /* handle options */
-  if (argc == 2) {
-    if (Qnil != (rval = rb_hash_aref(roptions, rhandle_parse_errors_key))) {
-      qp->handle_parse_errors = RTEST(rval);
-    }
-    if (Qnil != (rval = rb_hash_aref(roptions, rallow_any_fields_key))) {
-      qp->allow_any_fields = RTEST(rval);
-    }
-    if (Qnil != (rval = rb_hash_aref(roptions, rwild_lower_key))) {
-      qp->wild_lower = RTEST(rval);
+    VALUE roptions;
+    VALUE rval;
+    Analyzer *analyzer = NULL;
+    bool has_options = false;
+
+    HashSet *all_fields = NULL;
+    HashSet *def_fields = NULL;
+    QParser *qp;
+
+    if (rb_scan_args(argc, argv, "01", &roptions) > 0) {
+        if (TYPE(roptions) == T_HASH) {
+            has_options = true;
+            if (Qnil != (rval = rb_hash_aref(roptions, sym_default_field))) {
+                def_fields = frt_get_fields(rval);
+            }
+            if (Qnil != (rval = rb_hash_aref(roptions, sym_analyzer))) {
+                analyzer = frt_get_cwrapped_analyzer(rval);
+            }
+            if (Qnil != (rval = rb_hash_aref(roptions, sym_all_fields))) {
+                all_fields = frt_get_fields(rval);
+            }
+        } else {
+            def_fields = frt_get_fields(roptions);
+        }
     }
-    if (Qnil != (rval = rb_hash_aref(roptions, roccur_default_key))) {
-      qp->or_default = (FIX2INT(rval) == BC_MUST) ? false : true;
+    if (all_fields == NULL) {
+        all_fields = hs_new_str(&free);
     }
-    if (Qnil != (rval = rb_hash_aref(roptions, rdefault_slop_key))) {
-      qp->def_slop = FIX2INT(rval);
+
+    if (!analyzer) {
+        analyzer = mb_standard_analyzer_new(true);
     }
-    if (Qnil != (rval = rb_hash_aref(roptions, rclean_str_key))) {
-      qp->clean_str = RTEST(rval);
+
+    qp = qp_new(all_fields, def_fields, analyzer);
+    qp->allow_any_fields = true;
+    qp->clean_str = true;
+    /* handle options */
+    if (argc > 0) {
+        if (Qnil != (rval = rb_hash_aref(roptions, sym_handle_parse_errors))) {
+            qp->handle_parse_errors = RTEST(rval);
+        }
+        if (Qnil != (rval = rb_hash_aref(roptions, sym_validate_fields))) {
+            qp->allow_any_fields = !RTEST(rval);
+        }
+        if (Qnil != (rval = rb_hash_aref(roptions, sym_wild_card_downcase))) {
+            qp->wild_lower = RTEST(rval);
+        }
+        if (Qnil != (rval = rb_hash_aref(roptions, sym_or_default))) {
+            qp->or_default = (FIX2INT(rval) == BC_MUST) ? false : true;
+        }
+        if (Qnil != (rval = rb_hash_aref(roptions, sym_default_slop))) {
+            qp->def_slop = FIX2INT(rval);
+        }
+        if (Qnil != (rval = rb_hash_aref(roptions, sym_clean_string))) {
+            qp->clean_str = RTEST(rval);
+        }
+        if (Qnil != (rval = rb_hash_aref(roptions, sym_max_clauses))) {
+            qp->max_clauses = FIX2INT(rval);
+        }
     }
-  }
-  Frt_Wrap_Struct(self, frt_qp_mark, frt_qp_free, qp);
-  object_add(qp, self);
-  return self;
+    Frt_Wrap_Struct(self, frt_qp_mark, frt_qp_free, qp);
+    object_add(qp, self);
+    return self;
 }
 
 #define GET_QP QParser *qp = (QParser *)DATA_PTR(self)
+/*
+ *  call-seq:
+ *     query_parser.parse(query_string) -> Query
+ *
+ *  Parse a query string returning a Query object if parsing was successful.
+ *  Will raise a QueryParseException if unsuccessful. 
+ */
 static VALUE
 frt_qp_parse(VALUE self, VALUE rstr)
 {
-  char *msg = NULL;
-  volatile VALUE rq;
-  GET_QP;
-  rstr = rb_obj_as_string(rstr);
-  TRY
-    rq = frt_get_q(qp_parse(qp, RSTRING(rstr)->ptr));
-    break;
-  default:
-    msg = xcontext.msg;
-    HANDLED();
-  XENDTRY
-
-  if (msg) {
-    rb_raise(cQueryParseException, msg);
-  }
-  
-  return rq;
+    const char *msg = NULL;
+    volatile VALUE rq;
+    GET_QP;
+    rstr = rb_obj_as_string(rstr);
+    TRY
+        rq = frt_get_q(qp_parse(qp, RSTRING(rstr)->ptr));
+        break;
+    default:
+        msg = xcontext.msg;
+        HANDLED();
+    XENDTRY
+
+    if (msg) {
+        rb_raise(cQueryParseException, msg);
+    }
+
+    return rq;
 }
 
+/*
+ *  call-seq:
+ *     query_parser.fields -> Array of Symbols
+ *
+ *  Returns the list of all fields that the QueryParser knows about.
+ */
 static VALUE
 frt_qp_get_fields(VALUE self)
 {
-  GET_QP;
-  int i;
-  HashSet *fields = qp->all_fields;
-  VALUE rfields = rb_ary_new();
+    GET_QP;
+    int i;
+    HashSet *fields = qp->all_fields;
+    VALUE rfields = rb_ary_new();
 
-  for (i = 0; i < fields->size; i++) {
-    rb_ary_push(rfields, rb_str_new2((char *)fields->elems[i]));
-  } 
+    for (i = 0; i < fields->size; i++) {
+        rb_ary_push(rfields, ID2SYM(rb_intern((char *)fields->elems[i])));
+    } 
 
-  return rfields;
+    return rfields;
 }
 
+/*
+ *  call-seq:
+ *     query_parser.fields = fields -> self
+ *
+ *  Set the list of fields. These fields are expanded for searches on "*".
+ */
 static VALUE
 frt_qp_set_fields(VALUE self, VALUE rfields)
 {
-  GET_QP;
-  HashSet *fields = frt_get_fields(rfields);
+    GET_QP;
+    HashSet *fields = frt_get_fields(rfields);
 
-  if (fields == NULL) {
-    fields = hs_str_create(&free);
-  }
-  hs_destroy(qp->all_fields);
-  qp->all_fields = fields;
+    if (qp->def_fields == qp->all_fields) {
+        qp->def_fields = NULL;
+    }
+    if (fields == NULL) {
+        fields = hs_new_str(&free);
+    }
+    hs_destroy(qp->all_fields);
+    qp->all_fields = fields;
+    if (qp->def_fields == NULL) {
+        qp->def_fields = fields;
+    }
 
-  return self;
+    return self;
 }
 
 /****************************************************************************
@@ -191,28 +261,245 @@ frt_qp_set_fields(VALUE self, VALUE rfields)
  *
  ****************************************************************************/
 
+/* rdoc hack
+extern VALUE mFerret = rb_define_module("Ferret");
+extern VALUE cQueryParser = rb_define_module_under(mFerret, "QueryParser");
+*/
+
+/*
+ *  Document-class: Ferret::QueryParser::QueryParseException
+ *
+ *  == Summary
+ *
+ *  Exception raised when there is an error parsing the query string passed to
+ *  QueryParser.
+ */
+void
+Init_QueryParseException(void)
+{
+    cQueryParseException = rb_define_class_under(cQueryParser,
+                                                 "QueryParseException",
+                                                 rb_eStandardError);
+}
+
+/*
+ *  Document-class: Ferret::QueryParser
+ *
+ *  == Summary
+ *
+ *  The QueryParser is used to transform user submitted query strings into
+ *  QueryObjects. Ferret using its own Query Language known from now on as
+ *  Ferret Query Language or FQL.
+ *
+ *  == Ferret Query Language
+ * 
+ *  === Preamble
+ * 
+ *  The following characters are special characters in FQL;
+ * 
+ *    :, (, ), [, ], {, }, !, +, ", ~, ^, -, |, <, >, =, *, ?, \
+ * 
+ *  If you want to use one of these characters in one of your terms you need
+ *  to escape it with a \ character. \ escapes itself. The exception to this
+ *  rule is within Phrases which a strings surrounded by double quotes (and
+ *  will be explained further bellow in the section on PhraseQueries). In
+ *  Phrases, only ", | and <> have special meaning and need to be escaped if
+ *  you want the literal value. <> is escaped \<\>.
+ * 
+ *  In the following examples I have only written the query string. This would
+ *  be parse like;
+ * 
+ *    query = query_parser.parse("pet:(dog AND cat)")
+ *    puts query    # => "+pet:dog +pet:cat"
+ * 
+ *  === TermQuery
+ * 
+ *  A term query is the most basic query of all and is what most of the other
+ *  queries are built upon. The term consists of a single word. eg;
+ * 
+ *    'term'
+ * 
+ *  Note that the analyzer will be run on the term and if it splits the term
+ *  in two then it will be turned into a phrase query. For example, with the
+ *  plain Ferret::Analysis::Analyzer, the following;
+ * 
+ *    'dave12balmain'
+ * 
+ *  is equivalent to;
+ * 
+ *    '"dave balmain"'
+ * 
+ *  Which we will explain now...
+ * 
+ *  === PhraseQuery
+ *  
+ *  A phrase query is a string of terms surrounded by double quotes. For
+ *  example you could write;
+ * 
+ *    '"quick brown fox"'
+ * 
+ *  But if a "fast" fox is just as good as a quick one you could use the |
+ *  character to specify alternate terms.
+ * 
+ *    '"quick|speedy|fast brown fox"'
+ * 
+ *  What if we don't care what colour the fox is. We can use the <> to specify
+ *  a place setter. eg;
+ * 
+ *    '"quick|speedy|fast <> fox"' 
+ * 
+ *  This will match any word in between quick and fox. Alternatively we could
+ *  set the "slop" for the phrase which allows a certain variation in the
+ *  match of the phrase. The slop for a phrase is an integer indicating how
+ *  many positions you are allowed to move the terms to get a match. Read more
+ *  about the slop factor in Ferret::Search::PhraseQuery. To set the slop
+ *  factor for a phrase you can type;
+ * 
+ *    '"big house"~2' 
+ * 
+ *  This would match "big house", "big red house", "big red brick house" and
+ *  even "house big". That's right, you don't need to have th terms in order
+ *  if you allow some slop in your phrases. (See Ferret::Search::Spans if you
+ *  need a phrase type query with ordered terms.)
+ *  
+ *  These basic queries will be run on the default field which is set when you
+ *  create the query_parser. But what if you want to search a different field.
+ *  You'll be needing a ...
+ * 
+ *  === FieldQuery
+ * 
+ *  A field query is any field prefixed by <fieldname>:. For example, to
+ *  search for all instances of the term "ski" in field "sport", you'd write;
+ * 
+ *    'sport:ski'
+ *  Or we can apply a field to phrase;
+ * 
+ *    'sport:"skiing is fun"'
+ * 
+ *  Now we have a few types of queries, we'll be needing to glue them together
+ *  with a ...
+ * 
+ *  === BooleanQuery
+ * 
+ *  There are a couple of ways of writing boolean queries. Firstly you can
+ *  specify which terms are required, optional or required not to exist (not).
+ *  
+ *  * '+' or "REQ" can be used to indicate a required query. "REQ" must be
+ *    surrounded by white space.
+ *  * '-', '!' or "NOT" are used to indicate query that is required to be
+ *    false. "NOT" must be surrounded by white space.
+ *  * all other queries are optional if the above symbols are used.
+ * 
+ *  Some examples;
+ * 
+ *    '+sport:ski -sport:snowboard sport:toboggan'
+ *    '+ingredient:chocolate +ingredient:strawberries -ingredient:wheat'
+ * 
+ *  You may also use the boolean operators "AND", "&&", "OR" and "||". eg;
+ * 
+ *    'sport:ski AND NOT sport:snowboard OR sport:toboggan'
+ *    'ingredient:chocolate AND ingredient:strawberries AND NOT ingredient:wheat'
+ * 
+ *  You can set the default operator when you create the query parse.
+ *  
+ *  === RangeQuery
+ * 
+ *  A range query finds all documents with terms between the two query terms.
+ *  This can be very useful in particular for dates. eg;
+ * 
+ *    'date:[20050725 20050905]' # all dates >= 20050725 and <= 20050905
+ *    'date:[20050725 20050905}' # all dates >= 20050725 and <  20050905
+ *    'date:{20050725 20050905]' # all dates >  20050725 and <= 20050905
+ *    'date:{20050725 20050905}' # all dates >  20050725 and <  20050905
+ * 
+ *  You can also do open ended queries like this;
+ * 
+ *    'date:[20050725>' # all dates >= 20050725
+ *    'date:{20050725>' # all dates >  20050725
+ *    'date:<20050905]' # all dates <= 20050905
+ *    'date:<20050905}' # all dates <  20050905
+ * 
+ *  Or like this;
+ * 
+ *    'date: >= 20050725'
+ *    'date: >  20050725'
+ *    'date: <= 20050905'
+ *    'date: <  20050905'
+ * 
+ *  If you prefer the above style you could use a boolean query but like this;
+ * 
+ *    'date:( >= 20050725 AND <= 20050905)'
+ * 
+ *  But rangequery only solution shown first will be faster.
+ *  
+ *  === WildQuery
+ * 
+ *  A wild query is a query using the pattern matching characters * and ?. *
+ *  matchs 0 or more characters while ? matchs a single character. This type
+ *  of query can be really useful for matching heirarchical categories for
+ *  example. Let's say we had this structure;
+ * 
+ *    /sport/skiing
+ *    /sport/cycling
+ *    /coding1/ruby
+ *    /coding1/c
+ *    /coding2/python
+ *    /coding2/perl
+ *  
+ *  If you wanted all categories with programming languages you could use the
+ *  query;
+ * 
+ *    'category:/coding?/?*'
+ * 
+ *  Note that this query can be quite expensive if not used carefully. In the
+ *  example above there would be no problem but you should be careful not use
+ *  the wild characters at the beginning of the query as it'll have to iterate
+ *  through every term in that field. Having said that, some fields like the
+ *  category field above will only have a small number of distinct fields so
+ *  this could be ok.
+ * 
+ *  === FuzzyQuery
+ * 
+ *  This is like the sloppy phrase query above, except you are now adding slop
+ *  to a term. Basically it measures the Levenshtein distance between two
+ *  terms and if the value is below the slop threshold the term is a match.
+ *  This time though the slop must be a float between 0 and 1.0, 1.0 being a
+ *  perfect match and 0 being far from a match. The default is set to 0.5 so
+ *  you don't need to give a slop value if you don't want to. You can set the
+ *  default in the Ferret::Search::FuzzyQuery class. Here are a couple of
+ *  examples;
+ * 
+ *    'content:ferret~'
+ *    'content:Ostralya~0.4'
+ * 
+ *  Note that this query can be quite expensive. If you'd like to use this
+ *  query, you may want to set a mininum prefix length in the FuzzyQuery
+ *  class. This can substantially reduce the number of terms that the query
+ *  will iterate over.
+ *  
+ */
 void
-Init_qparser(void)
+Init_QueryParser(void)
 {
-  /* hash keys */
-  rhandle_parse_errors_key = ID2SYM(rb_intern("handle_parse_errors"));
-  rallow_any_fields_key = ID2SYM(rb_intern("allow_any_fields"));
-  rwild_lower_key = ID2SYM(rb_intern("wild_lower"));
-  roccur_default_key = ID2SYM(rb_intern("occur_default"));
-  rdefault_slop_key = ID2SYM(rb_intern("default_slop"));
-  rclean_str_key = ID2SYM(rb_intern("clean_string"));
-  rfields_key = ID2SYM(rb_intern("fields"));
-
-  /* QueryParser */
-  cQueryParser = rb_define_class_under(mFerret, "QueryParser", rb_cObject);
-  rb_define_alloc_func(cQueryParser, frt_data_alloc);
-
-  rb_define_method(cQueryParser, "initialize", frt_qp_init, -1);
-  rb_define_method(cQueryParser, "parse", frt_qp_parse, 1);
-  rb_define_method(cQueryParser, "fields", frt_qp_get_fields, 0);
-  rb_define_method(cQueryParser, "fields=", frt_qp_set_fields, 1);
-
-  /* QueryParseException */
-  cQueryParseException = rb_define_class_under(cQueryParser,
-      "QueryParseException", rb_eStandardError);
+    /* hash keys */
+    sym_wild_card_downcase = ID2SYM(rb_intern("wild_card_downcase"));
+    sym_all_fields = ID2SYM(rb_intern("fields"));
+    sym_default_field = ID2SYM(rb_intern("default_field"));
+    sym_validate_fields = ID2SYM(rb_intern("validate_fields"));
+    sym_or_default = ID2SYM(rb_intern("or_default"));
+    sym_default_slop = ID2SYM(rb_intern("default_slop"));
+    sym_handle_parse_errors = ID2SYM(rb_intern("handle_parse_errors"));
+    sym_clean_string = ID2SYM(rb_intern("clean_string"));
+    sym_max_clauses = ID2SYM(rb_intern("max_clauses"));
+
+    /* QueryParser */
+    cQueryParser = rb_define_class_under(mFerret, "QueryParser", rb_cObject);
+    rb_define_alloc_func(cQueryParser, frt_data_alloc);
+
+    rb_define_method(cQueryParser, "initialize", frt_qp_init, -1);
+    rb_define_method(cQueryParser, "parse", frt_qp_parse, 1);
+    rb_define_method(cQueryParser, "fields", frt_qp_get_fields, 0);
+    rb_define_method(cQueryParser, "fields=", frt_qp_set_fields, 1);
+
+    Init_QueryParseException();
 }