blacklight_advanced_search 1.0.0pre1

data/lib/generators/blacklight_advanced_search/templates/public/stylesheets/blacklight_advanced_search_styles.css

@@ -0,0 +1,129 @@
+
+/* Kind of wacky stuff to make scrolling on limit column work right. */
+
+.input_columns {
+  position: relative;
+}
+
+.limit_column {
+  position: absolute;
+  top: 0;
+  bottom: 0;
+  right: 0;
+  width: 49.1%;
+  overflow-y: hidden; 
+}
+
+.limit_input {
+  position: absolute;
+  top: 6em;
+  bottom: 0;
+  right: 0;
+  left: 0;
+  overflow-y: auto;
+}
+
+/* Random styles */
+
+.advanced_search_field label {
+  display:block; 
+}
+
+.advanced_search_field input {
+  margin-bottom: 0.666em;
+  width: 80%;
+}
+
+form.advanced label {
+  font-weight:normal;
+}
+
+form.advanced h2 {
+  font-weight: normal;
+  background-color: #EEEEEE;
+  height: 3em;
+}
+
+form.advanced .limit_column ul {
+  margin: 1em; 
+}
+
+form.advanced .limit_column li {
+  list-style: none;  
+  padding: 0.1em 0.4em;
+  font-size: 80%;
+}
+
+form.advanced .facet_item h3 {
+  cursor: pointer;
+}
+  
+form.advanced .adv_facet_selections {
+  color:green;
+  font-size: 80%;
+  display: block;
+  margin-top: 0.25em;
+}
+
+form.advanced .advanced_button {
+    -moz-border-radius: 4px 4px 4px 4px;
+    -webkit-border-radius: 4px 4px 4px 4px;
+    border-radius: 4px 4px 4px 4px;
+    background-color: #F6F6F6;
+    border: 1px solid #CCCCCC;
+    color: #2E4F81;
+    display: inline-block;
+    float: right;
+    margin-right: 1em;
+    padding: 0.4em 1em;
+    text-decoration: none;
+}
+
+form.advanced .reset {  
+}
+
+
+
+.advanced_help li {
+  margin-top: 0.5em;
+  margin-bottom: 0.5em;
+}
+
+form.advanced .sort_submit_buttons {
+ background-color: #EEEEEE;
+ padding: 1em; 
+ margin-top: 1em;
+ overflow: hidden; /* trick into containing floats please */
+}
+
+form.advanced .constraints {
+  padding: 1em;
+  margin-top: 1em;
+  background-color: #E2EDFE;
+  border: 1px solid #C4DAFE;
+}
+
+form.advanced .constraints h4 {
+  margin-bottom: 0.66em; 
+}
+
+form.advanced .constraints .constraint {
+  display:block;
+  padding-left:2em;
+  text-indent:-2em;
+}
+
+form.advanced .constraints .constraint .filterName {
+  font-weight: bold;
+  margin-right: 0.66em;
+}
+
+form.advanced .column > h2 {
+  padding: 0.33em;
+}
+form.advanced .column > div {
+  padding-left: 0.33em;
+  padding-right: 0.33em;
+}
+
+

data/lib/parsing_nesting/Readme.rdoc

@@ -0,0 +1,160 @@
+= The "Parsing Nesting" parser and Solr query transformer 
+ 
+== User-entered queries handled
+
+* simple lists of terms and phrases, possibly with + or -, are translated 
+  directly to dismax queries, respecting whatever mm is operative for the
+  Blacklight search field definition (either as a specified mm param in the
+  search field definition, or in Solr request handler default)
+  * one two three
+  * one +two -"three phrase"
+
+* AND/OR/NOT operators can be used for boolean logic. Parenthesis can
+  be used to be clear about grouping, or to make arbitrarily complex
+  nested logic. These operators always apply to only the immediately
+  adjacent terms, unless parens are used, and "OR" 'binds more tightly'
+  than 'AND'
+  * big OR small AND blue OR green === (big OR small) AND (blue OR green)
+  * one AND two OR three AND four   ===    one AND (two OR three) AND four
+    * alternative, with different meaning:  (one AND two) OR (three AND four)
+  * NOT one two three ===  (NOT one) two three === -one two three
+    * alternative, with different meaning: NOT(one two three)
+  
+* lists of terms can be combined with AND/OR/NOT in a variety of ways
+  * one two three OR four  === one two (three OR four)
+  * (one two three) AND (big small medium)
+  * NOT(one two) three ((four OR -five) AND (blue green red))
+    * Note that some of these latter ones can have confusing semantics
+      if your dismax mm isn't 100%.  
+      
+      For instance (one two three) will be
+      a dismax query, let's say mm=1, then the result set would actually
+      be the equivalent of: 
+        (one OR two OR three). 
+      NOT(one two three) will be an actual complementary NOT, the 
+      complementary/inverted set -- so NOT(one two three) 
+      (if you had dismax mm=1) will essentially 
+      have the same semantics as:
+        NOT(one OR two OR three)
+      which isn't
+      neccesarily what the user is expecting. But if the user always uses
+      explicit boolean connectors, they can exert complete control over
+      the semantics, and not get the 'fuzziness'. Alternately, the local
+      implementer could use only mm=100%, in which case everything is much
+      less fuzzy/hard-to-predict
+
+== Conversion to Solr
+
+As mentioned, a straight list of terms such as (in the most complicated)
+case:  one -two +"three four" >> is translated directly to a dismax
+query for those entered terms. Using the qf/pf/mm/etc you have configured
+for the Blacklight search_field in question. (While by default the advanced
+search plugin uses exactly the same field configurations you already have
+for simple search, you could also choose to pass in different ones for
+advanced search, perhaps setting mm to 100% if desired for adv search)
+
+There are a few motivations for doing things this way:
+
+* To be consistent with simple search, so moving to advanced is less of a 
+  conceptual break for the user. If you take a legal simple search, and
+  enter it in a given field in advanced search, it will work exactly the
+  same as it did in simple (even if mm is not 100% in simple), rather than
+  having entirely different semantics.
+* Taking advantage of that, one might eventually want to actually use this
+  parser in simple search, so user can enter single-field boolean expressions
+  even in simple/basic search.
+* In the future, we might want to provide actual fielded searches in an
+  'expert' mode. +title: foo AND author:bar+ or 
+  +(title:(one two) AND author:(three four)) OR isbn:X+
+  For explicit fielded searching, it is convenient if you can combine
+  dismax searches. 
+
+Once you start putting boolean operators AND, OR, NOT in, the query will
+no longer neccesarily be converted to a _single_ nested dismax query, a single
+user-entered string may be converted to multiple nested queries. In some
+common cases, multiple clauses will still be collapsed into fewer dismax
+queries than the 'naive' translation. Examples:
+
+* one two three (blue AND green AND -purple)
+    _query_:"{!dismax}one two three +four +five -purple"
+* one two three (blue OR green OR purple)
+    _query_:"{!dismax}one two three" AND _query_:"{!dismax mm=1}blue green purple"
+
+However, if you use complicated crazy nesting, you can get a lot of nested
+queries generated:
+* ((one two) AND (three OR four)) OR (blue AND NOT (green OR purple))
+    ( ( _query_:"{!dismax }one two" AND _query_:"{!dismax mm=1}three four" ) OR ( _query_:"{!dismax }blue" AND NOT _query_:"{!dismax mm=1}green purple" ) )
+
+= Note on pure negative queries
+
+In Solr 1.4.1, the dismax query parser can't handle queries with only "-"
+excluded terms. And while the lucene query parser can handle certain types
+of pure negative queries, it can't properly handle a NOT(x) as one of the
+operands of the "OR".  Our query generation strategy notices these cases
+and transforms to semantically equivalent query that can be handled by
+Solr properly. At least it tries, this is the least clean part of the code.
+But there are specs showing it works for some fairly complicated queries. 
+
+* -one -two  =>is transformed to=>  NOT _query_:"{!dismax mm=1}one two"
+* $x OR NOT $y =>is transformed to=> $x OR (*:* AND NOT $y)
+
+This works with very complicated queries when the bad pure negative part
+would be just a sub-clause or sub-query. Sometimes the result is not
+the most concise query possible, but it should hold to it's semantics. 
+
+* -red -blue (-foo OR -bar) (big OR NOT small)
+    turns into ==>
+    NOT _query_:"{!dismax mm=1}red blue" AND NOT _query_:"{!dismax mm=100%}foo bar" AND ( _query_:\"{!dismax }big" OR (*:* AND NOT _query_:"{!dismax }small") )
+
+== Why not use e-dismax?
+
+That would be a potentially reasonable choice. Why didn't I? 
+
+One, at the time of this writing, edismax is not available in a tagged stable
+Solr release, and I write code for Blacklight that works with tagged stable
+releases. 
+
+Two, edismax doesn't neccesarily entirely support the semantics I want,
+especially for features I would like to add in the future. I am not sure
+exactly what edismax does with complicated deeply nested expressions. 
+For fielded searches, dismax supports actual individual solr fields, but not
+the "fields" as dismax qf aggregates that we need. These things could
+be added to dismax, but with my lack of Java chops and familiarity with
+Solr code, it would have taken me much longer to do (and been much less
+enjoyable). 
+
+I think it may be a reasonable choice to seperate concerns between Solr
+and the app layer like this, let Solr handle basic search expressions,
+but let the app layer handle more complicated query parsing, translating
+to those simple expressions. 
+
+On the other hand, there are definite downsides to this approach. Including
+having to deal with idiosyncracies of built-in query parsers ("pure
+negative" behavior), depend upon other idiosyncracies (dismax does not
+apply mm to -excluded terms), etc. And not being able to share the code
+at the Solr/Java level. 
+
+In the future, a different approach that might be best of all could be
+using the not-yet-finished XML query parser, to do initial parsing in
+ruby at the app level, but translate to specified lucene primitives using
+XML query parser, instead of having to translate to lucene/dismax query
+parsers. 
+
+== Future Enhancement Ideas
+Just ideas. 
+ 
+1. Allow expert "fielded" searches. title:foo
+   which would correspond not to actual solr index field "title", but
+   to a Blacklight-configured "search field" qf/pf. 
+2. Insert this app-level parser even in "simple" search, so users
+   can use boolean operators even in a single-fielded simple search. 
+3. Allow a different set of qf to be used for any "phrase term", so
+   phrases would search only on non-stemming fields. This would be cool,
+   but kind of do weird things with dismax mm effects, since it would
+   mean all phrases would be extracted into seperate nested queries. 
+4. Better error handling of syntax errors in query entry. Both in the 
+   plugin as a whole, error messages should be displayed on the input
+   screen, so the entry can be fixed. But also using Parslet for parsing,
+   we can potentially deliver better error messages guessing what they
+   got wrong where in their entry. 
+

data/lib/parsing_nesting/grammar.rb

@@ -0,0 +1,78 @@
+require 'rubygems'
+require 'parslet'
+
+# Parslet uses Object#tap, which is in ruby 1.8.7+, but not 1.8.6. 
+# But it's easy enough to implement in pure ruby, let's monkey patch
+# it in if it's not there, so we'll still work with 1.8.6
+unless Object.method_defined?(:tap)
+  class Object
+    def tap
+      yield(self)
+      return self
+    end
+  end
+end
+module ParsingNesting
+  class Grammar < Parslet::Parser
+    root :query
+    
+    # query is actually a list of expressions. 
+    rule :query do
+      (spacing? >>  (expression | paren_unit ) >> spacing?).repeat
+    end
+    
+    rule :paren_list do
+      (str('(') >> query >> str(')')).as(:list)
+    end
+    
+    rule :paren_unit do
+      (str('(') >> spacing? >> (expression ) >> spacing? >> str(')')) |
+        paren_list
+    end
+    
+    # Note well: It was tricky to parse the thing we want where you can
+    # have a flat list with boolean operators, but where 'OR' takes precedence.
+    # eg "A AND B OR C AND C" or "A OR B AND C OR D". Tricky to parse at all,
+    # tricky to make precedence work. Important things that seem to make it work:
+    # and_list comes BEFORE or_list in :expression.
+    # and_list's operand can be an or_list, but NOT vice versa
+    # There are others, it was an iterative process with testing. 
+    rule :expression do
+      (and_list | or_list | unary_expression )
+    end
+      
+    rule :and_list do
+      ((or_list | unary_expression | paren_unit) >> 
+        (spacing >> str("AND") >> spacing >> (or_list | unary_expression | paren_unit)).repeat(1)).as(:and_list) 
+    end
+    
+    rule :or_list do
+      ((unary_expression | paren_unit) >> 
+      (spacing >> str("OR") >> spacing >> (unary_expression | paren_unit)).repeat(1)).as(:or_list) 
+    end
+      
+    rule :unary_expression do
+      (str('+') >> (phrase | token)).as(:mandatory) |
+      (str('-') >> (phrase | token)).as(:excluded) |
+      (str('NOT') >> spacing? >> (unary_expression | paren_unit)).as(:not_expression) |
+      (phrase | token)
+    end
+    
+    rule :token do
+      match['^ ")('].repeat(1).as(:token)
+    end
+    rule :phrase do
+      match('"') >> match['^"'].repeat(1).as(:phrase)  >> match('"')  
+    end
+    
+    
+    rule :spacing do
+      match[' '].repeat(1)    
+    end
+    rule :spacing? do
+      spacing.maybe
+    end
+  end
+
+  
+end