wvanbergen-scoped_search 1.2.0 → 2.0.0

data/lib/scoped_search/query_language.rb

@@ -0,0 +1,30 @@
+module ScopedSearch::QueryLanguage
+
+  require 'scoped_search/query_language/ast'
+  require 'scoped_search/query_language/tokenizer'
+  require 'scoped_search/query_language/parser'
+
+  class Compiler 
+
+    include Tokenizer
+    include Parser
+    include Enumerable
+    
+    def initialize(str)
+      @str = str
+    end
+    
+    def self.parse(str)
+      compiler = self.new(str)
+      compiler.parse
+    end
+    
+    def self.tokenize(str)
+      compiler = self.new(str)
+      compiler.tokenize
+    end
+        
+  end
+end
+
+

data/lib/scoped_search/query_language/ast.rb

@@ -0,0 +1,141 @@
+module ScopedSearch::QueryLanguage::AST
+  
+  # Constructs an AST from an array notation.
+  def self.from_array(arg) 
+    if arg.kind_of?(Array)
+      operator = arg.shift
+      case operator
+      when :and, :or
+        LogicalOperatorNode.new(operator, arg.map { |c| from_array(c) })
+      when Symbol
+        OperatorNode.new(operator, arg.map { |c| from_array(c) })
+      else
+        raise ScopedSearch::Exception, "Not a valid array representation of an AST!"
+      end
+    else
+      return LeafNode.new(arg)
+    end
+  end
+  
+  # Base AST node class. Instances of this class are used to represent an abstract syntax tree.
+  # This syntax tree is created by the ScopedSearch::QueryLanguage parser and visited by the
+  # ScopedSearch::QueryBuilder to create SQL query conditions.
+  class Node
+    
+    def inspect # :nodoc
+      "<AST::#{self.class.to_s.split('::').last} #{self.to_a.inspect}>"
+    end
+    
+    # Tree simplification. By default, do nothing and return the node as is.
+    def simplify 
+      return self
+    end
+    
+    def compatible_with(node) # :nodoc
+      false  
+    end 
+  end
+  
+  # AST lead node. This node represents leafs in the AST and can represent
+  # either a search phrase or a search field name. 
+  class LeafNode < Node
+    attr_reader :value
+   
+    def initialize(value) # :nodoc
+      @value = value
+    end
+     
+    # Return an array representation for the node     
+    def to_a
+      value
+    end
+    
+    def eql?(node) # :nodoc
+      node.kind_of?(LeafNode) && node.value == value
+    end
+  end
+
+  # AST class for representing operators in the query. An operator node has an operator 
+  # and operands that are represented as AST child nodes. Usually, operator nodes have
+  # one or two children.
+  # For logical operators, a distinct subclass exists to implement some tree
+  # simplification rules.
+  class OperatorNode < Node
+    attr_reader :operator
+    attr_reader :children
+     
+    def initialize(operator, children) # :nodoc
+      @operator = operator
+      @children = children
+    end
+    
+    # Tree simplicication: returns itself after simpifying its children 
+    def simplify
+      @children = children.map { |c| c.simplify }
+      return self
+    end
+    
+    # Return an array representation for the node 
+    def to_a
+      [@operator] + @children.map { |c| c.to_a }
+    end
+    
+    def eql?(node) # :nodoc
+      node.kind_of?(OperatorNode) && node.operator == operator && node.children.eql?(children)
+    end
+    
+    # Return the left-hand side (LHS) operand for this operator.
+    def lhs
+      raise ScopedSearch::Exception, "Operator does not have a LHS" if prefix?
+      raise ScopedSearch::Exception, "Operators with more than 2 children do not have LHS/RHS" if children.length > 2
+      children[0]
+    end       
+    
+    # Return the right-hand side (RHS) operand for this operator.    
+    def rhs
+      raise ScopedSearch::Exception, "Operators with more than 2 children do not have LHS/RHS" if children.length > 2      
+      children.length == 1 ? children[0] : children[1]
+    end
+    
+    # Returns true if this is an infix operator
+    def infix?
+      children.length > 1
+    end
+    
+    # Returns true if this is a prefix operator
+    def prefix?
+      children.length == 1
+    end
+    
+    # Returns a child node by index, starting with 0.
+    def [](child_nr)
+      children[child_nr]
+    end
+    
+  end
+  
+  # AST class for representing AND or OR constructs.
+  # Logical constructs can be simplified resulting in a less complex AST.
+  class LogicalOperatorNode < OperatorNode
+   
+    # Checks whether another node is comparable so that it can be used for tree simplification.
+    # A node can only be simplified if the logical operator is equal.
+    def compatible_with(node)
+      node.kind_of?(LogicalOperatorNode) && node.operator == self.operator
+    end
+   
+    # Simplifies nested AND and OR constructs to single constructs with multiple arguments:
+    # e.g. (a AND (b AND c)) -> (a AND b AND c)
+    def simplify
+      if children.length == 1
+        # AND or OR constructs do nothing if they only have one operand
+        # So remove the logal operator from the AST by simply using the opeand
+        return children.first.simplify
+      else
+        # nested AND or OR constructs can be combined into one construct
+        @children = children.map { |c| c.simplify }.map { |c| self.compatible_with(c) ? c.children : c }.flatten
+        return self  
+      end
+    end
+  end
+end

data/lib/scoped_search/query_language/parser.rb

@@ -0,0 +1,115 @@
+module ScopedSearch::QueryLanguage::Parser
+
+  DEFAULT_SEQUENCE_OPERATOR = :and
+  
+  LOGICAL_INFIX_OPERATORS  = [:and, :or]
+  LOGICAL_PREFIX_OPERATORS = [:not]
+  NULL_PREFIX_OPERATORS    = [:null, :notnull]
+  COMPARISON_OPERATORS = [:eq, :ne, :gt, :gte, :lt, :lte, :like, :unlike]
+  ALL_INFIX_OPERATORS = LOGICAL_INFIX_OPERATORS + COMPARISON_OPERATORS
+  ALL_PREFIX_OPERATORS = LOGICAL_PREFIX_OPERATORS + COMPARISON_OPERATORS + NULL_PREFIX_OPERATORS
+  
+  # Start the parsing process by parsing an expression sequence
+  def parse
+    @tokens = tokenize   
+    parse_expression_sequence(true).simplify
+  end
+
+  # Parses a sequence of expressions
+  def parse_expression_sequence(initial = false)
+    expressions = []
+    next_token if !initial && peek_token == :lparen # skip staring :lparen    
+    expressions << parse_logical_expression until peek_token.nil? || peek_token == :rparen
+    next_token if !initial && peek_token == :rparen # skip final :rparen
+    return ScopedSearch::QueryLanguage::AST::LogicalOperatorNode.new(DEFAULT_SEQUENCE_OPERATOR, expressions)
+  end
+  
+
+  # Parses a logical expression.
+  def parse_logical_expression
+    lhs = case peek_token
+      when nil;             nil
+      when :lparen;         parse_expression_sequence
+      when :not;            parse_logical_not_expression
+      when :null, :notnull; parse_null_expression
+      else;                 parse_comparison
+    end
+
+    if LOGICAL_INFIX_OPERATORS.include?(peek_token)
+      operator = next_token
+      rhs = parse_logical_expression
+      ScopedSearch::QueryLanguage::AST::LogicalOperatorNode.new(operator, [lhs, rhs])
+    else
+      lhs
+    end
+  end  
+  
+  # Parses a NOT expression
+  def parse_logical_not_expression
+    next_token # = skip NOT operator
+    negated_expression = case peek_token
+      when :not;    parse_logical_not_expression 
+      when :lparen; parse_expression_sequence
+      else          parse_comparison
+    end
+    return ScopedSearch::QueryLanguage::AST::OperatorNode.new(:not, [negated_expression])
+  end
+  
+  # Parses a set? or null? expression
+  def parse_null_expression
+    return ScopedSearch::QueryLanguage::AST::OperatorNode.new(next_token, [parse_value])
+  end
+  
+  # Parses a comparison
+  def parse_comparison
+    next_token if peek_token == :comma # skip comma      
+    return (String === peek_token) ? parse_infix_comparison : parse_prefix_comparison
+  end
+  
+  # Parses a prefix comparison, i.e. without an explicit field: <operator> <value>
+  def parse_prefix_comparison
+    return ScopedSearch::QueryLanguage::AST::OperatorNode.new(next_token, [parse_value])
+  end  
+
+  # Parses an infix expression, i.e. <field> <operator> <value>
+  def parse_infix_comparison
+    lhs = parse_value
+    return case peek_token
+      when nil
+        lhs
+      when :comma
+        next_token # skip comma
+        lhs
+      else
+        if COMPARISON_OPERATORS.include?(peek_token)
+          comparison_operator = next_token
+          rhs = parse_value
+          ScopedSearch::QueryLanguage::AST::OperatorNode.new(comparison_operator, [lhs, rhs])
+        else
+          lhs
+        end
+    end
+  end  
+  
+  # Parses a single value.
+  # This can either be a constant value or a field name.
+  def parse_value
+    raise ScopedSearch::QueryNotSupported, "Value expected but found #{peek_token.inspect}" unless String === peek_token
+    ScopedSearch::QueryLanguage::AST::LeafNode.new(next_token)
+  end
+
+  protected 
+  
+  def current_token
+    @current_token
+  end
+  
+  def peek_token(amount = 1)
+    @tokens[amount - 1]
+  end
+
+  def next_token
+    @current_token = @tokens.shift
+  end  
+
+end

data/lib/scoped_search/query_language/tokenizer.rb

@@ -0,0 +1,62 @@
+module ScopedSearch::QueryLanguage::Tokenizer
+
+  KEYWORDS = { 'and' => :and, 'or' => :or, 'not' => :not, 'set?' => :notnull, 'null?' => :null }
+  OPERATORS = { '&' => :and, '|' => :or, '&&' => :and, '||' => :or, '-'=> :not, '!' => :not, '~' => :like, '!~' => :unlike,
+      '=' => :eq, '==' => :eq, '!=' => :ne, '<>' => :ne, '>' => :gt, '<' => :lt, '>=' => :gte, '<=' => :lte }
+
+  
+  def tokenize
+    @current_char_pos = -1
+    to_a
+  end
+
+  def current_char
+    @current_char
+  end
+
+  def peek_char(amount = 1)
+    @str[@current_char_pos + amount, 1]
+  end
+
+  def next_char
+    @current_char_pos += 1
+    @current_char = @str[@current_char_pos, 1]
+  end
+
+  def each_token(&block)
+    while next_char
+      case current_char
+      when /^\s?$/; # ignore
+      when '(';  yield(:lparen)
+      when ')';  yield(:rparen)
+      when ',';  yield(:comma)
+      when /\&|\||=|<|>|!|~|-/;  tokenize_operator(&block)
+      when '"';                  tokenize_quoted_keyword(&block)
+      else;                      tokenize_keyword(&block)
+      end
+    end
+  end  
+
+  def tokenize_operator(&block)
+    operator = current_char
+    operator << next_char if OPERATORS.has_key?(operator + peek_char)
+    yield(OPERATORS[operator])
+  end
+
+  def tokenize_keyword(&block)
+    keyword = current_char
+    keyword << next_char while /[^=<>\s\&\|\)\(,]/ =~ peek_char      
+    KEYWORDS.has_key?(keyword.downcase) ? yield(KEYWORDS[keyword.downcase]) : yield(keyword)
+  end
+
+  def tokenize_quoted_keyword(&block)
+    keyword = ""
+    until next_char.nil? || current_char == '"'
+      keyword << (current_char == "\\" ? next_char : current_char)
+    end
+    yield(keyword)      
+  end
+
+  alias :each :each_token
+  
+end

data/spec/integration/api_spec.rb

@@ -0,0 +1,82 @@
+require "#{File.dirname(__FILE__)}/../spec_helper"
+
+describe ScopedSearch, "API" do
+  
+  # This spec requires the API to be stable, so that projects using
+  # scoped_search do not have to update their code if a new (minor)
+  # version is released. 
+  #
+  # API compatibility is only guaranteed for minor version changes;
+  # New major versions may change the API and require code changes 
+  # in projects using this plugin.
+  #
+  # Because of the API stability guarantee, these spec's may only
+  # be changed for new major releases.
+  
+  before(:all) do
+    ScopedSearch::Spec::Database.establish_connection
+  end
+
+  after(:all) do
+    ScopedSearch::Spec::Database.close_connection    
+  end  
+  
+  context 'for unprepared ActiveRecord model' do
+        
+    it "should respond to :scoped_search to setup scoped_search for the model" do
+      Class.new(ActiveRecord::Base).should respond_to(:scoped_search)
+    end
+  end
+  
+  context 'for a prepared ActiveRecord model' do
+    
+    before(:all) do
+      @class = ScopedSearch::Spec::Database.create_model(:field => :string) do |klass|
+        klass.scoped_search :on => :field
+      end
+    end
+    
+    after(:all) do
+      ScopedSearch::Spec::Database.drop_model(@class)    
+    end
+    
+    it "should respond to :search_for to perform searches" do
+      @class.should respond_to(:search_for)
+    end
+    
+    it "should return an ActiveRecord::NamedScope::Scope when :search_for is called" do
+      @class.search_for('query').class.should eql(ActiveRecord::NamedScope::Scope)
+    end
+  end
+  
+  context 'having backwards compatibility' do
+    
+    before(:each) do
+      class Foo < ActiveRecord::Base
+        belongs_to :bar
+      end
+    end
+    
+    after(:each) do
+      Object.send :remove_const, :Foo
+    end
+    
+    it "should respond to :searchable_on" do
+      Foo.should respond_to(:searchable_on)
+    end
+
+    it "should create a Field instance for every argument" do
+      ScopedSearch::Definition::Field.should_receive(:new).exactly(3).times
+      Foo.searchable_on(:field_1, :field_2, :field_3)
+    end
+
+    it "should create a Field with a valid relation when using the underscore notation" do
+      ScopedSearch::Definition::Field.should_receive(:new).with(
+          instance_of(ScopedSearch::Definition), hash_including(:in => :bar, :on => :related_field))
+          
+      Foo.searchable_on(:bar_related_field)
+    end
+    
+  end
+
+end

data/spec/integration/ordinal_querying_spec.rb

@@ -0,0 +1,153 @@
+require "#{File.dirname(__FILE__)}/../spec_helper"
+
+describe ScopedSearch do
+
+  before(:all) do
+    ScopedSearch::Spec::Database.establish_connection
+
+    @class = ScopedSearch::Spec::Database.create_model(:int => :integer, :timestamp => :datetime, :date => :date, :unindexed => :integer) do |klass|
+      klass.scoped_search :on => [:int, :timestamp]
+      klass.scoped_search :on => :date, :only_explicit => true
+    end
+  end
+
+  after(:all) do
+    ScopedSearch::Spec::Database.drop_model(@class)
+    ScopedSearch::Spec::Database.close_connection
+  end
+
+  context 'quering numerical fields' do
+
+    before(:all) do
+      @record = @class.create!(:int =>  9)
+    end
+
+    after(:all) do
+      @record.destroy
+    end
+
+    it "should find the record with an exact integer match" do
+      @class.search_for('9').should have(1).item
+    end
+
+    it "should find the record with an exact integer match with an explicit operator" do
+      @class.search_for('= 9').should have(1).item
+    end
+
+    it "should find the record with an exact integer match with an explicit field name" do
+      @class.search_for('int = 9').should have(1).item
+    end
+
+    it "should find the record with an exact integer match with an explicit field name" do
+      @class.search_for('int > 8').should have(1).item
+    end
+
+    it "should find the record with a grater than operator and explicit field" do
+      @class.search_for('int > 9').should have(0).item
+    end
+
+    it "should find the record with an >= operator with an implicit field name" do
+      @class.search_for('>= 9').should have(1).item
+    end
+
+    it "should not return the record if only one predicate is true and AND is used (by default)" do
+      @class.search_for('int <= 8, int > 8').should have(0).item
+    end
+
+    it "should return the record in only one predicate is true and OR is used as operator" do
+      @class.search_for('int <= 8 || int > 8').should have(1).item
+    end
+  end
+
+  context 'querying unindexed fields' do
+
+    before(:all) do
+      @record = @class.create!(:int =>  9, :unindexed => 10)
+    end
+
+    after(:all) do
+      @record.destroy
+    end
+
+    it "should raise an error when explicitly searching in the non-indexed column" do
+      lambda { @class.search_for('unindexed = 10') }.should raise_error(ScopedSearch::Exception)
+    end
+
+    it "should not return records for which the query matches unindex records" do
+      @class.search_for('= 10').should have(0).item
+    end
+  end
+
+  context 'querying date and time fields' do
+
+    before(:all) do
+      @record = @class.create!(:timestamp => Time.parse('2009-01-02 14:51:44'), :date => Date.parse('2009-01-02'))
+      @nil_record = @class.create!(:timestamp => nil, :date => nil)
+    end
+
+    after(:all) do
+      @record.destroy
+      @nil_record.destroy
+    end
+
+    it "should accept YYYY-MM-DD as date format" do
+      @class.search_for('date = 2009-01-02').should have(1).item
+    end
+
+    it "should accept YY-MM-DD as date format" do
+      @class.search_for('date = 09-01-02').should have(1).item
+    end
+    
+    it "should accept MM/DD/YY as date format" do
+      @class.search_for('date = 01/02/09').should have(1).item
+    end    
+    
+    it "should accept YYYY/MM/DD as date format" do
+      @class.search_for('date = 2009/01/02').should have(1).item
+    end
+    
+    it "should accept MM/DD/YYYY as date format" do
+      @class.search_for('date = 01/02/2009').should have(1).item
+    end      
+    
+    it "should ignore an invalid date and thus return all records" do
+      @class.search_for('>= 2009-14-57').should have(2).items
+    end      
+    
+    it "should find the records with a timestamp set some point on the provided date" do
+      @class.search_for('>= 2009-01-02').should have(1).item
+    end
+    
+    it "should support full timestamps" do
+      @class.search_for('> "2009-01-02 02:02:02"').should have(1).item
+    end
+    
+    it "should find no record with a timestamp in the past" do
+      @class.search_for('< 2009-01-02').should have(0).item
+    end
+
+    it "should find all timestamps on a date if no time is given using the = operator" do
+      @class.search_for('= 2009-01-02').should have(1).item
+    end
+    
+    it "should find all timestamps on a date if no time is when no operator is given" do
+      @class.search_for('2009-01-02').should have(1).item
+    end    
+    
+    it "should find all timestamps not on a date if no time is given using the != operator" do
+      @class.search_for('!= 2009-01-02').should have(0).item
+    end      
+
+    it "should find the records when the date part of a timestamp matches a date" do
+      @class.search_for('>= 2009-01-02').should have(1).item
+    end
+
+    it "should find the record with the timestamp today or in the past" do
+      @class.search_for('<= 2009-01-02').should have(1).item
+    end
+
+    it "should find no record with a timestamp later than today" do
+      @class.search_for('> 2009-01-02').should have(0).item
+    end
+  end
+end