sparkql 0.1.8

data/test/unit/parser_compatability_test.rb

@@ -0,0 +1,437 @@
+require 'test_helper'
+
+class ParserCompatabilityTest < Test::Unit::TestCase
+  include Sparkql
+  
+  def setup
+    @expression_keys = [:field, :operator, :value]
+    @multiple_types = [:character,:integer]
+    @bad_character_strings = ["'Fargo's Boat'", "Fargo", "''Fargo''", "'Fargo''s'",
+      "'Fargo", "Fargo'", "\\'Fargo\\'"]
+    @bad_multiple_character_strings = ["'Fargo's Boat'", "Fargo", "''Fargo''", "'Fargo''s'",
+      "'Fargo", "Fargo'", "\\'Fargo\\'"]
+    @all_bad_strings = @bad_character_strings + @bad_multiple_character_strings
+    @test_filters = [
+      {
+        :string => "City Eq 'Fargo'",
+        :type => :character,
+        :operator => "Eq"
+      },
+      {
+        :string => "City Ne 'Fargo'",
+        :type => :character,
+        :operator => "Not Eq"
+      },
+      {
+        :string => "City Eq 'Fargo','Moorhead'",
+        :type => :character,
+        :operator => "In"
+      },
+      {
+        :string => "City Eq 'Fargo','Moorhead','Bemidji','Duluth'",
+        :type => :character,
+        :operator => "In"
+      },
+      {
+        :string => "City Ne 'Fargo','Moorhead','Bemidji','Duluth'",
+        :type => :character,
+        :operator => "Not In"
+      },
+      {
+        :string => "IntegerField Eq 2001",
+        :type => :integer,
+        :operator => "Eq"
+      },
+      {
+        :string => "IntegerField Eq -2001",
+        :type => :integer,
+        :operator => "Eq"
+      },
+      {
+        :string => "IntegerField Eq 2001,2002",
+        :type => :integer,
+        :operator => "In"
+      },
+      {
+        :string => "IntegerField Eq -2001,-2002",
+        :type => :integer,
+        :operator => "In"
+      },
+      {
+        :string => "FloatField Eq 2001.120",
+        :type => :decimal,
+        :operator => "Eq"
+      },
+      {
+        :string => "FloatField Eq -2001.120",
+        :type => :decimal,
+        :operator => "Eq"
+      },
+      {
+        :string => "FloatField Eq -2001.120,-2002.0",
+        :type => :decimal,
+        :operator => "In"
+      },
+      {
+        :string => "DateField Eq 2010-10-10",
+        :type => :date,
+        :operator => "Eq"
+      },
+      {
+        :string => "TimestampField Eq 2010-10-10T10:10:30.000000",
+        :type => :datetime,
+        :operator => "Eq"
+      },
+      {
+        :string => "TimestampField Lt 2010-10-10T10:10:30.000000",
+        :type => :datetime,
+        :operator => "Lt"
+      },
+      {
+        :string => "TimestampField Gt 2010-10-10T10:10:30.000000",
+        :type => :datetime,
+        :operator => "Gt"
+      },
+      {
+        :string => "TimestampField Ge 2010-10-10T10:10:30.000000",
+        :type => :datetime,
+        :operator => "Ge"
+      },
+      {
+        :string => "TimestampField Le 2010-10-10T10:10:30.000000",
+        :type => :datetime,
+        :operator => "Le"
+      },
+      {
+        :string => "BooleanField Eq true",
+        :type => :boolean,
+        :operator => "Eq"
+      },
+      {
+        :string => "BooleanField Eq false",
+        :type => :boolean,
+        :operator => "Eq"
+      }]
+      
+  end
+
+  def compare_expression_to_tokens( expression, tokens )
+    counter = 0
+    @expression_keys.each do |key|
+      assert_equal tokens[counter], expression[key]
+      counter += 1
+    end
+  end
+
+  def find_operator(string)
+    ["Eq","Ne","Gt","Ge","Lt","Le"].each do |op|
+      return op if string.include? " #{op} "
+    end
+    nil
+  end
+
+  test "simple tokenize" do
+    filter = "City Eq 'Fargo'"
+    filter_tokens = filter.split(" ")
+    parser = Parser.new
+    expressions = parser.tokenize( filter )
+
+    assert !parser.errors?
+    assert_equal 1, expressions.size, "#Expressions {expressions.inspect}"
+    compare_expression_to_tokens(expressions.first, filter_tokens)
+  end
+ 
+  test "types" do
+    @test_filters.each do |elem|
+      parser = Parser.new
+      expressions = parser.tokenize( elem[:string] )
+
+      assert !parser.errors?, "Query: #{elem.inspect}"
+      assert_equal elem[:type], expressions.first[:type]
+    end
+  end
+
+  test "operators" do
+    @test_filters.each do |elem|
+      parser = Parser.new
+      expressions = parser.tokenize( elem[:string] )
+      assert !parser.errors?, "Query: #{elem.inspect}"
+      assert_equal elem[:operator], expressions.first[:operator]
+    end
+  end
+
+  test "tokenize with and" do
+    filter = "City Eq 'Fargo' And PropertyType Eq 'A'"
+    filter_a = filter.split(" And ")
+    filter_tokens = []
+    filter_a.each do |f|
+      filter_tokens << f.split(" ")
+    end
+    parser = Parser.new
+    expressions = parser.tokenize( filter )
+
+    assert !parser.errors?
+    assert_equal 2, expressions.size
+
+    counter = 0
+    filter_tokens.each do |t|
+      compare_expression_to_tokens(expressions[counter], t)
+      counter += 1
+    end
+  end
+
+  test "tokenize with or" do
+    filter = "City Eq 'Fargo' Or PropertyType Eq 'A'"
+    filter_a = filter.split(" Or ")
+    filter_tokens = []
+    filter_a.each do |f|
+      filter_tokens << f.split(" ")
+    end
+    parser = Parser.new
+    expressions = parser.tokenize( filter )
+
+    assert !parser.errors?
+    assert_equal 2, expressions.size
+
+    counter = 0
+    filter_tokens.each do |t|
+      compare_expression_to_tokens(expressions[counter], t)
+      counter += 1
+    end
+  end
+
+  test "tokenize fail on missing" do
+    # We want to cut out each piece of this individually, and make sure
+    # tokenization fails
+    filter = "City Eq 'Fargo' And PropertyType Eq 'A'"
+    filter_tokens = filter.split(" ")
+
+    filter_tokens.each do |token|
+      f = filter.gsub(token, "").gsub(/\s+/," ")
+      parser = Parser.new
+      expressions = parser.tokenize( f )
+      assert_nil expressions
+      assert parser.errors?
+    end
+  end
+
+  test "tokenize fail on invalid string operator" do
+
+    filter = "City Eq "
+
+    @bad_character_strings.each do |string|
+      f = filter + string
+      parser = Parser.new
+      expressions = parser.tokenize( f )
+      assert_nil expressions
+      assert parser.errors?
+    end
+  end
+
+  test "tokenize fail on invalid operator or field" do
+    filters = ["Eq Eq 'Fargo'","City City 'Fargo'", "And Eq 'Fargo'",
+      "City And 'Fargo'", "city eq 'Fargo'"]
+    filters.each do |f|
+      parser = Parser.new
+      expressions = parser.tokenize( f )
+      assert_nil expressions, "filter: #{f}"
+      assert parser.errors?
+    end
+  end
+
+  test "tokenize edge case string value" do
+    good_strings = ["'Fargo\\'s Boat'", "'Fargo'", "'Fargo\\'\\'s'",
+      "' Fargo '", " 'Fargo' "]
+
+    filters = ["City Eq ","City  Eq ", "City    Eq    "]
+
+    filters.each do |filter|
+      good_strings.each do |string|
+        f = filter + string
+        parser = Parser.new
+        expressions = parser.tokenize( f )
+        assert !parser.errors?
+        assert_equal 1, expressions.size
+        assert_equal string.strip, expressions.first[:value]
+      end
+    end
+  end
+  
+  test "get multiple values" do
+    @test_filters.each do |f|
+      op = find_operator f[:string] 
+      next unless @multiple_types.include?(f[:type]) || op.nil? 
+      parser = Parser.new
+      val = f[:string].split(" #{op} ")[1]
+      vals = parser.tokenize(f[:string]).first[:value]
+      assert_equal val, Array(vals).join(',')
+    end
+  end
+
+  test "multiples fail with unsupported operators" do
+    ["Gt","Ge","Lt","Le"].each do |op|
+      f = "IntegerType #{op} 100,200" 
+      parser = Parser.new
+      expressions = parser.tokenize( f )
+      assert parser.errors?
+      assert_equal op, parser.errors.first.token
+    end 
+  end
+
+  test "bad multiples" do
+    @all_bad_strings.each do |bad|
+      parser = Parser.new
+      ex = parser.tokenize("City Eq #{bad}")
+      assert parser.errors?
+      assert_nil ex
+    end
+  end
+
+  test "max out values" do
+      parser = Parser.new
+      to_the_max = []
+      35.times do |x|
+        to_the_max << x
+      end
+      ex = parser.tokenize("City Eq #{to_the_max.join(',')}")
+      vals = ex.first[:value]
+      assert_equal 25, vals.size
+  end
+
+  test "max out expressions" do
+      parser = Parser.new
+      to_the_max = []
+      60.times do |x|
+        to_the_max << "City Eq 'Fargo'"
+      end
+      vals = parser.tokenize(to_the_max.join(" And "))
+      assert_equal 50, vals.size
+  end
+
+  test "API-107 And/Or in string spiel" do
+      search_strings = ['Tom And Jerry', 'Tom Or Jerry', 'And Or Eq', 'City Eq \\\'Fargo\\\'',
+        ' And Eq Or ', 'Or And Or']
+      search_strings.each do |s|
+        parser = Parser.new
+        parser.tokenize("City Eq '#{s}' And PropertyType Eq 'A'")
+        assert !parser.errors?
+      end
+  end
+
+  test "general paren test" do
+    [
+      "(City Eq 'Fargo')",
+      "(City Eq 'Fargo') And PropertyType Eq 'A'",
+      "(City Eq 'Fargo') And (City Eq 'Moorhead')"
+    ].each do |filter|
+      parser = Parser.new
+      p = parser.tokenize(filter)
+      assert !parser.errors?
+    end
+  end
+
+  test "general nesting fail test" do
+    [
+      "((City Eq 'Fargo')",
+      "((City Eq 'Fargo') And PropertyType Eq 'A'",
+      "(City Eq 'Fargo')) And (City Eq 'Moorhead')",
+      "City Eq 'Fargo')",
+      "(City Eq 'Fargo') And PropertyType Eq 'A')",
+      "City Eq 'Fargo' (And) City Eq 'Moorhead'"
+    ].each do |filter|
+      parser = Parser.new
+      p = parser.tokenize(filter)
+      assert parser.errors?
+    end
+  end
+
+  test "block group" do
+    parser = Parser.new
+    p = parser.tokenize("(City Eq 'Fargo' Or TotalBr Eq 2) And (City Eq 'Moorhead')")
+    assert !parser.errors?
+    assert p.first[:block_group] == p[1][:block_group]
+    assert p.first[:block_group] == p[2][:block_group] - 1
+  end
+
+  test "proper nesting" do
+    parser = Parser.new
+    p = parser.tokenize("(City Eq 'Fargo' Or TotalBr Eq 2) And PropertyType Eq 'A'")
+    assert !parser.errors?
+    p.each do |token|
+      if ["City","TotalBr"].include?(token[:field])
+        assert_equal 1, token[:level], "Token: #{token.inspect}"
+      else
+        assert_equal 0, token[:level]
+      end
+    end
+
+    parser = Parser.new
+    p = parser.tokenize("(City Eq 'Fargo' Or TotalBr Eq 2 Or City Eq 'Moorhead') " +
+                        "And PropertyType Eq 'A' And (TotalBr Eq 1 And TotalBr Eq 2)")
+    assert !parser.errors?
+    p.each do |token|
+      if ["City","TotalBr"].include?(token[:field])
+        assert_equal 1, token[:level]
+      else
+        assert_equal 0, token[:level]
+      end
+    end
+  end
+
+  test "maximum nesting of 2" do
+    parser = Parser.new
+    p = parser.tokenize("(City Eq 'Fargo' Or (TotalBr Eq 2 And (City Eq 'Moorhead'))) And PropertyType Eq 'A'")
+    assert parser.errors?
+    assert_equal "You have exceeded the maximum nesting level.  Please nest no more than 2 levels deep.", parser.errors.first.message
+  end
+
+  test "tokenize custom field" do
+    filter = '"General Property Description"."Zoning" Eq \'Commercial\''
+    filter_tokens = ['"General Property Description"."Zoning"', 'Eq', "'Commercial'"]
+    parser = Parser.new
+    expressions = parser.tokenize( filter )
+    
+    assert !parser.errors?, "Parser errrors [#{filter}]: #{parser.errors.inspect}"
+    assert_equal 1, expressions.size, "Expression #{expressions.inspect}"
+    compare_expression_to_tokens(expressions.first, filter_tokens)
+    assert expressions.first[:custom_field], "Expression #{expressions.first.inspect}"
+  end
+  
+  test "tokenize custom field with special characters" do
+    filter = '"Security"."@R080T$\' ` ` `#" Eq \'R2D2\''
+    filter_tokens = ['"Security"."@R080T$\' ` ` `#"', 'Eq', "'R2D2'"]
+    parser = Parser.new
+    expressions = parser.tokenize( filter )
+    assert !parser.errors?, "Parser errrors [#{filter}]: #{parser.errors.inspect}"
+    assert_equal 1, expressions.size, "Expression #{expressions.inspect}"
+    compare_expression_to_tokens(expressions.first, filter_tokens)
+    assert expressions.first[:custom_field], "Expression #{expressions.first.inspect}"
+  end
+  
+  test "custom field supports all types" do
+    types = {
+      :character => "'character'",
+      :integer => 1234,
+      :decimal => 12.34,
+      :boolean => true
+    }
+    types.each_pair do |key, value|
+      filter = '"Details"."Random" Eq ' + "#{value}"
+      filter_tokens = ['"Details"."Random"', 'Eq', "#{value}"]
+      parser = Parser.new
+      expressions = parser.tokenize( filter )
+      
+      assert !parser.errors?, "Parser errrors [#{filter}]: #{parser.errors.inspect}"
+      assert_equal 1, expressions.size, "Expression #{expressions.inspect}"
+      compare_expression_to_tokens(expressions.first, filter_tokens)
+      assert expressions.first[:custom_field], "Expression #{expressions.first.inspect}"
+    end
+  end
+
+  test "escape boolean value" do
+    parser = Parser.new
+    expressions = parser.tokenize( "BooleanField Eq true" )
+    assert_equal true, parser.escape_value(expressions.first)
+  end
+  
+end

data/test/unit/parser_test.rb

@@ -0,0 +1,215 @@
+require 'test_helper'
+
+class ParserTest < Test::Unit::TestCase
+  include Sparkql
+
+  def test_simple
+    @parser = Parser.new
+    parse 'Test Eq 10',10.to_s
+    parse 'Test Eq 10.0',10.0.to_s
+    parse 'Test Eq true',true.to_s
+    parse "Test Eq 'false'","'false'"
+  end
+
+  def test_conjunction
+    @parser = Parser.new
+    expression = @parser.parse('Test Eq 10 And Test Ne 11')
+    assert_equal 10.to_s, expression.first[:value]
+    assert_equal 11.to_s, expression.last[:value]
+    assert_equal 'And', expression.last[:conjunction]
+    expression = @parser.parse('Test Eq 10 Or Test Ne 11')
+    assert_equal 10.to_s, expression.first[:value]
+    assert_equal 11.to_s, expression.last[:value]
+    assert_equal 'Or', expression.last[:conjunction]
+  end
+  
+  def test_tough_conjunction
+    @parser = Parser.new
+    expression = @parser.parse('Test Eq 10 Or Test Ne 11 And Test Ne 9')
+    assert_equal 9.to_s, expression.last[:value]
+    assert_equal 'And', expression.last[:conjunction]
+  end
+
+  def test_grouping
+    @parser = Parser.new
+    expression = @parser.parse('(Test Eq 10)').first
+    assert_equal 10.to_s, expression[:value]
+    expression = @parser.parse('(Test Eq 10 Or Test Ne 11)')
+    assert_equal 10.to_s, expression.first[:value]
+    assert_equal 11.to_s, expression.last[:value]
+    assert_equal 'Or', expression.last[:conjunction]
+    expression = @parser.parse('(Test Eq 10 Or (Test Ne 11))')
+    assert_equal 10.to_s, expression.first[:value]
+    assert_equal 11.to_s, expression.last[:value]
+    assert_equal 'Or', expression.last[:conjunction]
+  end
+
+  def test_multiples
+    @parser = Parser.new
+    expression = @parser.parse('(Test Eq 10,11,12)').first
+    assert_equal [10.to_s,11.to_s,12.to_s], expression[:value]
+  end
+    
+  def parse(q,v)
+    expressions = @parser.parse(q)
+    assert !@parser.errors?, "Unexpected error parsing #{q} #{@parser.errors.inspect}"
+    assert_equal v, expressions.first[:value], "Expression #{expressions.inspect}"
+    assert !expressions.first[:custom_field], "Unexepected custom field #{expressions.inspect}"
+  end
+
+  def test_invalid_syntax
+    @parser = Parser.new
+    expression = @parser.parse('Test Eq DERP')
+    assert @parser.errors?, "Should be nil: #{expression}"
+  end
+  
+  def test_nesting
+    assert_nesting(
+      "City Eq 'Fargo' Or (BathsFull Eq 1 Or BathsFull Eq 2) Or City Eq 'Moorhead' Or City Eq 'Dilworth'",
+      [0,1,1,0,0]
+    )
+  end
+
+  def test_multilevel_nesting
+    assert_nesting(
+      "(City Eq 'Fargo' And (BathsFull Eq 1 Or BathsFull Eq 2)) Or City Eq 'Moorhead' Or City Eq 'Dilworth'",
+      [1,2,2,0,0]
+    )
+    
+    # API-629
+    assert_nesting(
+      "((MlsStatus Eq 'A') Or (MlsStatus Eq 'D' And CloseDate Ge 2011-05-17)) And ListPrice Ge 150000.0 And PropertyType Eq 'A'",
+      [2,2,2,0,0],
+      [2,3,3,0,0]
+    )
+    assert_nesting(
+      "ListPrice Ge 150000.0 And PropertyType Eq 'A' And ((MlsStatus Eq 'A') Or (MlsStatus Eq 'D' And CloseDate Ge 2011-05-17))",
+      [0,0,2,2,2],
+      [0,0,2,3,3]
+    )
+  end
+  
+  # verify each expression in the query is at the right nesting level and group
+  def assert_nesting(sparkql, levels=[], block_groups=nil)
+    block_groups = levels.clone if block_groups.nil?
+    parser = Parser.new
+    expressions = parser.parse(sparkql)
+    assert !parser.errors?, "Unexpected error parsing #{sparkql}: #{parser.errors.inspect}"
+    count = 0
+    expressions.each do |ex|
+      assert_equal levels[count],  ex[:level], "Nesting level wrong for #{ex.inspect}"
+      assert_equal(block_groups[count],  ex[:block_group], "Nesting block group wrong for #{ex.inspect}")
+      count +=1
+    end
+  end    
+
+  def test_bad_queries
+    filter = "City IsLikeA 'Town'"
+    @parser = Parser.new
+    expressions = @parser.parse(filter)
+    assert @parser.errors?, "Should be nil: #{expressions}"
+    assert @parser.fatal_errors?, "Should be nil: #{@parser.errors.inspect}"
+  end
+
+  def test_function_days
+    d = Date.today
+    dt = DateTime.new(d.year, d.month,d.day, 0,0,0, DateTime.now.offset)
+    start = Time.parse(dt.to_s)
+    filter = "OriginalEntryTimestamp Ge days(-7)"
+    @parser = Parser.new
+    expressions = @parser.parse(filter)
+    assert !@parser.errors?, "errors #{@parser.errors.inspect}"
+    test_time = Time.parse(expressions.first[:value])
+    
+    assert (-605000 < test_time - start && -604000 > test_time - start), "Time range off by more than five seconds #{test_time - start} '#{test_time} - #{start}'"
+  end
+
+  def test_function_now
+    start = Time.now
+    filter = "City Eq now()"
+    @parser = Parser.new
+    expressions = @parser.parse(filter)
+    assert !@parser.errors?, "errors #{@parser.errors.inspect}"
+    test_time = Time.parse(expressions.first[:value])
+    assert 5 > test_time - start, "Time range off by more than five seconds #{test_time - start}"
+    assert -5 < test_time - start, "Time range off by more than five seconds #{test_time - start}"
+  end
+  
+  def test_for_reserved_words_first_literals_second
+    ["OrOrOr Eq true", "Equador Eq true", "Oregon Ge 10"].each do |filter|
+      @parser = Parser.new
+      expressions = @parser.parse(filter)
+      assert !@parser.errors?, "Filter '#{filter}' errors: #{@parser.errors.inspect}"
+    end
+  end
+  
+  def test_custom_fields
+    filter = '"General Property Description"."Taxes" Lt 500.0'
+    @parser = Parser.new
+    expressions = @parser.parse(filter)
+    assert !@parser.errors?, "errors #{@parser.errors.inspect}"
+    assert_equal '"General Property Description"."Taxes"', expressions.first[:field], "Custom field expression #{expressions.inspect}"
+    assert expressions.first[:custom_field], "Custom field expression #{expressions.inspect}"
+    assert_equal '500.0', expressions.first[:value], "Custom field expression #{expressions.inspect}"
+  end
+  
+  def test_valid_custom_field_filters
+    ['"General Property Description"."Taxes$" Lt 500.0',
+      '"General Property Desc\'"."Taxes" Lt 500.0',
+      '"General Property Description"."Taxes" Lt 500.0',
+      '"General \'Property\' Description"."Taxes" Lt 500.0',
+      '"General Property Description"."Taxes #" Lt 500.0',
+      '"General$Description"."Taxes" Lt 500.0',
+      '" a "." b " Lt 500.0'
+    ].each do |filter|
+      @parser = Parser.new
+      expressions = @parser.parse(filter)
+      assert !@parser.errors?, "errors '#{filter}'\n#{@parser.errors.inspect}"
+    end
+  end
+  
+  def test_invalid_custom_field_filters
+    ['"$General Property Description"."Taxes" Lt 500.0',
+      '"General Property Description"."$Taxes" Lt 500.0',
+      '"General Property Description"."Tax.es" Lt 500.0',
+      '"General Property Description".".Taxes" Lt 500.0',
+      '"General Property Description".".Taxes"."SUB" Lt 500.0',
+      '"General.Description"."Taxes" Lt 500.0',
+      '""."" Lt 500.0'
+    ].each do |filter|
+      @parser = Parser.new
+      expressions = @parser.parse(filter)
+      assert @parser.errors?, "No errors? '#{filter}'\n#{@parser.inspect}"
+    end
+  end
+
+  def test_case_insensitve_ops_and_conjunctions
+    @parser = Parser.new
+    parse 'Test EQ 10',10.to_s
+    parse 'Test eq 10.0',10.0.to_s
+    parse 'Test eQ true',true.to_s
+    parse 'Test EQ 10 AND Test NE 11', 10.to_s
+    parse 'Test eq 10 or Test ne 11', 10.to_s
+  end
+  
+  def test_null
+    @parser = Parser.new
+    parse 'Test Eq NULL', "NULL"
+    parse 'Test Eq NULL Or Test Ne 11', "NULL"
+  end
+
+  def test_invalid_operators
+    (Sparkql::Token::OPERATORS - Sparkql::Token::EQUALITY_OPERATORS).each do |o|
+      ["NULL", "true", "'My String'"].each do |v|
+        parser_errors("Test #{o} #{v}")
+      end
+    end
+  end
+
+  def parser_errors(filter)  
+    @parser = Parser.new
+    expression = @parser.parse(filter)
+    assert @parser.errors?, "Should find errors for '#{filter}': #{expression}"
+  end
+    
+end