cql-ruby 0.7.1 → 0.8.0

data/lib/cql_ruby/cql_to_solr.rb

@@ -1,57 +1,160 @@
+# This file adds #to_solr method to CqlRuby::CqlNodes, to
+# convert parsed CQL to a query in the lucene-solr syntax.
+# http://wiki.apache.org/solr/SolrQuerySyntax
+#
+# CQL version 1.2 spec was used to understand CQL semantics, although this
+# will likely work for other CQL versions as well.  
+#
+# All indexes specified in CQL are mapped to Solr fields, assumed to exist, 
+# with the same name as CQL index. Any 'context set' namespace prefixes
+# on indexes are ignored, just the base name is mapped to solr field.
+
+# The server-choice index specifications cql.anyindexes, cql.serverchoice,
+# cql.keywords all map by default to no specified index (let
+# solr use default from perhaps a solr 'df' param), but this can be changed
+# with CqlRuby.to_solr_defaults[:default_index].  cql.allindexes maps
+# by default to 'text', which can be changed with
+# CqlRuby.to_solr_defaults[:all_index]
+#
+# Not all CQL can currently be converted to a solr query. If the CQL includes
+# nodes that can not be converted, an exception will be raised.
+#
+# == CQL expressions that can be converted: ==  
+#   * expressions using the following relations, which can be specified with "cql" prefix or without. 
+#   ** adj
+#   ** all
+#   ** any
+#   ** == (note, for typical tokenized solr fields, this will be the same as adj, which is not quite proper CQL semantics, but best we can do on an arbitrary solr field).
+#   ** <> (note, for typical tokenized solr fields, won't have quite the right CQL semantics, instead of "not exactly equal to", it will be "does not contain the phrase").
+#   ** >, <, <=, >=, within (note, solr range/comparison queries may or may not actually produce anything sensical depending on solr field definition, but CQL to_solr will translate into solr range syntax anyway and hope for the best. )
+#   ** =, the server's choice relation, defaults to 'adj', but can be specified in CqlRuby.to_solr_defaults.
+#   * CQL Boolean Operators AND, OR, and NOT. 
+#
+# == CQL expressions that can NOT be converted (at least in present version) ==
+# And will raise exceptions if you try to call #to_solr on a CQL node which
+# includes or has children that include the following:
+# * PROX (boolean) operator.
+# * cql.encloses relation
+# * Any relation modifiers.
+# * Any boolean (operator) modifiers.
+# * sortBy
+# * inline prefix map/prefix assignment specification. 
+# * cql.resultsetid
+
+# == TODO==
+# * support modifiers on adj relation to map to solr '~' slop param?
+# * change all tests to rspec instead of Test::Unit
+# * implemented for acts_as_solr, either more flavors or more general (from chick, jrochkind isn't sure what this means)
+
+
 module CqlRuby
+  def self.to_solr_defaults
+    @to_solr_params ||= {
+        #What's our default relation for "=" server's choice relation? how about
+        # adj.
+      :default_relation => "cql.adj",
+        # What's our default index for various server's choice index choices?
+        # nil means don't specify an index, let solr take care of it.
+        # Or you can specify one. 
+      :default_index => nil,
+        # What index should we use for cql.allIndexes? Again can be nil
+        # meaning let the solr server use it's default.  
+      :all_index => "text"
+      }
+  end
+  
+
   
-# This set of of overrides to the CqlRuby::CqlNodes provides to_solr methods, where not
-# specified for a given node the CqlNode.to_solr method will be used
-# TODO: SOLR can cover much more functionality of CQL than is captured here
-# TODO: implemented for acts_as_solr, either more flavors or more general
 
 class CqlNode
-  def to_solr
-    quoted_index = maybe_quote( @index )
-    quoted_term = maybe_quote( @term )
-    relation_prefix = @relation.to_solr
-    case quoted_index
-      when 'cql.resultSetId': raise CqlException, "resultSet not supported"
-      when 'cql.allRecords': "[* TO *]"
-      when 'cql.allIndexes': "#{relation_prefix}text:#{quoted_term}"
-      when 'cql.anyIndexes': "#{relation_prefix}text:#{quoted_term}"
-      when 'cql.serverChoice': "#{relation_prefix}#{quoted_term}"
-    else
-      quoted_index.gsub!( /(dc|bath)\./, "" )
-      "#{relation_prefix}#{quoted_index}:#{quoted_term}"
-    end
-    
+  # Default, raise not supported, will be implemented by specific
+  # classes where supported. 
+  def to_solr    
+    raise CqlException.new("#to_solr not supported for #{self.class}:  #{self.to_cql}")
   end
 end
 
-# looks like we are just handling not equal now
-class CqlRelation
-  def to_solr
-    ms = @modifier_set.to_solr
-    if ms == " <> "
-      return "-"
-    end
-    ""
-  end
-end
+
 
 class CqlTermNode
-#  def to_solr
-#    "arghh"
-#  end
+ def to_solr
+    relation = @relation.modifier_set.base            
+
+    relation = CqlRuby.to_solr_defaults[:default_relation] if  relation == "="
+    # If no prefix to relation, normalize to "cql"
+    relation = "cql.#{relation}" unless relation.index(".") || ["<>", "<=", ">=", "<", ">", "=", "=="].include?(relation)
+
+    
+    # What's our default index for server choice indexes? Let's call it
+    # "text".
+    # Otherwise, remove the namespace/"context set" prefix.
+    solr_field = case @index.downcase
+                   when "cql.anyindexes", "cql.serverchoice", "cql.keywords"
+                     CqlRuby.to_solr_defaults[:default_index]
+                   when "cql.allindexes"
+                     CqlRuby.to_solr_defaults[:all_index]
+                   else
+                     @index.gsub(/^[^.]*\./, "")
+                 end
+    
+    raise CqlException.new("resultSet not supported") if @index.downcase == "cql.resultsetid"
+    raise CqlException.new("relation modifiers not supported: #{@relation.modifier_set.to_cql}") if @relation.modifier_set.modifiers.length > 0
+
+    if index.downcase == "cql.allrecords"
+      #WARNING: Not sure if this will actually always work as intended, its
+      # a bit odd. 
+      return "[* TO *]"
+    end
+
+    
+    negate = false
+    
+    value = 
+    case relation
+      # WARNING: Depending on how you've tokenized, <> and == semantics
+      # may not be fully respected. For typical solr fields, will
+      # match/exclude on partial matches too, not only complete matches. 
+      when "<>"
+        negate = true
+        maybe_quote(@term)
+      when "cql.adj", "==":   maybe_quote(@term)                                
+      when "cql.all": '(' + @term.split(/\s/).collect{|a| '+'+a}.join(" ") + ')'          
+      when "cql.any":         '(' + @term.split(/\s/).join(" OR ") + ')'          
+      when ">=":              "[" + maybe_quote(@term) + " TO *]"          
+      when ">":               "{" + maybe_quote(@term) + " TO *}"          
+      when "<=":              "[* TO " + maybe_quote(@term) + "]"          
+      when "<":               "{* TO " + maybe_quote(@term) + "}"
+      when "cql.within"
+        bounds = @term.gsub('"', "").split(/\s/)
+        raise CqlException.new("can not extract two bounding values from within relation term: #{@term}") unless bounds.length == 2
+
+        "[" + maybe_quote(bounds[0]) + " TO " + maybe_quote(bounds[1]) + "]"                  
+      else
+        raise CqlException.new("relation not supported: #{relation}")
+    end
+
+    ret = ""
+    ret += "-" if negate
+    ret += "#{solr_field}:" if solr_field
+    ret += value
+    
+    return ret     
+  end
 end
 
 class CqlBooleanNode
   def to_solr
-    "(#{@left_node.to_solr})#{@modifier_set.to_solr}(#{@right_node.to_solr})"  
+    "(#{@left_node.to_solr} #{@modifier_set.to_solr} #{@right_node.to_solr})"  
   end
 end
 
 class ModifierSet
   def to_solr
-    raise CqlException, "PROX not supported" if @base.upcase == "prox"
-    " #{@base.upcase} "
+    raise CqlException.new("#to_solr not supported for PROX operator") if @base.upcase == "PROX"
+    raise CqlException.new("#to_solr does not support boolean modifiers: #{to_cql}") if @modifiers.length != 0
+    
+    "#{@base.upcase}"
   end
 end
 
-end
+end

data/test/helper.rb

@@ -0,0 +1,9 @@
+require 'rubygems'
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'cql_ruby'
+
+class Test::Unit::TestCase
+end

data/test/test_cql_generator.rb

@@ -14,8 +14,8 @@ class TestCqlGenerator < Test::Unit::TestCase
       puts tree.to_cql
       new_tree = parser.parse( tree.to_cql )
       assert( new_tree )
-      assert( new_tree.to_solr )
-      puts new_tree.to_solr
+      #assert( new_tree.to_solr )
+      #puts new_tree.to_solr
       # puts tree.to_xcql
     end
     puts "done"

data/test/test_cql_to_solr.rb

@@ -2,20 +2,109 @@ require 'test/unit'
 require File.dirname(__FILE__) + '/../lib/cql_ruby'
 
 class CqlToSolrTest < Test::Unit::TestCase
-  def test_cql_to_solr
-    parser = CqlRuby::CqlParser.new
-    tree = parser.parse( "dc.title = dog" )
-    assert_equal( "title:dog", tree.to_solr )
-    tree = parser.parse( "cql.resultSetId = dog" )
-    assert_raises( CqlRuby::CqlException ) { tree.to_solr }
-    tree = parser.parse( "cql.allIndexes = dog" )
-    assert_equal( "text:dog", tree.to_solr )
-    tree = parser.parse( "dog AND cat" )
-    assert_equal( "(dog) AND (cat)", tree.to_solr )
-    tree = parser.parse( "dog and cat" )
-    assert_equal( "(dog) AND (cat)", tree.to_solr )
-    tree = parser.parse( "dc.title <> dog" )
-    assert_equal( "-title:dog", tree.to_solr )
-    
+  @@parser = CqlRuby::CqlParser.new
+  
+  def test_boolean
+    assert_to_solr_eq("dog or cat and mammal", '((dog OR cat) AND mammal)')
+    assert_to_solr_eq("dog or (cat and mammal)", '(dog OR (cat AND mammal))')
+    assert_to_solr_eq('dog not cat', "(dog NOT cat)")
+  end
+
+  def test_unsupported_cql
+    assert_can_not_to_solr("cql.resultSetId = dog")
+    assert_can_not_to_solr("field = value PROX field2 = value2")
+    assert_can_not_to_solr("something cql.encloses 2000")
+    assert_can_not_to_solr("field = dog sortBy someField")
+    assert_can_not_to_solr("field unknownrelation value")
+
+    assert_can_not_to_solr("cat or/rel.combine=sum dog")
+    assert_can_not_to_solr("title any/relevant fish")
+
+    assert_can_not_to_solr('> dc = "http://deepcustard.org/" dc.custardDepth > 10')
+  end
+
+  def test_rel_adj
+    assert_to_solr_eq('column cql.adj "one two three"', 'column:"one two three"')
+  end
+
+  def test_rel_eq
+    # '==' is same as 'adj', best we can do
+    assert_to_solr_eq('column == "one two three"', @@parser.parse('column adj "one two three"').to_solr)
+  end
+
+  def test_rel_any
+    assert_to_solr_eq('column cql.any "one two three"', 'column:(one OR two OR three)')
+  end
+
+  def test_rel_all
+    assert_to_solr_eq('column cql.all "one two three"', 'column:(+one +two +three)')
+  end
+
+  def test_rel_not
+    # Depending on solr schema, this will really map to "does not include phrase", not "does not exactly equal", best we can do. 
+    assert_to_solr_eq('column <> "one two three"', '-column:"one two three"')
+  end
+
+  def test_rel_default
+    # '=' defaults to adj
+    assert_to_solr_eq('column = value', @@parser.parse("column adj value").to_solr)
+
+    # unless we set it otherwise
+    with_cql_default(:default_relation, "any") do
+      assert_to_solr_eq('column = value', @@parser.parse("column any value").to_solr)
+    end
+  end
+
+  def test_range
+    assert_to_solr_eq('column > 100', 'column:{100 TO *}')
+    assert_to_solr_eq('column < 100', 'column:{* TO 100}')
+    assert_to_solr_eq('column >= 100', 'column:[100 TO *]')
+    assert_to_solr_eq('column <= 100', 'column:[* TO 100]')
+    assert_to_solr_eq('column cql.within "100 200"', 'column:[100 TO 200]')
+  end
+
+  def test_drop_index_prefix
+    assert_to_solr_eq("dc.title = frog", @@parser.parse("title = frog").to_solr)
+  end
+
+  def test_specified_default_index    
+    with_cql_default(:default_index, "default_index") do    
+      ["cql.anyindexes", "cql.serverchoice", "cql.keywords"].each do |index|
+        assert_to_solr_eq("#{index} = val", @@parser.parse("default_index = val").to_solr)
+      end
+    end            
+  end
+
+  def test_all_index
+    assert_to_solr_eq("cql.allindexes = val", @@parser.parse("text = val").to_solr)
+
+    with_cql_default(:all_index, "my_all_index") do
+      assert_to_solr_eq("cql.allindexes = val", @@parser.parse("my_all_index = val").to_solr)
+    end        
+  end
+
+#############
+# Helpers
+##############
+  
+  def assert_to_solr_eq(cql, should_solr)
+    solr = @@parser.parse(cql).to_solr
+    assert_equal(should_solr, solr)
+  end
+  
+  def assert_can_not_to_solr(string)
+    assert_raises(CqlRuby::CqlException) do 
+      CqlRuby::CqlParser.new.parse(string).to_solr
+    end
+  end
+
+  def with_cql_default(key, value)
+    old_value = CqlRuby.to_solr_defaults[key]
+    CqlRuby.to_solr_defaults[key] = value
+    begin
+      yield
+    ensure
+      CqlRuby.to_solr_defaults[key] = old_value
+    end    
   end
 end

metadata

@@ -1,58 +1,43 @@
 --- !ruby/object:Gem::Specification 
 name: cql-ruby
 version: !ruby/object:Gem::Version 
-  version: 0.7.1
+  hash: 63
+  prerelease: false
+  segments: 
+  - 0
+  - 8
+  - 0
+  version: 0.8.0
 platform: ruby
 authors: 
+- Jonathan Rochkind
 - Chick Markley
 autorequire: 
 bindir: bin
 cert_chain: []
 
-date: 2008-04-10 00:00:00 -07:00
+date: 2010-06-14 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
-description: CQL parser for Ruby
-email: 
-- chick@qrhino.com
+description: " CQL Parser, with serialization from cql node tree to cql, xcql, and solr query"
+email: cql_ruby@googlegroups.com
 executables: []
 
 extensions: []
 
 extra_rdoc_files: 
-- History.txt
-- License.txt
-- Manifest.txt
 - README.txt
-- test/fixtures/sample_queries.txt
-- website/index.txt
 files: 
-- History.txt
-- License.txt
-- Manifest.txt
-- README.txt
-- Rakefile
-- config/hoe.rb
-- config/requirements.rb
 - lib/cql_ruby.rb
-- lib/cql_ruby/version.rb
 - lib/cql_ruby/cql_generator.rb
 - lib/cql_ruby/cql_lexer.rb
 - lib/cql_ruby/cql_nodes.rb
 - lib/cql_ruby/cql_parser.rb
 - lib/cql_ruby/cql_to_solr.rb
-- log/debug.log
-- script/console
-- script/destroy
-- script/generate
-- script/txt2html
-- setup.rb
-- tasks/deployment.rake
-- tasks/environment.rake
-- tasks/website.rake
-- test/fixtures
-- test/fixtures/sample_queries.txt
+- lib/cql_ruby/version.rb
+- README.txt
+- test/helper.rb
 - test/test_cql_generator.rb
 - test/test_cql_lexer.rb
 - test/test_cql_nodes.rb
@@ -60,39 +45,42 @@ files:
 - test/test_cql_ruby.rb
 - test/test_cql_to_solr.rb
 - test/test_helper.rb
-- website/index.html
-- website/index.txt
-- website/javascripts/rounded_corners_lite.inc.js
-- website/stylesheets/screen.css
-- website/template.html.erb
 has_rdoc: true
-homepage: http://cql-ruby.rubyforge.org
+homepage: http://cql-ruby.rubyforge.org/
+licenses: []
+
 post_install_message: 
 rdoc_options: 
-- --main
-- README.txt
+- --charset=UTF-8
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: cql-ruby
-rubygems_version: 1.0.1
+rubygems_version: 1.3.7
 signing_key: 
-specification_version: 2
-summary: CQL parser for Ruby
+specification_version: 3
+summary: CQL Parser
 test_files: 
+- test/helper.rb
 - test/test_cql_generator.rb
 - test/test_cql_lexer.rb
 - test/test_cql_nodes.rb