cql-ruby 0.7.1

data/tasks/deployment.rake

@@ -0,0 +1,34 @@
+desc 'Release the website and new gem version'
+task :deploy => [:check_version, :website, :release] do
+  puts "Remember to create SVN tag:"
+  puts "svn copy svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/trunk " +
+    "svn+ssh://#{rubyforge_username}@rubyforge.org/var/svn/#{PATH}/tags/REL-#{VERS} "
+  puts "Suggested comment:"
+  puts "Tagging release #{CHANGES}"
+end
+
+desc 'Runs tasks website_generate and install_gem as a local deployment of the gem'
+task :local_deploy => [:website_generate, :install_gem]
+
+task :check_version do
+  unless ENV['VERSION']
+    puts 'Must pass a VERSION=x.y.z release version'
+    exit
+  end
+  unless ENV['VERSION'] == VERS
+    puts "Please update your version.rb to match the release version, currently #{VERS}"
+    exit
+  end
+end
+
+desc 'Install the package as a gem, without generating documentation(ri/rdoc)'
+task :install_gem_no_doc => [:clean, :package] do
+  sh "#{'sudo ' unless Hoe::WINDOZE }gem install pkg/*.gem --no-rdoc --no-ri"
+end
+
+namespace :manifest do
+  desc 'Recreate Manifest.txt to include ALL files'
+  task :refresh do
+    `rake check_manifest | patch -p0 > Manifest.txt`
+  end
+end

data/tasks/environment.rake

@@ -0,0 +1,7 @@
+task :ruby_env do
+  RUBY_APP = if RUBY_PLATFORM =~ /java/
+    "jruby"
+  else
+    "ruby"
+  end unless defined? RUBY_APP
+end

data/tasks/website.rake

@@ -0,0 +1,17 @@
+desc 'Generate website files'
+task :website_generate => :ruby_env do
+  (Dir['website/**/*.txt'] - Dir['website/version*.txt']).each do |txt|
+    sh %{ #{RUBY_APP} script/txt2html #{txt} > #{txt.gsub(/txt$/,'html')} }
+  end
+end
+
+desc 'Upload website files to rubyforge'
+task :website_upload do
+  host = "#{rubyforge_username}@rubyforge.org"
+  remote_dir = "/var/www/gforge-projects/#{PATH}/"
+  local_dir = 'website'
+  sh %{rsync -aCv #{local_dir}/ #{host}:#{remote_dir}}
+end
+
+desc 'Generate and upload website files'
+task :website => [:website_generate, :website_upload, :publish_docs]

data/test/fixtures/sample_queries.txt

@@ -0,0 +1,112 @@
+# These are from the java distribution
+# Simple 
+
+cat
+"cat"
+comp.os.linux
+xml:element
+"<xml:element>"
+"="
+"prox/distance<3/unit=word"
+("cat")
+((dog))
+all
+prox
+
+# Index Relation Term
+
+title = "fish"
+title exact fish
+title any fish
+title all fish
+title > 9
+title >= 23
+dc.title any "fish chips"
+dc.title any/stem fish
+dc.fish all/stem/fuzzy "fish chips"
+(title any frog)
+((dc.title any/stem "frog pond"))
+dc.title scr "fish frog chicken"
+dc.title =/rel.algorithm=CORI squid
+author any/f.foo/b.bar>1 "sanderson taylor"
+numberOfLegs <= 4
+numberOfLegs <> 4
+title == jaws
+
+# Simple Boolean
+
+cat or dog
+cat and fish
+cat not frog
+(cat not frog)
+"cat" not "fish food"
+xml and "prox///"
+fred and any
+((fred or all))
+a or b and c not d
+
+# I/R/T plus Boolean
+
+bath.author any fish and dc.title all "cat dog"
+(title any/stem "fish dog" or and)
+
+# Prox
+
+cat prox hat
+cat prox/distance=3/unit=word/ordered hat
+cat prox/distance<3 hat
+"fish food" prox/unit=sentence and
+title all "chips frog" prox/distance<=5 exact
+(dc.author exact "jones" prox/distance>5/unit=element title >= "smith")
+((cat prox hat))
+a and/rel.SumOfScores b
+a and/rel.algorithm=CORI b
+
+# Special Characters
+
+(cat^)
+"cat"
+"^cat says \"fish\""
+"cat*fish"
+cat?dog
+(("^cat*fishdog\"horse?"))
+
+# Nesting Parentheses
+
+(((cat or dog) or horse) and frog)
+(cat and dog) or (horse and frog)
+(cat and (horse or frog)) and chips
+
+# Prefix Maps
+
+> foo="http://www.loc.gov/zing/cql/dc-indexes/" dc.title="fish"
+> "http://www.loc.gov/zing/cql/dc-indexes/" title="fish"
+> foo="http://www.loc.gov/zing/cql/dc-indexes" > ccg = "http://srw.o-r-g.org/cql/indexSets/ccg/" foo.title="fish" and ccg.force=3
+
+# Lame Searches
+
+any or all:stem and all exact any prox prox=fuzzy
+(((((((((any)))))))))
+""
+> any > any = exact any > any
+sortby sortby sortby
+sortby sortby sortby sortby sortby
+
+# Sorting
+
+kernighan sortby title
+kernighan and ritchie sortby title
+dc.creator=kernighan sortby dc.title
+dc.creator=kernighan sortby numberOfLegs/cql.number
+dc.creator=kernighan sortby dc.title/sort.respectCase
+dc.creator=kernighan sortby dc.title/sort.respectCase/sort.descending
+dc.creator=kernighan sortby dc.date dc.title
+dc.creator=kernighan sortby dc.date/sort.missingOmit
+dc.creator=kernighan sortby dc.date/sort.missingValue=1970
+>dc="http://deepcustard.org/1.0" blah sortby dc.custardDepth
+>ns1="http://uri1" >ns2="http://uri2" whatever sortby ns1.key/a/b/c=1 ns2.key2
+(>dc=x b=c) sortby d
+b=(>dc=x c) sortby d
+(>dc=x c1 and c2) sortby d
+>dc="http://deepcustard.org" (>dc="http://dublincore.org" dc.title=jaws) sortby dc.custardDepth
+>dc="http://deepcustard.org" (fish or >dc="http://dublincore.org" dc.title=jaws) sortby dc.custardDepth

data/test/test_cql_generator.rb

@@ -0,0 +1,23 @@
+require 'test/unit'
+require File.dirname(__FILE__) + '/../lib/cql_ruby'
+require File.dirname(__FILE__) + '/../lib/cql_ruby/cql_generator'
+
+class TestCqlGenerator < Test::Unit::TestCase
+  def test_generator
+    params = {:seed => 1, :debug => false }
+    
+    parser = CqlRuby::CqlParser.new
+    generator = CqlRuby::CqlGenerator.new( params )
+    100.times do 
+      tree = generator.generate_cql_query
+      puts "tree built"
+      puts tree.to_cql
+      new_tree = parser.parse( tree.to_cql )
+      assert( new_tree )
+      assert( new_tree.to_solr )
+      puts new_tree.to_solr
+      # puts tree.to_xcql
+    end
+    puts "done"
+  end
+end

data/test/test_cql_lexer.rb

@@ -0,0 +1,54 @@
+require 'test/unit'
+require File.dirname(__FILE__) + '/../lib/cql_ruby'
+
+class TestCqlLexer < Test::Unit::TestCase
+  def test_lexer
+    cl = CqlRuby::CqlLexer.new
+    cl.tokenize( "<a" )
+    assert_equal cl.tokenize( 'one' ), ["one"]
+    assert_equal cl.tokenize( 'one two'), ["one","two"]
+    assert_equal cl.tokenize( '"a b c"'), ['"a b c"']
+    assert_equal cl.tokenize( '"a ( b ) c"'), ['"a ( b ) c"']
+    assert_raises( CqlRuby::CqlException ) { cl.tokenize( '"a ( b ) c') }
+    
+    c0 = CqlRuby::CqlLexer.new( '<a' )
+    assert_equal( '<', c0.next_token )
+    assert_equal( 'a', c0.next_token )
+    
+    c2 = CqlRuby::CqlLexer.new( 'abc 123 "a b c" x < <a and OR Not prox sortby >= <> <= ==' )
+    c2.next_token
+    assert_equal( CqlRuby::CqlLexer::TT_WORD, c2.token_type )
+    assert( c2.word? )
+    c2.next_token
+    assert_equal( CqlRuby::CqlLexer::TT_NUMBER, c2.token_type )
+    assert( c2.number? )
+    assert_equal( '"', c2.next_token )
+    assert_equal( 'a b c', c2.value )
+    assert_equal( 'x', c2.next_token )
+    assert_equal( '<',c2.next_token )
+    assert_equal( '<', c2.next_token )
+    assert_equal( 'a', c2.next_token )
+    assert_equal( CqlRuby::CqlLexer::TT_AND , c2.next_token )
+    assert( c2.and? )
+    assert_equal( CqlRuby::CqlLexer::TT_OR, c2.next_token )
+    assert( c2.or? )
+    assert_equal( CqlRuby::CqlLexer::TT_NOT, c2.next_token )
+    assert( c2.not? )
+    assert_equal( CqlRuby::CqlLexer::TT_PROX, c2.next_token  )
+    assert( c2.prox? )
+    assert_equal( CqlRuby::CqlLexer::TT_SORTBY, c2.next_token )
+    assert( c2.sortby? )
+    assert_equal( CqlRuby::CqlLexer::TT_GE, c2.next_token )
+    assert( c2.greater_than_or_equal? )
+    assert_equal( CqlRuby::CqlLexer::TT_NE, c2.next_token )
+    assert( c2.not_equal? )
+    assert_equal( CqlRuby::CqlLexer::TT_LE, c2.next_token )
+    assert( c2.less_than_or_equal? )
+    assert_equal( CqlRuby::CqlLexer::TT_EQEQ, c2.next_token )
+    assert( c2.double_equal? )
+    assert_equal( CqlRuby::CqlLexer::TT_EOF, c2.next_token )
+    assert( c2.eof? )
+    
+  end
+end
+

data/test/test_cql_nodes.rb

@@ -0,0 +1,105 @@
+require 'test/unit'
+require 'rubygems'
+require File.dirname(__FILE__) + '/../lib/cql_ruby'
+
+class TestCqlNodes < Test::Unit::TestCase
+  require 'builder'
+  
+  def test_node
+    cn = CqlRuby::CqlNode.new
+    
+    cp = CqlRuby::CqlPrefix.new( 'bob', 'jones')
+    
+    es = <<-end_of_xml
+<prefixes>
+ <prefix>
+  <name>bob</name>
+  <identifier>jones</identifier>
+ </prefix>
+</prefixes>
+end_of_xml
+
+    xml = Builder::XmlMarkup.new(:indent => 1)
+    assert_equal( es, cn.render_prefixes( xml, [cp] ) )
+
+
+end
+
+  def test_modifier_set
+
+  end
+
+  def test_cql_relation_node
+    xml = Builder::XmlMarkup.new(:indent => 1)
+    relation = CqlRuby::CqlRelation.new( "=" )
+    
+    assert_equal( "<relation>\n <value>=</value>\n</relation>\n", relation.to_xcql(xml) )
+    assert_equal( "=", relation.to_cql )
+    relation = CqlRuby::CqlRelation.new( "=" )
+    
+    xml = Builder::XmlMarkup.new(:indent => 1)
+    assert_equal( "<relation>\n <value>=</value>\n</relation>\n", relation.to_xcql(xml) )
+    assert_equal( "=", relation.to_cql )
+  end
+  
+  def test_modifier
+    modifier = CqlRuby::Modifier.new( "dog" )
+    assert_equal( "dog", modifier.to_cql )
+    modifier = CqlRuby::Modifier.new( "dog", "=", "cat" )
+    assert_equal( "dog = cat", modifier.to_cql )
+  end
+  
+  def test_modifier_set
+        me = <<-end_of_modifier
+<modifier>
+ <type>a</type>
+ <test>b</test>
+ <value>c</value>
+</modifier>
+end_of_modifier
+
+    m1 = CqlRuby::Modifier.new( 'a', 'b', 'c' )
+    assert_equal( "a b c", m1.to_cql() )
+    xml = Builder::XmlMarkup.new(:indent => 1)
+    assert_equal( me, m1.to_xcql( xml, "test") )
+    m1 = CqlRuby::Modifier.new( 'a' )
+    xml = Builder::XmlMarkup.new(:indent => 1)
+    assert_not_equal( me, m1.to_xcql( xml, "test") )
+    
+    ms = CqlRuby::ModifierSet.new( "base" )
+    ms.add_modifier( "dog" )
+    assert_equal( "base/dog", ms.to_cql )
+    ms.add_modifier( "cat", "wont", "eat" )
+    assert_equal("base/dog/cat wont eat",  ms.to_cql )
+  end
+  
+  def test_cql_boolean_node
+    n1 = CqlRuby::CqlNode.new
+    n2 = CqlRuby::CqlNode.new
+    ms = CqlRuby::ModifierSet.new( "=" )
+    
+    bn = CqlRuby::CqlBooleanNode.new( n1, n2, ms )
+    assert_equal( "() = ()", bn.to_cql )
+    # assert_equal( "() = ()", bn.to_xcql(0) )
+  end
+  
+  def test_cql_term_node
+    rel = CqlRuby::CqlRelation.new( "likes" )
+    tn = CqlRuby::CqlTermNode.new( "dog", rel, "cat" )
+    assert_equal( "dog likes cat", tn.to_cql )
+    xml = Builder::XmlMarkup.new(:indent => 1)
+
+    expected = <<end_of_expected
+<searchClause>
+ <index>dog</index>
+ <relation>
+  <value>likes</value>
+ </relation>
+ <term>cat</term>
+</searchClause>
+end_of_expected
+
+    xml = Builder::XmlMarkup.new(:indent => 1)
+    assert_equal( expected, tn.to_xcql(xml) )
+  end
+end

data/test/test_cql_parser.rb

@@ -0,0 +1,113 @@
+require 'test/unit'
+require File.dirname(__FILE__) + '/../lib/cql_ruby'
+
+class TestCqlParser < Test::Unit::TestCase
+  def test_coverage
+    parser = CqlRuby::CqlParser.new
+    lines = IO.readlines( File.dirname(__FILE__) + '/fixtures/sample_queries.txt' )
+    lines.each do |line|
+      next if /^\s*#/ =~ line
+      begin
+      tree = parser.parse( line )
+      puts "in=#{line} out=#{tree.to_cql}" if tree
+      rescue Exception
+        puts "exception: line=#{line} error=#{$!}"
+      end
+    end
+  end
+  
+  def test_parser
+    parser = CqlRuby::CqlParser.new
+    parser.debug = false
+    
+    
+    tree = parser.parse( ">fox=lynx dog = cat" )
+    assert_equal( '>fox="lynx" (dog = cat)', tree.to_cql )
+    
+    assert_raises( CqlRuby::CqlException ) { parser.parse( "dog sortby" ) }
+    
+    tree = parser.parse( "> dog cat" )
+    assert_equal( '>"dog" (cat)', tree.to_cql )
+    
+    tree = parser.parse( "dc.title < cat" )
+    assert_equal( 'dc.title < cat', tree.to_cql )
+    
+    tree = parser.parse( "dog ** fish" )
+    assert_equal( 'dog ** fish', tree.to_cql )
+    
+    tree = parser.parse( "dog prox fish" )
+    assert_equal( '(dog) prox (fish)', tree.to_cql )
+    
+    tree = parser.parse( "dog not fish" )
+    assert_equal( '(dog) not (fish)', tree.to_cql )
+    
+    assert_raises( CqlRuby::CqlException ) { parser.parse( 'abc = "def' ) }
+
+    tree = parser.parse( "au=(Kerninghan or Ritchie) and ti=Unix" )
+    assert_equal( '((au = Kerninghan) or (au = Ritchie)) and (ti = Unix)', tree.to_cql )
+    
+    query = 'cql.resultSetId = "resultA" and cql.resultSetId = "resultB"'
+    result = '(cql.resultSetId = resulta) and (cql.resultSetId = resultb)'
+    tree = parser.parse( query )
+    assert_equal( result.downcase, tree.to_cql.downcase )
+
+    query = 'dc.title any/relevant/rel.CORI "cat fish"'
+    tree = parser.parse( query )
+    assert_equal( query.downcase, tree.to_cql )
+
+    query = 'cql.resultSetId = "resultA" or cql.resultSetId = "resultB"'
+    tree = parser.parse( query )
+    result = '(cql.resultSetId = resultA) or (cql.resultSetId = resultB)'
+    assert_equal( result, tree.to_cql )
+
+    query = 'dc.title any/relevant/rel.CORI "cat fish" sortBy dc.date/sort.ascending'
+    tree = parser.parse( query )
+    puts "tree=#{tree.to_cql}="
+    assert_equal( 'dc.title any/relevant/rel.cori "cat fish" sortby  dc.date/sort.ascending', tree.to_cql )
+
+    query = 'cql.resultSetId = "resultA" or cql.resultSetId = "resultB"'
+    result = '(cql.resultSetId = resulta) or (cql.resultSetId = resultb)'
+    tree = parser.parse( query )
+    assert_equal( result.downcase, tree.to_cql.downcase )
+
+    query = 'author = "smith"'
+    result = 'author = smith'
+    tree = parser.parse( query )
+    result1 = tree.to_cql
+    assert_equal( result, result1 )
+    tree = parser.parse( result1 )
+    result2 = tree.to_cql
+    assert_equal( result1, result2 )
+
+    result = <<end_of_result
+<searchClause>
+ <index>
+  dc.title
+ </index>
+ <relation>
+  <value>
+   any
+  </value>
+  <modifiers>
+   <modifier>
+    <type>relevant</type>
+   </modifier>
+   <modifier>
+    <type>rel.cori</type>
+   </modifier>
+  </modifiers>
+ </relation>
+ <term>
+  cat fish
+ </term>
+</searchClause>
+end_of_result
+
+    tree = parser.parse( 'dc.title any/relevant/rel.CORI "cat fish"')
+    result.gsub!( /\s+/, '' )
+    tree_string = tree.to_xcql
+    tree_string.gsub!( /\s+/, '' )
+    assert_equal( result, tree_string )
+    
+  end
+end