cql-ruby 0.7.1

data/lib/cql_ruby/cql_lexer.rb

@@ -0,0 +1,221 @@
+module CqlRuby
+  
+# This is derived from java based a semi-trivial subclass for java.io.StreamTokenizer that:
+#  * Includes a render() method
+#  * Knows about the multi-character tokens "<=", ">=" and "<>"
+#  * Recognises a set of keywords as tokens in their own right
+#  * Includes some primitive debugging-output facilities
+# It's used only by CQLParser
+
+class CqlLexer
+  attr_accessor :tokenizer, :simple_tokens, :index, :token_type, :value
+
+  TT_EOF        = 1
+  TT_EOL        = 2
+  TT_NUMBER     = 3
+  TT_WORD       = 4
+  TT_LE        = 1000   # The "<=" relation
+  TT_GE        = 1001   # The ">=" relation
+  TT_NE        = 1002   # The "<>" relation
+  TT_EQEQ      = 1003   # The "==" relation
+  TT_AND       = 1004   # The "and" boolean
+  TT_OR        = 1005   # The "or" boolean
+  TT_NOT       = 1006   # The "not" boolean
+  TT_PROX      = 1007   # The "prox" boolean
+  TT_SORTBY     = 1008  # The "sortby" operator
+  
+  @@keywords = { "and" => CqlLexer::TT_AND, "or" => CqlLexer::TT_OR, "not" => CqlLexer::TT_NOT,
+                 "prox" => CqlLexer::TT_PROX, "sortby" => CqlLexer::TT_SORTBY}
+  @@symbol_tokens = @@keywords.values + [CqlLexer::TT_WORD,CqlLexer::TT_NUMBER,'"']
+
+  @@ordinary_chars = /[=<>\/()]/
+  @@re_any_token_start = /[\w(\)<>\/="*]/
+  @@re_string_end = /[ \t(\)<>\/="]/
+  
+  def initialize( s="", debug=false )
+    debug
+    @simple_tokens = tokenize( s )
+    @index = -1
+  end
+  
+  def CqlLexer.symbol_tokens
+    @@symbol_tokens
+  end
+  
+  def find_string_end( s, position )
+    s.index( @@re_string_end, position) || s.length
+  end
+  
+  def find_quoted_string_end( s, position )
+    is_backslashed = false
+    for i in position+1..s.length
+      if s[i..i] == '\\'
+        is_backslashed = ! is_backslashed
+      else
+        if s[i..i] == '"' and not is_backslashed
+          return i + 1
+        end
+        is_backslashed = false
+      end
+    end 
+    raise CqlException,  "unterminated quoted string at position #{position} in '#{s}'"
+    return s.length
+  end
+  
+  def tokenize( cql_string )
+    position = 0
+    previous_backslash = false
+    length = cql_string.length
+    
+    @tokens = []
+    while position < length
+      token_begin = cql_string.index( @@re_any_token_start, position )
+      break unless token_begin
+      
+
+      case cql_string[token_begin..token_begin]
+        when @@ordinary_chars
+          token_end = token_begin+1
+        when /[\w*]/
+          token_end = find_string_end( cql_string, token_begin )
+        when '"'
+          token_end = find_quoted_string_end( cql_string, token_begin )
+        else
+          token_end = token_begin+1
+      end
+      
+      @tokens << cql_string[token_begin..token_end-1]
+      position = token_end
+    end
+    
+    # puts "tokens=#{@tokens.inspect}"
+    @tokens
+  end
+  
+  def next_token
+    underlying_next_token
+    return CqlLexer::TT_EOF if eof? or eol?
+    
+    if @token_type == '<'
+      underlying_next_token
+      if @token_type == '=' 
+        @token_type = CqlLexer::TT_LE
+      elsif @token_type == ">"
+        @token_type = CqlLexer::TT_NE
+      else
+        push_back
+        @token_type = '<'
+      end
+    elsif @token_type == '>'
+      underlying_next_token
+      if @token_type == '=' 
+        @token_type = CqlLexer::TT_GE
+      else
+        push_back
+        @token_type = '>'
+      end
+    elsif @token_type == '='
+      underlying_next_token
+      if @token_type == '=' 
+        @token_type = CqlLexer::TT_EQEQ
+      else
+        push_back
+        @token_type = '='
+      end
+    end
+    @token_type
+  end
+
+  def underlying_next_token
+    @index += 1
+    if @index >= @simple_tokens.length
+      @token_type = CqlLexer::TT_EOF
+      @value = nil
+      return
+    end
+    @value = @simple_tokens[ @index ]
+    if /[0-9]+/ =~ @value
+      @token_type = CqlLexer::TT_NUMBER
+    elsif @value.length > 1
+      if @value.slice(0..0) == '"'
+        @token_type = '"'
+        @value = @value.slice(1..-2)
+      else
+        @token_type = @@keywords[ @value.downcase ] || CqlLexer::TT_WORD
+      end
+    else
+      @token_type = @value
+    end
+  end
+  
+  def push_back
+    @index -= 1
+  end
+  
+  def render( token=nil, quote_chars=true )
+    token = @token_type unless token
+    case token
+      when CqlLexer::TT_EOF: "EOF"
+      when CqlLexer::TT_NUMBER: @value
+      when CqlLexer::TT_WORD: "word:#{@value}"
+      when "'": "string:\"#{@value}\""
+      when CqlLexer::TT_LE: "<="
+      when CqlLexer::TT_GE: ">="
+      when CqlLexer::TT_NE: "<>"
+      when CqlLexer::TT_EQEQ: "=="
+      when CqlLexer::TT_EOF: "EOF"
+    else
+      if @@keywords.has_value?( @value )
+        @value
+      else
+        if quote_chars 
+          "'#{token}'"
+        else
+          token
+        end
+      end
+    end
+  end
+  
+  def eof?
+    @token_type == CqlLexer::TT_EOF
+  end
+  def eol?
+    @token_type == CqlLexer::TT_EOL
+  end
+  def number?
+    @token_type == CqlLexer::TT_NUMBER
+  end
+  def word?
+    @token_type == CqlLexer::TT_WORD
+  end
+  def less_than_or_equal?
+    @token_type == CqlLexer::TT_LE
+  end
+  def greater_than_or_equal?
+    @token_type == CqlLexer::TT_GE
+  end
+  def double_equal?
+    @token_type == CqlLexer::TT_EQEQ
+  end
+  def not_equal?
+    @token_type == CqlLexer::TT_NE
+  end
+  def not?
+    @token_type == CqlLexer::TT_NOT
+  end
+  def and?
+    @token_type == CqlLexer::TT_AND
+  end
+  def or?
+    @token_type == CqlLexer::TT_OR
+  end
+    def prox?
+    @token_type == CqlLexer::TT_PROX
+  end
+  def sortby?
+    @token_type == CqlLexer::TT_SORTBY
+  end
+
+end 
+end

data/lib/cql_ruby/cql_nodes.rb

@@ -0,0 +1,328 @@
+module CqlRuby
+  
+class CqlNode
+  require 'rubygems'
+  require 'builder'
+  def check_xml( xml )
+    xml = Builder::XmlMarkup.new(:indent => 1) unless xml
+#    (0..1).each {|x| puts x}
+    xml
+  end
+  
+  def initialize
+    
+  end
+  
+  def getResultSetName
+    nil
+  end
+  
+  # arguments kept for symmetry
+  def to_xcql( xml=nil, prefixes=nil, sortkeys=nil )
+    nil
+  end
+
+  def to_cql
+    nil
+  end
+  
+  def render_prefixes( xml=nil, prefixes=nil )
+    return unless prefixes and prefixes.length > 0 
+    
+    xml = check_xml( xml )
+    xml.prefixes do
+      prefixes.each do |prefix|
+        xml.prefix do
+          xml.name( prefix.name )
+          xml.identifier( prefix.identifier )
+        end
+      end
+    end
+  end
+  
+  def render_sortkeys( xml=nil, sortkeys=nil )
+    return "" unless sortkeys and sortkeys.any?
+    
+    xml = check_xml( xml )
+    xml.sortKeys do
+      sortkeys.each do |key|
+        key.sortkey_to_xcql( xml )
+      end
+    end
+  end
+end
+
+class CqlPrefix
+  attr_accessor :name, :identifier
+  
+  def initialize( name, identifier )
+    super()
+    @name = name
+    @identifier = identifier
+  end
+end
+
+class CqlPrefixNode < CqlNode
+  attr_accessor :prefix, :subtree
+  def initialize( name=nil, identifier=nil, new_subtree=nil )
+    super()
+    @prefix = CqlPrefix.new( name, identifier )
+    @subtree = new_subtree
+  end
+  
+  def to_cql
+    if @prefix.name 
+      ">#{@prefix.name}=\"#{@prefix.identifier}\" (#{@subtree.to_cql})"
+    else
+      ">\"#{@prefix.identifier}\" (#{@subtree.to_cql})"
+    end
+  end
+
+  def to_xcql( xml=nil, prefixes=nil, sortkeys=nil )
+    xml = check_xml( xml )
+    tmp = []
+    tmp = prefixes.dup if prefixes
+    tmp << @prefix
+    
+    @subtree.to_xcql( xml, tmp, sortkeys )
+  end
+end
+
+class Modifier < CqlNode
+  attr_accessor :type, :comparison, :value
+  
+  def initialize( new_type, new_comparison=nil, new_value=nil)
+    super()
+    @type = new_type
+    @comparison = new_comparison
+    @value = new_value
+  end
+  
+  def to_cql
+    res = "#{@type}"
+    res << " #{@comparison} #{@value}" if @value
+    res
+  end
+  
+  def to_xcql( xml=nil, relation_element="unknown_relation" )
+    xml = check_xml( xml )
+    xml.modifier do 
+      xml.type( @type )
+      if( @value )
+        xml.tag!( relation_element, @comparison ) if @comparison
+        xml.value( @value )
+      end
+    end
+  end
+end
+
+class ModifierSet < CqlNode
+  attr_accessor :base, :modifiers
+  
+  def initialize( base )
+    # super
+    @base = base.dup
+    @modifiers = []
+  end
+  
+  def add_modifier( type, comparison=nil, value=nil)
+    modifiers << Modifier.new( type, comparison, value )
+  end
+  
+  def to_cql
+    res = @base.dup
+    @modifiers.each do |m| 
+      res << "/#{m.to_cql}"
+    end
+    res
+  end
+  
+  def to_xcql( xml=nil, top_level_element="unknowElement" )
+    xml = check_xml( xml )
+    underlying_to_xcql( xml, top_level_element, "value" )
+  end
+  
+  def sortkey_to_xcql( xml=nil )
+    xml = check_xml( xml )
+    underlying_to_xcql( xml, "key", "index" )
+  end
+  
+  def underlying_to_xcql( xml, top_level_element, value_element )
+    xml.tag!( top_level_element ) do
+      xml.tag!( value_element, @base )
+      if @modifiers.any?
+        xml.modifiers do
+          @modifiers.each { |modifier| modifier.to_xcql( xml, "comparison" ) }
+        end
+      end
+    end
+  end
+end
+
+class CqlBooleanNode < CqlNode
+  attr_accessor :left_node, :right_node, :modifier_set
+  
+  def initialize( left_node, right_node, modifier_set )
+    super()
+    @left_node = left_node.dup
+    @right_node = right_node.dup
+    @modifier_set = modifier_set || ModifierSet.new( "=" )
+  end
+  
+  def to_xcql( xml=nil, prefixes=nil, sortkeys=nil )
+    xml = check_xml( xml )
+    xml.triple do
+      render_prefixes( xml, prefixes )
+      modifier_set.to_xcql( xml, "boolean" )
+      xml.leftOperand do
+        left_node.to_xcql( xml )
+      end
+      xml.rightOperand do
+        right_node.to_xcql( xml )
+      end
+      render_sortkeys( xml, sortkeys )
+    end
+  end
+
+  def to_cql
+    "(#{@left_node.to_cql}) #{@modifier_set.to_cql} (#{@right_node.to_cql})"
+  end
+end
+
+
+ # Represents a terminal node in a CQL parse-tree.
+ # A term node consists of the term String itself, together with,
+ # optionally, an index string and a relation.  Neither or both of
+ # these must be provided - you can't have an index without a
+ # relation or vice versa.
+
+class CqlTermNode < CqlNode
+  attr_accessor :index, :relation, :term
+  def initialize( index, relation, term )
+    super()
+    
+    @index = index.dup
+    @relation = relation.dup
+    @term = term.dup
+  end
+  
+  def result_set_index?( qual )
+    /(srw|cql).resultSet(|Id|Name|SetName)/ =~ qual
+  end
+  
+  def result_set_name
+    return term if result_set_index?( @index )
+    nil
+  end
+  
+  def to_xcql( xml=nil, prefixes=nil, sortkeys=nil )
+    xml = check_xml( xml )
+    
+    xml.searchClause do
+      render_prefixes( xml, prefixes )
+      xml.index( @index )
+      @relation.to_xcql( xml )
+      xml.term( @term )
+      render_sortkeys( xml, sortkeys )
+    end
+  end
+  
+  def to_cql
+    quoted_index = maybe_quote( @index )
+    quoted_term = maybe_quote( @term )
+    res = quoted_term
+    
+    if @index && /(srw|cql)\.serverChoice/i !~ @index
+      res = "#{quoted_index} #{@relation.to_cql} #{quoted_term}"
+    end
+    res
+  end
+  
+  def maybe_quote( s )
+    if s == "" || s =~ /[" \t=<>()\/]/
+      return "\"#{s.gsub( /"/, "\\\"" )}\""
+    end
+    s
+  end
+end
+
+class CqlAndNode < CqlBooleanNode
+  def initialize( left_node, right_node, modifier_set )
+    super( left_node, right_node, modifier_set )
+  end
+end
+class CqlOrNode < CqlBooleanNode
+  def initialize( left_node, right_node, modifier_set )
+    super( left_node, right_node, modifier_set )
+  end
+end
+class CqlNotNode < CqlBooleanNode
+  def initialize( left_node, right_node, modifier_set )
+    super( left_node, right_node, modifier_set )
+  end
+end
+class CqlProxNode < CqlBooleanNode
+  def initialize( left_node, right_node, modifier_set )
+    super( left_node, right_node, modifier_set )
+  end
+end
+
+class CqlRelation < CqlNode
+  attr_accessor :modifier_set
+  
+  def initialize( base )
+    super()
+    @modifier_set = ModifierSet.new( base )
+  end
+  
+  def set_modifiers( ms )
+    @modifier_set = ms
+  end
+  
+  def to_xcql( xml=nil, prefixes=nil, sortkeys=nil )
+    raise CqlException,  "CqlRelation.to_xcql called with no relation" if sortkeys 
+    xml = check_xml( xml )
+    @modifier_set.to_xcql( xml, "relation" )
+  end
+  
+  def to_cql
+    @modifier_set.to_cql()
+  end
+end
+
+class CqlSortNode < CqlNode
+  attr_accessor :subtree, :keys
+  
+  def initialize( subtree )
+    super()
+    @subtree = subtree
+    @keys = []
+  end
+  
+  def add_sort_index( key )
+    @keys << key
+  end
+  
+  def to_xcql( xml=nil, prefixes=nil, sortkeys=nil )
+    raise CqlException,  "CqlSortNode.to_xcql called with sortkeys" if sortkeys and sortkeys.any?
+
+    xml = check_xml( xml )
+    return @subtree.to_xcql( xml, prefixes, @keys )
+  end
+  
+  def to_cql
+    buf = @subtree.to_cql
+    if @keys.any?
+      buf << " sortby "
+      @keys.each do |key|
+        buf << " #{key.to_cql}"
+      end
+    end
+    buf
+  end
+end
+
+class CqlException < RuntimeError
+end
+
+end