sparkql 0.3.20 → 0.3.21

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZjQ1YjRjYWNmMmE1ODNiMDY3NmZmODQ2ZjYwYWMyOTZlNDc1ODFmMg==
+    NWY3NjdhZWI4ZDViNDE4YTJhYmI1YTI1YzE0YzExYzRhMTVlNzFiMQ==
   data.tar.gz: !binary |-
-    ZWZkMWE2NmIzYzM0YjAwYzUyYjM0NDBiYWVlZTc0ZmQ1MGQ0MDlhZQ==
+    Y2Y1OTQwNjQ2NWQzYWMwMjI2YmEwZDU4NzgyMjNjN2FhMDllODI4NQ==
 SHA512:
   metadata.gz: !binary |-
-    MWFhZDQ5MTdiOWNmMzVlMWM2MDBkYzdmMjU0OGYwOGUzZTQ0ZmExYjUxY2Ex
-    ZWFlOTQwM2FiMGMxMGYwNzJjYjU4NDdkNmNlMDFiMjg0M2VmMTRhZTM4OWU0
-    ZjE4Yjc0YjA4ZjczMjlhYzA5MDEyZGMxNzliMThlM2U4YTFlMGI=
+    N2IwY2IyMzQwNmViODZiMzQxOTM3MGZkNDMwNTI2YzNjYjQ4N2E3MWZkZTBl
+    MmMwNjQ5MGI5MTdmYTQ3Y2Y3YTEwNTIyYmFlOTAxNWIwNzI4NDZlYjk2NDU3
+    YTZhYjFiMTk0ZDE0Y2MxNGNiN2ExZTQxY2Y0NzYzZDMxYTJkNTA=
   data.tar.gz: !binary |-
-    OWI1MWY5NDkxYWQ1NDg3NTcyZGUxODU0ZTVhOWE3YjM2NzQyMjFiOTUxMGRh
-    MTQwN2IzMjIxOTU1MGY0ZmMzY2IxYjc3ZTkwMzlhYzFlZjBmZjNiNWNjOTUx
-    ZjY4NDQyOWViOGM3Y2RmYmY5N2MzOTY0YTRlNTA3MDkwNWEyODg=
+    MWVkMDNhYzNmOTk2ZGNmYzg2ODcwOWRmMWVkNGQzYTM4ZDQ2NjY5NmQ5YzQ2
+    NjZhZTU1ZWFhMjlhZTEzNWYzNGQ5Y2MzY2RiNjY4Yzc2Nzk0NmYyNGQ0ZWNk
+    MGM2NjYyN2U2ZGZkMmExMzI0MTY4YTU4MWQ4NzE0ZWFiN2Y4ZTE=

data/.gitignore

@@ -5,3 +5,4 @@ lib/sparkql/*.output
 pkg/*
 *.swp
 test/reports
+vendor/bundle

data/CHANGELOG.md

@@ -1,5 +1,9 @@
+v0.3.21, 2015-09-24 ([changes](https://github.com/sparkapi/sparkql/compare/v0.3.20...v0.3.21))
+-------------------
+
+  * [IMPROVEMENT] Record token index and current token in lexer, for error reporting
 
-v0.3.2, 2015-04-14 ([changes](https://github.com/sparkapi/sparkql/compare/v0.3.18...v0.3.20))
+v0.3.20, 2015-04-14 ([changes](https://github.com/sparkapi/sparkql/compare/v0.3.18...v0.3.20))
 -------------------
 
   * [BUGFIX] Allow seconds for ISO-8601

data/VERSION

@@ -1 +1 @@
-0.3.20
+0.3.21

data/lib/sparkql/errors.rb

@@ -38,11 +38,12 @@ class ErrorsProcessor
 end
 
 class ParserError
-  attr_accessor :token, :expression, :message, :status, :recovered_as
+  attr_accessor :token, :token_index, :expression, :message, :status, :recovered_as
   attr_writer :syntax, :constraint
 
   def initialize(error_hash={})
     @token = error_hash[:token]
+    @token_index = error_hash[:token_index]
     @expression = error_hash[:expression]
     @message = error_hash[:message]
     @status = error_hash[:status]
@@ -80,4 +81,4 @@ class ParserError
   end
 end
 
-end
+end

data/lib/sparkql/lexer.rb

@@ -3,7 +3,7 @@ class Sparkql::Lexer < StringScanner
   
   attr_accessor :level, :block_group_identifier
   
-  attr_reader :last_field
+  attr_reader :last_field, :current_token_value, :token_index
 
   def initialize(str)
     str.freeze
@@ -18,43 +18,45 @@ class Sparkql::Lexer < StringScanner
   # TODO the old implementation did value type detection conversion at a later date, we can perform
   # this at parse time if we want!!!!
   def shift
+    @token_index = self.pos
+
     token = case
-      when value = scan(SPACE)
-        [:SPACE, value]
-      when value = scan(LPAREN)
+      when @current_token_value = scan(SPACE)
+        [:SPACE, @current_token_value]
+      when @current_token_value = scan(LPAREN)
         levelup
-        [:LPAREN, value]
-      when value = scan(RPAREN)
+        [:LPAREN, @current_token_value]
+      when @current_token_value = scan(RPAREN)
         # leveldown: do this after parsing group
-        [:RPAREN, value]
-      when value = scan(/\,/)
-        [:COMMA,value]
-      when value = scan(NULL)
+        [:RPAREN, @current_token_value]
+      when @current_token_value = scan(/\,/)
+        [:COMMA,@current_token_value]
+      when @current_token_value = scan(NULL)
         literal :NULL, "NULL"
-      when value = scan(STANDARD_FIELD)
-        check_standard_fields(value)
-      when value = scan(DATETIME)
-        literal :DATETIME, value
-      when value = scan(DATE)
-        literal :DATE, value
-      when value = scan(DECIMAL)
-        literal :DECIMAL, value
-      when value = scan(INTEGER)
-        literal :INTEGER, value
-      when value = scan(CHARACTER)
-        literal :CHARACTER, value
-      when value = scan(BOOLEAN)
-        literal :BOOLEAN, value
-      when value = scan(KEYWORD)
-        check_keywords(value)
-      when value = scan(CUSTOM_FIELD)
-        [:CUSTOM_FIELD,value]
+      when @current_token_value = scan(STANDARD_FIELD)
+        check_standard_fields(@current_token_value)
+      when @current_token_value = scan(DATETIME)
+        literal :DATETIME, @current_token_value
+      when @current_token_value = scan(DATE)
+        literal :DATE, @current_token_value
+      when @current_token_value = scan(DECIMAL)
+        literal :DECIMAL, @current_token_value
+      when @current_token_value = scan(INTEGER)
+        literal :INTEGER, @current_token_value
+      when @current_token_value = scan(CHARACTER)
+        literal :CHARACTER, @current_token_value
+      when @current_token_value = scan(BOOLEAN)
+        literal :BOOLEAN, @current_token_value
+      when @current_token_value = scan(KEYWORD)
+        check_keywords(@current_token_value)
+      when @current_token_value = scan(CUSTOM_FIELD)
+        [:CUSTOM_FIELD,@current_token_value]
       when empty?
         [false, false] # end of file, \Z don't work with StringScanner
       else
         [:UNKNOWN, "ERROR: '#{self.string}'"]
     end
-    #value.freeze
+
     token.freeze
   end
   

data/lib/sparkql/parser_compatibility.rb

@@ -193,6 +193,11 @@ module Sparkql::ParserCompatibility
   private
 
   def tokenizer_error( error_hash )
+
+    if @lexer
+      error_hash[:token_index] = @lexer.token_index
+    end
+
     self.errors << Sparkql::ParserError.new( error_hash )
   end
   alias :compile_error :tokenizer_error

data/lib/sparkql/parser_tools.rb

@@ -118,7 +118,7 @@ module Sparkql::ParserTools
     token_name = token_to_str(error_token_id)
     token_name.downcase!
     token = error_value.to_s.inspect
-    tokenizer_error(:token => @lexer.last_field, 
+    tokenizer_error(:token => @lexer.current_token_value, 
                     :message => "Error parsing token #{token_name}",
                     :status => :fatal, 
                     :syntax => true)    

data/test/unit/lexer_test.rb

@@ -2,6 +2,21 @@ require 'test_helper'
 
 class LexerTest < Test::Unit::TestCase
   include Sparkql
+
+  test "record the current token and current oken position" do
+      @lexer = Lexer.new "City Eq 'Fargo'"
+      token = @lexer.shift
+      assert_equal "City", @lexer.current_token_value
+      assert_equal 0, @lexer.token_index
+
+      token = @lexer.shift
+      assert_equal " ", @lexer.current_token_value
+      assert_equal 4, @lexer.token_index
+
+      token = @lexer.shift
+      assert_equal "Eq", @lexer.current_token_value
+      assert_equal 5, @lexer.token_index
+  end
   
   def test_check_reserved_words_standard_fields
     ["OrOrOr Eq true", "Equador Eq true", "Oregon Ge 10"].each do |standard_field|

data/test/unit/parser_compatability_test.rb

@@ -241,9 +241,19 @@ class ParserCompatabilityTest < Test::Unit::TestCase
       expressions = parser.tokenize( f )
       assert_nil expressions, "filter: #{f}"
       assert parser.errors?
+      error = parser.errors.first
     end
   end
 
+  test "report token index on error" do
+    parser = Parser.new
+    expressions = parser.tokenize( "MlsStatus 2eq 'Active'" )
+    error = parser.errors.first
+
+    assert_equal "2", error.token
+    assert_equal 10, error.token_index
+  end
+
   test "tokenize edge case string value" do
     good_strings = ["'Fargo\\'s Boat'", "'Fargo'", "'Fargo\\'\\'s'",
       "' Fargo '", " 'Fargo' "]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sparkql
 version: !ruby/object:Gem::Version
-  version: 0.3.20
+  version: 0.3.21
 platform: ruby
 authors:
 - Wade McEwen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-14 00:00:00.000000000 Z
+date: 2015-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: georuby