sparkql 0.3.20 → 0.3.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/.gitignore +1 -0
- data/CHANGELOG.md +5 -1
- data/VERSION +1 -1
- data/lib/sparkql/errors.rb +3 -2
- data/lib/sparkql/lexer.rb +31 -29
- data/lib/sparkql/parser_compatibility.rb +5 -0
- data/lib/sparkql/parser_tools.rb +1 -1
- data/test/unit/lexer_test.rb +15 -0
- data/test/unit/parser_compatability_test.rb +10 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
NWY3NjdhZWI4ZDViNDE4YTJhYmI1YTI1YzE0YzExYzRhMTVlNzFiMQ==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
Y2Y1OTQwNjQ2NWQzYWMwMjI2YmEwZDU4NzgyMjNjN2FhMDllODI4NQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
N2IwY2IyMzQwNmViODZiMzQxOTM3MGZkNDMwNTI2YzNjYjQ4N2E3MWZkZTBl
|
10
|
+
MmMwNjQ5MGI5MTdmYTQ3Y2Y3YTEwNTIyYmFlOTAxNWIwNzI4NDZlYjk2NDU3
|
11
|
+
YTZhYjFiMTk0ZDE0Y2MxNGNiN2ExZTQxY2Y0NzYzZDMxYTJkNTA=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
MWVkMDNhYzNmOTk2ZGNmYzg2ODcwOWRmMWVkNGQzYTM4ZDQ2NjY5NmQ5YzQ2
|
14
|
+
NjZhZTU1ZWFhMjlhZTEzNWYzNGQ5Y2MzY2RiNjY4Yzc2Nzk0NmYyNGQ0ZWNk
|
15
|
+
MGM2NjYyN2U2ZGZkMmExMzI0MTY4YTU4MWQ4NzE0ZWFiN2Y4ZTE=
|
data/.gitignore
CHANGED
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,9 @@
|
|
1
|
+
v0.3.21, 2015-09-24 ([changes](https://github.com/sparkapi/sparkql/compare/v0.3.20...v0.3.21))
|
2
|
+
-------------------
|
3
|
+
|
4
|
+
* [IMPROVEMENT] Record token index and current token in lexer, for error reporting
|
1
5
|
|
2
|
-
v0.3.
|
6
|
+
v0.3.20, 2015-04-14 ([changes](https://github.com/sparkapi/sparkql/compare/v0.3.18...v0.3.20))
|
3
7
|
-------------------
|
4
8
|
|
5
9
|
* [BUGFIX] Allow seconds for ISO-8601
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.3.
|
1
|
+
0.3.21
|
data/lib/sparkql/errors.rb
CHANGED
@@ -38,11 +38,12 @@ class ErrorsProcessor
|
|
38
38
|
end
|
39
39
|
|
40
40
|
class ParserError
|
41
|
-
attr_accessor :token, :expression, :message, :status, :recovered_as
|
41
|
+
attr_accessor :token, :token_index, :expression, :message, :status, :recovered_as
|
42
42
|
attr_writer :syntax, :constraint
|
43
43
|
|
44
44
|
def initialize(error_hash={})
|
45
45
|
@token = error_hash[:token]
|
46
|
+
@token_index = error_hash[:token_index]
|
46
47
|
@expression = error_hash[:expression]
|
47
48
|
@message = error_hash[:message]
|
48
49
|
@status = error_hash[:status]
|
@@ -80,4 +81,4 @@ class ParserError
|
|
80
81
|
end
|
81
82
|
end
|
82
83
|
|
83
|
-
end
|
84
|
+
end
|
data/lib/sparkql/lexer.rb
CHANGED
@@ -3,7 +3,7 @@ class Sparkql::Lexer < StringScanner
|
|
3
3
|
|
4
4
|
attr_accessor :level, :block_group_identifier
|
5
5
|
|
6
|
-
attr_reader :last_field
|
6
|
+
attr_reader :last_field, :current_token_value, :token_index
|
7
7
|
|
8
8
|
def initialize(str)
|
9
9
|
str.freeze
|
@@ -18,43 +18,45 @@ class Sparkql::Lexer < StringScanner
|
|
18
18
|
# TODO the old implementation did value type detection conversion at a later date, we can perform
|
19
19
|
# this at parse time if we want!!!!
|
20
20
|
def shift
|
21
|
+
@token_index = self.pos
|
22
|
+
|
21
23
|
token = case
|
22
|
-
when
|
23
|
-
[:SPACE,
|
24
|
-
when
|
24
|
+
when @current_token_value = scan(SPACE)
|
25
|
+
[:SPACE, @current_token_value]
|
26
|
+
when @current_token_value = scan(LPAREN)
|
25
27
|
levelup
|
26
|
-
[:LPAREN,
|
27
|
-
when
|
28
|
+
[:LPAREN, @current_token_value]
|
29
|
+
when @current_token_value = scan(RPAREN)
|
28
30
|
# leveldown: do this after parsing group
|
29
|
-
[:RPAREN,
|
30
|
-
when
|
31
|
-
[:COMMA
|
32
|
-
when
|
31
|
+
[:RPAREN, @current_token_value]
|
32
|
+
when @current_token_value = scan(/\,/)
|
33
|
+
[:COMMA,@current_token_value]
|
34
|
+
when @current_token_value = scan(NULL)
|
33
35
|
literal :NULL, "NULL"
|
34
|
-
when
|
35
|
-
check_standard_fields(
|
36
|
-
when
|
37
|
-
literal :DATETIME,
|
38
|
-
when
|
39
|
-
literal :DATE,
|
40
|
-
when
|
41
|
-
literal :DECIMAL,
|
42
|
-
when
|
43
|
-
literal :INTEGER,
|
44
|
-
when
|
45
|
-
literal :CHARACTER,
|
46
|
-
when
|
47
|
-
literal :BOOLEAN,
|
48
|
-
when
|
49
|
-
check_keywords(
|
50
|
-
when
|
51
|
-
[:CUSTOM_FIELD
|
36
|
+
when @current_token_value = scan(STANDARD_FIELD)
|
37
|
+
check_standard_fields(@current_token_value)
|
38
|
+
when @current_token_value = scan(DATETIME)
|
39
|
+
literal :DATETIME, @current_token_value
|
40
|
+
when @current_token_value = scan(DATE)
|
41
|
+
literal :DATE, @current_token_value
|
42
|
+
when @current_token_value = scan(DECIMAL)
|
43
|
+
literal :DECIMAL, @current_token_value
|
44
|
+
when @current_token_value = scan(INTEGER)
|
45
|
+
literal :INTEGER, @current_token_value
|
46
|
+
when @current_token_value = scan(CHARACTER)
|
47
|
+
literal :CHARACTER, @current_token_value
|
48
|
+
when @current_token_value = scan(BOOLEAN)
|
49
|
+
literal :BOOLEAN, @current_token_value
|
50
|
+
when @current_token_value = scan(KEYWORD)
|
51
|
+
check_keywords(@current_token_value)
|
52
|
+
when @current_token_value = scan(CUSTOM_FIELD)
|
53
|
+
[:CUSTOM_FIELD,@current_token_value]
|
52
54
|
when empty?
|
53
55
|
[false, false] # end of file, \Z don't work with StringScanner
|
54
56
|
else
|
55
57
|
[:UNKNOWN, "ERROR: '#{self.string}'"]
|
56
58
|
end
|
57
|
-
|
59
|
+
|
58
60
|
token.freeze
|
59
61
|
end
|
60
62
|
|
@@ -193,6 +193,11 @@ module Sparkql::ParserCompatibility
|
|
193
193
|
private
|
194
194
|
|
195
195
|
def tokenizer_error( error_hash )
|
196
|
+
|
197
|
+
if @lexer
|
198
|
+
error_hash[:token_index] = @lexer.token_index
|
199
|
+
end
|
200
|
+
|
196
201
|
self.errors << Sparkql::ParserError.new( error_hash )
|
197
202
|
end
|
198
203
|
alias :compile_error :tokenizer_error
|
data/lib/sparkql/parser_tools.rb
CHANGED
@@ -118,7 +118,7 @@ module Sparkql::ParserTools
|
|
118
118
|
token_name = token_to_str(error_token_id)
|
119
119
|
token_name.downcase!
|
120
120
|
token = error_value.to_s.inspect
|
121
|
-
tokenizer_error(:token => @lexer.
|
121
|
+
tokenizer_error(:token => @lexer.current_token_value,
|
122
122
|
:message => "Error parsing token #{token_name}",
|
123
123
|
:status => :fatal,
|
124
124
|
:syntax => true)
|
data/test/unit/lexer_test.rb
CHANGED
@@ -2,6 +2,21 @@ require 'test_helper'
|
|
2
2
|
|
3
3
|
class LexerTest < Test::Unit::TestCase
|
4
4
|
include Sparkql
|
5
|
+
|
6
|
+
test "record the current token and current oken position" do
|
7
|
+
@lexer = Lexer.new "City Eq 'Fargo'"
|
8
|
+
token = @lexer.shift
|
9
|
+
assert_equal "City", @lexer.current_token_value
|
10
|
+
assert_equal 0, @lexer.token_index
|
11
|
+
|
12
|
+
token = @lexer.shift
|
13
|
+
assert_equal " ", @lexer.current_token_value
|
14
|
+
assert_equal 4, @lexer.token_index
|
15
|
+
|
16
|
+
token = @lexer.shift
|
17
|
+
assert_equal "Eq", @lexer.current_token_value
|
18
|
+
assert_equal 5, @lexer.token_index
|
19
|
+
end
|
5
20
|
|
6
21
|
def test_check_reserved_words_standard_fields
|
7
22
|
["OrOrOr Eq true", "Equador Eq true", "Oregon Ge 10"].each do |standard_field|
|
@@ -241,9 +241,19 @@ class ParserCompatabilityTest < Test::Unit::TestCase
|
|
241
241
|
expressions = parser.tokenize( f )
|
242
242
|
assert_nil expressions, "filter: #{f}"
|
243
243
|
assert parser.errors?
|
244
|
+
error = parser.errors.first
|
244
245
|
end
|
245
246
|
end
|
246
247
|
|
248
|
+
test "report token index on error" do
|
249
|
+
parser = Parser.new
|
250
|
+
expressions = parser.tokenize( "MlsStatus 2eq 'Active'" )
|
251
|
+
error = parser.errors.first
|
252
|
+
|
253
|
+
assert_equal "2", error.token
|
254
|
+
assert_equal 10, error.token_index
|
255
|
+
end
|
256
|
+
|
247
257
|
test "tokenize edge case string value" do
|
248
258
|
good_strings = ["'Fargo\\'s Boat'", "'Fargo'", "'Fargo\\'\\'s'",
|
249
259
|
"' Fargo '", " 'Fargo' "]
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sparkql
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.21
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Wade McEwen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-09-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: georuby
|