yajl-ffi 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE +19 -0
- data/README.md +180 -0
- data/Rakefile +22 -0
- data/lib/yajl/ffi.rb +54 -0
- data/lib/yajl/ffi/builder.rb +70 -0
- data/lib/yajl/ffi/parser.rb +316 -0
- data/lib/yajl/ffi/tasks/benchmark.rake +61 -0
- data/lib/yajl/ffi/version.rb +5 -0
- data/spec/builder_spec.rb +155 -0
- data/spec/fixtures/repository.json +107 -0
- data/spec/parser_spec.rb +795 -0
- data/yajl-ffi.gemspec +22 -0
- metadata +87 -0
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
desc 'Compare parser performance'
|
|
2
|
+
task :benchmark do
|
|
3
|
+
$: << './lib'
|
|
4
|
+
require 'benchmark'
|
|
5
|
+
require 'json'
|
|
6
|
+
require 'json/stream'
|
|
7
|
+
require 'yajl'
|
|
8
|
+
require 'yajl/ffi'
|
|
9
|
+
require 'tempfile'
|
|
10
|
+
|
|
11
|
+
# Copy the JSON test document into a temporary file several thousand times
|
|
12
|
+
# to give us a decent sized text with which to benchmark the parsers.
|
|
13
|
+
#
|
|
14
|
+
# Returns a File.
|
|
15
|
+
def generate
|
|
16
|
+
json = File.read('spec/fixtures/repository.json')
|
|
17
|
+
Tempfile.new('json-bench').tap do |file|
|
|
18
|
+
file.puts '['
|
|
19
|
+
1500.times do
|
|
20
|
+
file.puts json
|
|
21
|
+
file.puts ','
|
|
22
|
+
end
|
|
23
|
+
file.puts json
|
|
24
|
+
file.puts ']'
|
|
25
|
+
file.close
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Run the benchmark test against several JSON parsers.
|
|
30
|
+
#
|
|
31
|
+
# Returns nothing.
|
|
32
|
+
def benchmark
|
|
33
|
+
file = generate
|
|
34
|
+
Benchmark.bmbm do |x|
|
|
35
|
+
x.report('json') do
|
|
36
|
+
json = File.read(file.path)
|
|
37
|
+
JSON.parse(json)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
x.report('yajl-ruby') do
|
|
41
|
+
json = File.open(file.path)
|
|
42
|
+
Yajl::Parser.new.parse(json)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
x.report('yajl-ffi') do
|
|
46
|
+
json = File.open(file.path)
|
|
47
|
+
Yajl::FFI::Parser.parse(json)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
x.report('json-stream') do
|
|
51
|
+
json = File.open(file.path)
|
|
52
|
+
JSON::Stream::Parser.parse(json)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
ensure
|
|
56
|
+
file.unlink if file
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Run it.
|
|
60
|
+
benchmark
|
|
61
|
+
end
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
require 'yajl/ffi'
|
|
2
|
+
require 'minitest/autorun'
|
|
3
|
+
|
|
4
|
+
describe Yajl::FFI::Builder do
|
|
5
|
+
let(:parser) { Yajl::FFI::Parser.new }
|
|
6
|
+
subject { Yajl::FFI::Builder.new(parser) }
|
|
7
|
+
|
|
8
|
+
it 'builds a false value' do
|
|
9
|
+
assert_nil subject.result
|
|
10
|
+
subject.start_document
|
|
11
|
+
subject.value(false)
|
|
12
|
+
assert_nil subject.result
|
|
13
|
+
subject.end_document
|
|
14
|
+
assert_equal false, subject.result
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
it 'builds a string value' do
|
|
18
|
+
assert_nil subject.result
|
|
19
|
+
subject.start_document
|
|
20
|
+
subject.value("test")
|
|
21
|
+
assert_nil subject.result
|
|
22
|
+
subject.end_document
|
|
23
|
+
assert_equal "test", subject.result
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
it 'builds an empty array' do
|
|
27
|
+
assert_nil subject.result
|
|
28
|
+
subject.start_document
|
|
29
|
+
subject.start_array
|
|
30
|
+
subject.end_array
|
|
31
|
+
assert_nil subject.result
|
|
32
|
+
subject.end_document
|
|
33
|
+
assert_equal [], subject.result
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
it 'builds an array of numbers' do
|
|
37
|
+
subject.start_document
|
|
38
|
+
subject.start_array
|
|
39
|
+
subject.value(1)
|
|
40
|
+
subject.value(2)
|
|
41
|
+
subject.value(3)
|
|
42
|
+
subject.end_array
|
|
43
|
+
subject.end_document
|
|
44
|
+
assert_equal [1, 2, 3], subject.result
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
it 'builds nested empty arrays' do
|
|
48
|
+
subject.start_document
|
|
49
|
+
subject.start_array
|
|
50
|
+
subject.start_array
|
|
51
|
+
subject.end_array
|
|
52
|
+
subject.end_array
|
|
53
|
+
subject.end_document
|
|
54
|
+
assert_equal [[]], subject.result
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
it 'builds nested arrays of numbers' do
|
|
58
|
+
subject.start_document
|
|
59
|
+
subject.start_array
|
|
60
|
+
subject.value(1)
|
|
61
|
+
subject.start_array
|
|
62
|
+
subject.value(2)
|
|
63
|
+
subject.end_array
|
|
64
|
+
subject.value(3)
|
|
65
|
+
subject.end_array
|
|
66
|
+
subject.end_document
|
|
67
|
+
assert_equal [1, [2], 3], subject.result
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
it 'builds an empty object' do
|
|
71
|
+
subject.start_document
|
|
72
|
+
subject.start_object
|
|
73
|
+
subject.end_object
|
|
74
|
+
subject.end_document
|
|
75
|
+
assert_equal({}, subject.result)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
it 'builds a complex object' do
|
|
79
|
+
subject.start_document
|
|
80
|
+
subject.start_object
|
|
81
|
+
subject.key("k1")
|
|
82
|
+
subject.value(1)
|
|
83
|
+
subject.key("k2")
|
|
84
|
+
subject.value(nil)
|
|
85
|
+
subject.key("k3")
|
|
86
|
+
subject.value(true)
|
|
87
|
+
subject.key("k4")
|
|
88
|
+
subject.value(false)
|
|
89
|
+
subject.key("k5")
|
|
90
|
+
subject.value("string value")
|
|
91
|
+
subject.end_object
|
|
92
|
+
subject.end_document
|
|
93
|
+
expected = {
|
|
94
|
+
"k1" => 1,
|
|
95
|
+
"k2" => nil,
|
|
96
|
+
"k3" => true,
|
|
97
|
+
"k4" => false,
|
|
98
|
+
"k5" => "string value"
|
|
99
|
+
}
|
|
100
|
+
assert_equal expected, subject.result
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
it 'builds a nested object' do
|
|
104
|
+
subject.start_document
|
|
105
|
+
subject.start_object
|
|
106
|
+
subject.key("k1")
|
|
107
|
+
subject.value(1)
|
|
108
|
+
|
|
109
|
+
subject.key("k2")
|
|
110
|
+
subject.start_object
|
|
111
|
+
subject.end_object
|
|
112
|
+
|
|
113
|
+
subject.key("k3")
|
|
114
|
+
subject.start_object
|
|
115
|
+
subject.key("sub1")
|
|
116
|
+
subject.start_array
|
|
117
|
+
subject.value(12)
|
|
118
|
+
subject.end_array
|
|
119
|
+
subject.end_object
|
|
120
|
+
|
|
121
|
+
subject.key("k4")
|
|
122
|
+
subject.start_array
|
|
123
|
+
subject.value(1)
|
|
124
|
+
subject.start_object
|
|
125
|
+
subject.key("sub2")
|
|
126
|
+
subject.start_array
|
|
127
|
+
subject.value(nil)
|
|
128
|
+
subject.end_array
|
|
129
|
+
subject.end_object
|
|
130
|
+
subject.end_array
|
|
131
|
+
|
|
132
|
+
subject.key("k5")
|
|
133
|
+
subject.value("string value")
|
|
134
|
+
subject.end_object
|
|
135
|
+
subject.end_document
|
|
136
|
+
expected = {
|
|
137
|
+
"k1" => 1,
|
|
138
|
+
"k2" => {},
|
|
139
|
+
"k3" => {"sub1" => [12]},
|
|
140
|
+
"k4" => [1, {"sub2" => [nil]}],
|
|
141
|
+
"k5" => "string value"
|
|
142
|
+
}
|
|
143
|
+
assert_equal expected, subject.result
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
it 'builds a real document' do
|
|
147
|
+
refute_nil subject
|
|
148
|
+
parser << File.read('spec/fixtures/repository.json')
|
|
149
|
+
refute_nil subject.result
|
|
150
|
+
assert_equal 'rails', subject.result['name']
|
|
151
|
+
assert_equal 4223, subject.result['owner']['id']
|
|
152
|
+
assert_equal false, subject.result['fork']
|
|
153
|
+
assert_equal nil, subject.result['mirror_url']
|
|
154
|
+
end
|
|
155
|
+
end
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": 8514,
|
|
3
|
+
"name": "rails",
|
|
4
|
+
"full_name": "rails/rails",
|
|
5
|
+
"owner": {
|
|
6
|
+
"login": "rails",
|
|
7
|
+
"id": 4223,
|
|
8
|
+
"avatar_url": "https://avatars.githubusercontent.com/u/4223?",
|
|
9
|
+
"gravatar_id": "30f39a09e233e8369dddf6feb4be0308",
|
|
10
|
+
"url": "https://api.github.com/users/rails",
|
|
11
|
+
"html_url": "https://github.com/rails",
|
|
12
|
+
"followers_url": "https://api.github.com/users/rails/followers",
|
|
13
|
+
"following_url": "https://api.github.com/users/rails/following{/other_user}",
|
|
14
|
+
"gists_url": "https://api.github.com/users/rails/gists{/gist_id}",
|
|
15
|
+
"starred_url": "https://api.github.com/users/rails/starred{/owner}{/repo}",
|
|
16
|
+
"subscriptions_url": "https://api.github.com/users/rails/subscriptions",
|
|
17
|
+
"organizations_url": "https://api.github.com/users/rails/orgs",
|
|
18
|
+
"repos_url": "https://api.github.com/users/rails/repos",
|
|
19
|
+
"events_url": "https://api.github.com/users/rails/events{/privacy}",
|
|
20
|
+
"received_events_url": "https://api.github.com/users/rails/received_events",
|
|
21
|
+
"type": "Organization",
|
|
22
|
+
"site_admin": false
|
|
23
|
+
},
|
|
24
|
+
"private": false,
|
|
25
|
+
"html_url": "https://github.com/rails/rails",
|
|
26
|
+
"description": "Ruby on Rails",
|
|
27
|
+
"fork": false,
|
|
28
|
+
"url": "https://api.github.com/repos/rails/rails",
|
|
29
|
+
"forks_url": "https://api.github.com/repos/rails/rails/forks",
|
|
30
|
+
"keys_url": "https://api.github.com/repos/rails/rails/keys{/key_id}",
|
|
31
|
+
"collaborators_url": "https://api.github.com/repos/rails/rails/collaborators{/collaborator}",
|
|
32
|
+
"teams_url": "https://api.github.com/repos/rails/rails/teams",
|
|
33
|
+
"hooks_url": "https://api.github.com/repos/rails/rails/hooks",
|
|
34
|
+
"issue_events_url": "https://api.github.com/repos/rails/rails/issues/events{/number}",
|
|
35
|
+
"events_url": "https://api.github.com/repos/rails/rails/events",
|
|
36
|
+
"assignees_url": "https://api.github.com/repos/rails/rails/assignees{/user}",
|
|
37
|
+
"branches_url": "https://api.github.com/repos/rails/rails/branches{/branch}",
|
|
38
|
+
"tags_url": "https://api.github.com/repos/rails/rails/tags",
|
|
39
|
+
"blobs_url": "https://api.github.com/repos/rails/rails/git/blobs{/sha}",
|
|
40
|
+
"git_tags_url": "https://api.github.com/repos/rails/rails/git/tags{/sha}",
|
|
41
|
+
"git_refs_url": "https://api.github.com/repos/rails/rails/git/refs{/sha}",
|
|
42
|
+
"trees_url": "https://api.github.com/repos/rails/rails/git/trees{/sha}",
|
|
43
|
+
"statuses_url": "https://api.github.com/repos/rails/rails/statuses/{sha}",
|
|
44
|
+
"languages_url": "https://api.github.com/repos/rails/rails/languages",
|
|
45
|
+
"stargazers_url": "https://api.github.com/repos/rails/rails/stargazers",
|
|
46
|
+
"contributors_url": "https://api.github.com/repos/rails/rails/contributors",
|
|
47
|
+
"subscribers_url": "https://api.github.com/repos/rails/rails/subscribers",
|
|
48
|
+
"subscription_url": "https://api.github.com/repos/rails/rails/subscription",
|
|
49
|
+
"commits_url": "https://api.github.com/repos/rails/rails/commits{/sha}",
|
|
50
|
+
"git_commits_url": "https://api.github.com/repos/rails/rails/git/commits{/sha}",
|
|
51
|
+
"comments_url": "https://api.github.com/repos/rails/rails/comments{/number}",
|
|
52
|
+
"issue_comment_url": "https://api.github.com/repos/rails/rails/issues/comments/{number}",
|
|
53
|
+
"contents_url": "https://api.github.com/repos/rails/rails/contents/{+path}",
|
|
54
|
+
"compare_url": "https://api.github.com/repos/rails/rails/compare/{base}...{head}",
|
|
55
|
+
"merges_url": "https://api.github.com/repos/rails/rails/merges",
|
|
56
|
+
"archive_url": "https://api.github.com/repos/rails/rails/{archive_format}{/ref}",
|
|
57
|
+
"downloads_url": "https://api.github.com/repos/rails/rails/downloads",
|
|
58
|
+
"issues_url": "https://api.github.com/repos/rails/rails/issues{/number}",
|
|
59
|
+
"pulls_url": "https://api.github.com/repos/rails/rails/pulls{/number}",
|
|
60
|
+
"milestones_url": "https://api.github.com/repos/rails/rails/milestones{/number}",
|
|
61
|
+
"notifications_url": "https://api.github.com/repos/rails/rails/notifications{?since,all,participating}",
|
|
62
|
+
"labels_url": "https://api.github.com/repos/rails/rails/labels{/name}",
|
|
63
|
+
"releases_url": "https://api.github.com/repos/rails/rails/releases{/id}",
|
|
64
|
+
"created_at": "2008-04-11T02:19:47Z",
|
|
65
|
+
"updated_at": "2014-06-25T21:08:45Z",
|
|
66
|
+
"pushed_at": "2014-06-25T17:47:52Z",
|
|
67
|
+
"git_url": "git://github.com/rails/rails.git",
|
|
68
|
+
"ssh_url": "git@github.com:rails/rails.git",
|
|
69
|
+
"clone_url": "https://github.com/rails/rails.git",
|
|
70
|
+
"svn_url": "https://github.com/rails/rails",
|
|
71
|
+
"homepage": "http://rubyonrails.org",
|
|
72
|
+
"size": 331047,
|
|
73
|
+
"stargazers_count": 22248,
|
|
74
|
+
"watchers_count": 22248,
|
|
75
|
+
"language": "Ruby",
|
|
76
|
+
"has_issues": true,
|
|
77
|
+
"has_downloads": true,
|
|
78
|
+
"has_wiki": false,
|
|
79
|
+
"forks_count": 8278,
|
|
80
|
+
"mirror_url": null,
|
|
81
|
+
"open_issues_count": 625,
|
|
82
|
+
"forks": 8278,
|
|
83
|
+
"open_issues": 625,
|
|
84
|
+
"watchers": 22248,
|
|
85
|
+
"default_branch": "master",
|
|
86
|
+
"organization": {
|
|
87
|
+
"login": "rails",
|
|
88
|
+
"id": 4223,
|
|
89
|
+
"avatar_url": "https://avatars.githubusercontent.com/u/4223?",
|
|
90
|
+
"gravatar_id": "30f39a09e233e8369dddf6feb4be0308",
|
|
91
|
+
"url": "https://api.github.com/users/rails",
|
|
92
|
+
"html_url": "https://github.com/rails",
|
|
93
|
+
"followers_url": "https://api.github.com/users/rails/followers",
|
|
94
|
+
"following_url": "https://api.github.com/users/rails/following{/other_user}",
|
|
95
|
+
"gists_url": "https://api.github.com/users/rails/gists{/gist_id}",
|
|
96
|
+
"starred_url": "https://api.github.com/users/rails/starred{/owner}{/repo}",
|
|
97
|
+
"subscriptions_url": "https://api.github.com/users/rails/subscriptions",
|
|
98
|
+
"organizations_url": "https://api.github.com/users/rails/orgs",
|
|
99
|
+
"repos_url": "https://api.github.com/users/rails/repos",
|
|
100
|
+
"events_url": "https://api.github.com/users/rails/events{/privacy}",
|
|
101
|
+
"received_events_url": "https://api.github.com/users/rails/received_events",
|
|
102
|
+
"type": "Organization",
|
|
103
|
+
"site_admin": false
|
|
104
|
+
},
|
|
105
|
+
"network_count": 8278,
|
|
106
|
+
"subscribers_count": 1521
|
|
107
|
+
}
|
data/spec/parser_spec.rb
ADDED
|
@@ -0,0 +1,795 @@
|
|
|
1
|
+
require 'yajl/ffi'
|
|
2
|
+
require 'minitest/autorun'
|
|
3
|
+
|
|
4
|
+
describe Yajl::FFI::Parser do
|
|
5
|
+
subject { Yajl::FFI::Parser.new }
|
|
6
|
+
|
|
7
|
+
describe 'parsing a document' do
|
|
8
|
+
it 'rejects documents containing bad start character' do
|
|
9
|
+
expected = [:error]
|
|
10
|
+
assert_equal expected, events('a')
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
it 'rejects documents starting with period' do
|
|
14
|
+
expected = [:error]
|
|
15
|
+
assert_equal expected, events('.')
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
it 'parses a null value document' do
|
|
19
|
+
expected = [:start_document, [:value, nil], :end_document]
|
|
20
|
+
assert_equal expected, events('null')
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
it 'parses a false value document' do
|
|
24
|
+
expected = [:start_document, [:value, false], :end_document]
|
|
25
|
+
assert_equal expected, events('false')
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
it 'parses a true value document' do
|
|
29
|
+
expected = [:start_document, [:value, true], :end_document]
|
|
30
|
+
assert_equal expected, events('true')
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
it 'parses a string document' do
|
|
34
|
+
expected = [:start_document, [:value, "test"], :end_document]
|
|
35
|
+
assert_equal expected, events('"test"')
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
it 'parses an integer value document' do
|
|
39
|
+
expected = [:start_document, [:value, 12], :end_document]
|
|
40
|
+
events = events('12', subject)
|
|
41
|
+
assert events.empty?
|
|
42
|
+
subject.finish
|
|
43
|
+
assert_equal expected, events
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
it 'parses a float value document' do
|
|
47
|
+
expected = [:start_document, [:value, 12.1], :end_document]
|
|
48
|
+
events = events('12.1', subject)
|
|
49
|
+
assert events.empty?
|
|
50
|
+
subject.finish
|
|
51
|
+
assert_equal expected, events
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
it 'parses a value document with leading whitespace' do
|
|
55
|
+
expected = [:start_document, [:value, false], :end_document]
|
|
56
|
+
assert_equal expected, events(' false ')
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
it 'parses array documents' do
|
|
60
|
+
expected = [:start_document, :start_array, :end_array, :end_document]
|
|
61
|
+
assert_equal expected, events('[]')
|
|
62
|
+
assert_equal expected, events('[ ]')
|
|
63
|
+
assert_equal expected, events(' [] ')
|
|
64
|
+
assert_equal expected, events(' [ ] ')
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
it 'parses object documents' do
|
|
68
|
+
expected = [:start_document, :start_object, :end_object, :end_document]
|
|
69
|
+
assert_equal expected, events('{}')
|
|
70
|
+
assert_equal expected, events('{ }')
|
|
71
|
+
assert_equal expected, events(' {} ')
|
|
72
|
+
assert_equal expected, events(' { } ')
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
it 'rejects documents with trailing characters' do
|
|
76
|
+
expected = [:start_document, :start_object, :end_object, :end_document, :error]
|
|
77
|
+
assert_equal expected, events('{}a')
|
|
78
|
+
assert_equal expected, events('{ } 12')
|
|
79
|
+
assert_equal expected, events(' {} false')
|
|
80
|
+
assert_equal expected, events(' { }, {}')
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
it 'ignores whitespace around tokens, preserves it within strings' do
|
|
84
|
+
json = %Q{
|
|
85
|
+
{ " key 1 " : \t [
|
|
86
|
+
1, 2, " my string ",\r
|
|
87
|
+
false, true, null ]
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
expected = [
|
|
91
|
+
:start_document,
|
|
92
|
+
:start_object,
|
|
93
|
+
[:key, " key 1 "],
|
|
94
|
+
:start_array,
|
|
95
|
+
[:value, 1],
|
|
96
|
+
[:value, 2],
|
|
97
|
+
[:value, " my string "],
|
|
98
|
+
[:value, false],
|
|
99
|
+
[:value, true],
|
|
100
|
+
[:value, nil],
|
|
101
|
+
:end_array,
|
|
102
|
+
:end_object,
|
|
103
|
+
:end_document
|
|
104
|
+
]
|
|
105
|
+
assert_equal expected, events(json)
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
it 'rejects partial keyword tokens' do
|
|
109
|
+
expected = [:start_document, :start_array, :error]
|
|
110
|
+
assert_equal expected, events('[tru]')
|
|
111
|
+
assert_equal expected, events('[fal]')
|
|
112
|
+
assert_equal expected, events('[nul,true]')
|
|
113
|
+
assert_equal expected, events('[fals1]')
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
it 'parses single keyword tokens' do
|
|
117
|
+
expected = [:start_document, :start_array, [:value, true], :end_array, :end_document]
|
|
118
|
+
assert_equal expected, events('[true]')
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
it 'parses keywords in series' do
|
|
122
|
+
expected = [:start_document, :start_array, [:value, true], [:value, nil], :end_array, :end_document]
|
|
123
|
+
assert_equal expected, events('[true, null]')
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
describe 'finishing the parse' do
|
|
128
|
+
it 'rejects finish with no json data provided' do
|
|
129
|
+
-> { subject.finish }.must_raise Yajl::FFI::ParserError
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
it 'rejects partial null keyword' do
|
|
133
|
+
subject << 'nul'
|
|
134
|
+
-> { subject.finish }.must_raise Yajl::FFI::ParserError
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
it 'rejects partial true keyword' do
|
|
138
|
+
subject << 'tru'
|
|
139
|
+
-> { subject.finish }.must_raise Yajl::FFI::ParserError
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
it 'rejects partial false keyword' do
|
|
143
|
+
subject << 'fals'
|
|
144
|
+
-> { subject.finish }.must_raise Yajl::FFI::ParserError
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
it 'rejects partial float literal' do
|
|
148
|
+
subject << '42.'
|
|
149
|
+
-> { subject.finish }.must_raise Yajl::FFI::ParserError
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
it 'does nothing on subsequent finish' do
|
|
153
|
+
begin
|
|
154
|
+
subject << 'false'
|
|
155
|
+
subject.finish
|
|
156
|
+
subject.finish
|
|
157
|
+
rescue
|
|
158
|
+
fail 'raised unexpected error'
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
describe 'parsing number tokens' do
|
|
164
|
+
it 'rejects invalid negative numbers' do
|
|
165
|
+
expected = [:start_document, :start_array, :error]
|
|
166
|
+
assert_equal expected, events('[-]')
|
|
167
|
+
|
|
168
|
+
expected = [:start_document, :start_array, [:value, 1], :error]
|
|
169
|
+
assert_equal expected, events('[1-0]')
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
it 'parses integer zero' do
|
|
173
|
+
expected = [:start_document, :start_array, [:value, 0], :end_array, :end_document]
|
|
174
|
+
assert_equal expected, events('[0]')
|
|
175
|
+
assert_equal expected, events('[-0]')
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
it 'parses float zero' do
|
|
179
|
+
expected = [:start_document, :start_array, [:value, 0.0], :end_array, :end_document]
|
|
180
|
+
assert_equal expected, events('[0.0]')
|
|
181
|
+
assert_equal expected, events('[-0.0]')
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
it 'rejects multi zero' do
|
|
185
|
+
expected = [:start_document, :start_array, [:value, 0], :error]
|
|
186
|
+
assert_equal expected, events('[00]')
|
|
187
|
+
assert_equal expected, events('[-00]')
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
it 'rejects integers that start with zero' do
|
|
191
|
+
expected = [:start_document, :start_array, [:value, 0], :error]
|
|
192
|
+
assert_equal expected, events('[01]')
|
|
193
|
+
assert_equal expected, events('[-01]')
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
it 'parses integer tokens' do
|
|
197
|
+
expected = [:start_document, :start_array, [:value, 1], :end_array, :end_document]
|
|
198
|
+
assert_equal expected, events('[1]')
|
|
199
|
+
|
|
200
|
+
expected = [:start_document, :start_array, [:value, -1], :end_array, :end_document]
|
|
201
|
+
assert_equal expected, events('[-1]')
|
|
202
|
+
|
|
203
|
+
expected = [:start_document, :start_array, [:value, 123], :end_array, :end_document]
|
|
204
|
+
assert_equal expected, events('[123]')
|
|
205
|
+
|
|
206
|
+
expected = [:start_document, :start_array, [:value, -123], :end_array, :end_document]
|
|
207
|
+
assert_equal expected, events('[-123]')
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
it 'parses float tokens' do
|
|
211
|
+
expected = [:start_document, :start_array, [:value, 1.0], :end_array, :end_document]
|
|
212
|
+
assert_equal expected, events('[1.0]')
|
|
213
|
+
assert_equal expected, events('[1.00]')
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
it 'parses negative floats' do
|
|
217
|
+
expected = [:start_document, :start_array, [:value, -1.0], :end_array, :end_document]
|
|
218
|
+
assert_equal expected, events('[-1.0]')
|
|
219
|
+
assert_equal expected, events('[-1.00]')
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
it 'parses multi-digit floats' do
|
|
223
|
+
expected = [:start_document, :start_array, [:value, 123.012], :end_array, :end_document]
|
|
224
|
+
assert_equal expected, events('[123.012]')
|
|
225
|
+
assert_equal expected, events('[123.0120]')
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
it 'parses negative multi-digit floats' do
|
|
229
|
+
expected = [:start_document, :start_array, [:value, -123.012], :end_array, :end_document]
|
|
230
|
+
assert_equal expected, events('[-123.012]')
|
|
231
|
+
assert_equal expected, events('[-123.0120]')
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
it 'rejects floats missing leading zero' do
|
|
235
|
+
expected = [:start_document, :start_array, :error]
|
|
236
|
+
assert_equal expected, events('[.1]')
|
|
237
|
+
assert_equal expected, events('[-.1]')
|
|
238
|
+
assert_equal expected, events('[.01]')
|
|
239
|
+
assert_equal expected, events('[-.01]')
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
it 'rejects float missing fraction' do
|
|
243
|
+
expected = [:start_document, :start_array, :error]
|
|
244
|
+
assert_equal expected, events('[.]')
|
|
245
|
+
assert_equal expected, events('[..]')
|
|
246
|
+
assert_equal expected, events('[0.]')
|
|
247
|
+
assert_equal expected, events('[12.]')
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
it 'parses positive exponent integers' do
|
|
251
|
+
expected = [:start_document, :start_array, [:value, 212], :end_array, :end_document]
|
|
252
|
+
assert_equal expected, events('[2.12e2]')
|
|
253
|
+
assert_equal expected, events('[2.12e02]')
|
|
254
|
+
assert_equal expected, events('[2.12e+2]')
|
|
255
|
+
assert_equal expected, events('[2.12e+02]')
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
it 'parses positive exponent floats' do
|
|
259
|
+
expected = [:start_document, :start_array, [:value, 21.2], :end_array, :end_document]
|
|
260
|
+
assert_equal expected, events('[2.12e1]')
|
|
261
|
+
assert_equal expected, events('[2.12e01]')
|
|
262
|
+
assert_equal expected, events('[2.12e+1]')
|
|
263
|
+
assert_equal expected, events('[2.12e+01]')
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
it 'parses negative exponent' do
|
|
267
|
+
expected = [:start_document, :start_array, [:value, 0.0212], :end_array, :end_document]
|
|
268
|
+
assert_equal expected, events('[2.12e-2]')
|
|
269
|
+
assert_equal expected, events('[2.12e-02]')
|
|
270
|
+
assert_equal expected, events('[2.12e-2]')
|
|
271
|
+
assert_equal expected, events('[2.12e-02]')
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
it 'parses zero exponent floats' do
|
|
275
|
+
expected = [:start_document, :start_array, [:value, 2.12], :end_array, :end_document]
|
|
276
|
+
assert_equal expected, events('[2.12e0]')
|
|
277
|
+
assert_equal expected, events('[2.12e00]')
|
|
278
|
+
assert_equal expected, events('[2.12e-0]')
|
|
279
|
+
assert_equal expected, events('[2.12e-00]')
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
it 'parses zero exponent integers' do
|
|
283
|
+
expected = [:start_document, :start_array, [:value, 2.0], :end_array, :end_document]
|
|
284
|
+
assert_equal expected, events('[2e0]')
|
|
285
|
+
assert_equal expected, events('[2e00]')
|
|
286
|
+
assert_equal expected, events('[2e-0]')
|
|
287
|
+
assert_equal expected, events('[2e-00]')
|
|
288
|
+
end
|
|
289
|
+
|
|
290
|
+
it 'rejects missing exponent' do
|
|
291
|
+
expected = [:start_document, :start_array, :error]
|
|
292
|
+
assert_equal expected, events('[e]')
|
|
293
|
+
assert_equal expected, events('[1e]')
|
|
294
|
+
assert_equal expected, events('[1e-]')
|
|
295
|
+
assert_equal expected, events('[1e--]')
|
|
296
|
+
assert_equal expected, events('[1e+]')
|
|
297
|
+
assert_equal expected, events('[1e++]')
|
|
298
|
+
assert_equal expected, events('[0.e]')
|
|
299
|
+
assert_equal expected, events('[10.e]')
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
it 'rejects float with trailing character' do
|
|
303
|
+
expected = [:start_document, :start_array, [:value, 0.0], :error]
|
|
304
|
+
assert_equal expected, events('[0.0q]')
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
it 'rejects integer with trailing character' do
|
|
308
|
+
expected = [:start_document, :start_array, [:value, 1], :error]
|
|
309
|
+
assert_equal expected, events('[1q]')
|
|
310
|
+
end
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
describe 'parsing string tokens' do
|
|
314
|
+
describe 'parsing two-character escapes' do
|
|
315
|
+
it 'rejects invalid escape characters' do
|
|
316
|
+
expected = [:start_document, :start_array, :error]
|
|
317
|
+
assert_equal expected, events('["\\a"]')
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
it 'parses quotation mark' do
|
|
321
|
+
expected = [:start_document, :start_array, [:value, "\""], :end_array, :end_document]
|
|
322
|
+
assert_equal expected, events('["\""]')
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
it 'parses reverse solidus' do
|
|
326
|
+
expected = [:start_document, :start_array, [:value, "\\"], :end_array, :end_document]
|
|
327
|
+
assert_equal expected, events('["\\\"]')
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
it 'parses solidus' do
|
|
331
|
+
expected = [:start_document, :start_array, [:value, "/"], :end_array, :end_document]
|
|
332
|
+
assert_equal expected, events('["\/"]')
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
it 'parses backspace' do
|
|
336
|
+
expected = [:start_document, :start_array, [:value, "\b"], :end_array, :end_document]
|
|
337
|
+
assert_equal expected, events('["\b"]')
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
it 'parses form feed' do
|
|
341
|
+
expected = [:start_document, :start_array, [:value, "\f"], :end_array, :end_document]
|
|
342
|
+
assert_equal expected, events('["\f"]')
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
it 'parses line feed' do
|
|
346
|
+
expected = [:start_document, :start_array, [:value, "\n"], :end_array, :end_document]
|
|
347
|
+
assert_equal expected, events('["\n"]')
|
|
348
|
+
end
|
|
349
|
+
|
|
350
|
+
it 'parses carriage return' do
|
|
351
|
+
expected = [:start_document, :start_array, [:value, "\r"], :end_array, :end_document]
|
|
352
|
+
assert_equal expected, events('["\r"]')
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
it 'parses tab' do
|
|
356
|
+
expected = [:start_document, :start_array, [:value, "\t"], :end_array, :end_document]
|
|
357
|
+
assert_equal expected, events('["\t"]')
|
|
358
|
+
end
|
|
359
|
+
|
|
360
|
+
it 'parses a series of escapes with whitespace' do
|
|
361
|
+
expected = [:start_document, :start_array, [:value, "\" \\ / \b \f \n \r \t"], :end_array, :end_document]
|
|
362
|
+
assert_equal expected, events('["\" \\\ \/ \b \f \n \r \t"]')
|
|
363
|
+
end
|
|
364
|
+
|
|
365
|
+
it 'parses a series of escapes without whitespace' do
|
|
366
|
+
expected = [:start_document, :start_array, [:value, "\"\\/\b\f\n\r\t"], :end_array, :end_document]
|
|
367
|
+
assert_equal expected, events('["\"\\\\/\b\f\n\r\t"]')
|
|
368
|
+
end
|
|
369
|
+
|
|
370
|
+
it 'parses a series of escapes with duplicate characters between them' do
|
|
371
|
+
expected = [:start_document, :start_array, [:value, "\"t\\b/f\bn\f/\nn\rr\t"], :end_array, :end_document]
|
|
372
|
+
assert_equal expected, events('["\"t\\\b\/f\bn\f/\nn\rr\t"]')
|
|
373
|
+
end
|
|
374
|
+
end
|
|
375
|
+
|
|
376
|
+
describe 'parsing control characters' do
|
|
377
|
+
it 'rejects control character in array' do
|
|
378
|
+
expected = [:start_document, :start_array, :error]
|
|
379
|
+
assert_equal expected, events("[\" \u0000 \"]")
|
|
380
|
+
end
|
|
381
|
+
|
|
382
|
+
it 'rejects control character in object' do
|
|
383
|
+
expected = [:start_document, :start_object, :error]
|
|
384
|
+
assert_equal expected, events("{\" \u0000 \":12}")
|
|
385
|
+
end
|
|
386
|
+
|
|
387
|
+
it 'parses escaped control character' do
|
|
388
|
+
expected = [:start_document, :start_array, [:value, "\u0000"], :end_array, :end_document]
|
|
389
|
+
assert_equal expected, events('["\\u0000"]')
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
it 'parses escaped control character in object key' do
|
|
393
|
+
expected = [:start_document, :start_object, [:key, "\u0000"], [:value, 12], :end_object, :end_document]
|
|
394
|
+
assert_equal expected, events('{"\\u0000": 12}')
|
|
395
|
+
end
|
|
396
|
+
|
|
397
|
+
it 'parses non-control character' do
|
|
398
|
+
# del ascii 127 is allowed unescaped in json
|
|
399
|
+
expected = [:start_document, :start_array, [:value, " \u007F "], :end_array, :end_document]
|
|
400
|
+
assert_equal expected, events("[\" \u007f \"]")
|
|
401
|
+
end
|
|
402
|
+
end
|
|
403
|
+
|
|
404
|
+
describe 'parsing unicode escape sequences' do
|
|
405
|
+
it 'parses escaped ascii character' do
|
|
406
|
+
a = "\x61"
|
|
407
|
+
escaped = '\u0061'
|
|
408
|
+
expected = [:start_document, :start_array, [:value, a], :end_array, :end_document]
|
|
409
|
+
assert_equal expected, events('["' + escaped + '"]')
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
it 'parses un-escaped raw unicode' do
|
|
413
|
+
# U+1F602 face with tears of joy
|
|
414
|
+
face = "\xf0\x9f\x98\x82"
|
|
415
|
+
expected = [:start_document, :start_array, [:value, face], :end_array, :end_document]
|
|
416
|
+
assert_equal expected, events('["' + face + '"]')
|
|
417
|
+
end
|
|
418
|
+
|
|
419
|
+
it 'parses escaped unicode surrogate pairs' do
|
|
420
|
+
# U+1F602 face with tears of joy
|
|
421
|
+
face = "\xf0\x9f\x98\x82"
|
|
422
|
+
escaped = '\uD83D\uDE02'
|
|
423
|
+
expected = [:start_document, :start_array, [:value, face], :end_array, :end_document]
|
|
424
|
+
assert_equal expected, events('["' + escaped + '"]')
|
|
425
|
+
end
|
|
426
|
+
|
|
427
|
+
it 'rejects partial unicode escapes' do
|
|
428
|
+
expected = [:start_document, :start_array, :error]
|
|
429
|
+
assert_equal expected, events('[" \\u "]')
|
|
430
|
+
assert_equal expected, events('[" \\u2 "]')
|
|
431
|
+
assert_equal expected, events('[" \\u26 "]')
|
|
432
|
+
assert_equal expected, events('[" \\u260 "]')
|
|
433
|
+
end
|
|
434
|
+
|
|
435
|
+
it 'parses unicode escapes' do
|
|
436
|
+
# U+2603 snowman
|
|
437
|
+
snowman = "\xe2\x98\x83"
|
|
438
|
+
escaped = '\u2603'
|
|
439
|
+
|
|
440
|
+
expected = [:start_document, :start_array, [:value, snowman], :end_array, :end_document]
|
|
441
|
+
assert_equal expected, events('["' + escaped + '"]')
|
|
442
|
+
|
|
443
|
+
expected = [:start_document, :start_array, [:value, 'snow' + snowman + ' man'], :end_array, :end_document]
|
|
444
|
+
assert_equal expected, events('["snow' + escaped + ' man"]')
|
|
445
|
+
|
|
446
|
+
expected = [:start_document, :start_array, [:value, 'snow' + snowman + '3 man'], :end_array, :end_document]
|
|
447
|
+
assert_equal expected, events('["snow' + escaped + '3 man"]')
|
|
448
|
+
|
|
449
|
+
expected = [:start_document, :start_object, [:key, 'snow' + snowman + '3 man'], [:value, 1], :end_object, :end_document]
|
|
450
|
+
assert_equal expected, events('{"snow\\u26033 man": 1}')
|
|
451
|
+
end
|
|
452
|
+
end
|
|
453
|
+
|
|
454
|
+
describe 'parsing unicode escapes with surrogate pairs' do
|
|
455
|
+
it 'rejects missing first pair' do
|
|
456
|
+
expected = [:start_document, :start_array, :error]
|
|
457
|
+
assert_equal expected, events('["\uDD1E"]')
|
|
458
|
+
end
|
|
459
|
+
|
|
460
|
+
it 'rejects double second pair' do
|
|
461
|
+
expected = [:start_document, :start_array, :error]
|
|
462
|
+
assert_equal expected, events('["\uDD1E\uDD1E"]')
|
|
463
|
+
end
|
|
464
|
+
|
|
465
|
+
it 'rejects reversed pair' do
|
|
466
|
+
expected = [:start_document, :start_array, :error]
|
|
467
|
+
assert_equal expected, events('["\uDD1E\uD834"]')
|
|
468
|
+
end
|
|
469
|
+
|
|
470
|
+
it 'parses correct pairs in object keys and values' do
|
|
471
|
+
# U+1D11E G-Clef
|
|
472
|
+
clef = "\xf0\x9d\x84\x9e"
|
|
473
|
+
expected = [
|
|
474
|
+
:start_document,
|
|
475
|
+
:start_object,
|
|
476
|
+
[:key, clef],
|
|
477
|
+
[:value, "g\u{1D11E}clef"],
|
|
478
|
+
:end_object,
|
|
479
|
+
:end_document
|
|
480
|
+
]
|
|
481
|
+
assert_equal expected, events(%q{ {"\uD834\uDD1E": "g\uD834\uDD1Eclef"} })
|
|
482
|
+
end
|
|
483
|
+
end
|
|
484
|
+
end
|
|
485
|
+
|
|
486
|
+
describe 'parsing arrays' do
|
|
487
|
+
it 'rejects trailing comma' do
|
|
488
|
+
expected = [:start_document, :start_array, [:value, 12], :error]
|
|
489
|
+
assert_equal expected, events('[12, ]')
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
it 'parses nested empty array' do
|
|
493
|
+
expected = [:start_document, :start_array, :start_array, :end_array, :end_array, :end_document]
|
|
494
|
+
assert_equal expected, events('[[]]')
|
|
495
|
+
end
|
|
496
|
+
|
|
497
|
+
it 'parses nested array with value' do
|
|
498
|
+
expected = [:start_document, :start_array, :start_array, [:value, 2.1], :end_array, :end_array, :end_document]
|
|
499
|
+
assert_equal expected, events('[[ 2.10 ]]')
|
|
500
|
+
end
|
|
501
|
+
|
|
502
|
+
it 'rejects malformed arrays' do
|
|
503
|
+
expected = [:start_document, :start_array, :error]
|
|
504
|
+
assert_equal expected, events('[}')
|
|
505
|
+
assert_equal expected, events('[,]')
|
|
506
|
+
assert_equal expected, events('[, 12]')
|
|
507
|
+
end
|
|
508
|
+
|
|
509
|
+
it 'rejects malformed nested arrays' do
|
|
510
|
+
expected = [:start_document, :start_array, :start_array, :error]
|
|
511
|
+
assert_equal(expected, events('[[}]'))
|
|
512
|
+
assert_equal expected, events('[[}]')
|
|
513
|
+
assert_equal expected, events('[[,]]')
|
|
514
|
+
end
|
|
515
|
+
|
|
516
|
+
it 'rejects malformed array value lists' do
|
|
517
|
+
expected = [:start_document, :start_array, [:value, "test"], :error]
|
|
518
|
+
assert_equal expected, events('["test"}')
|
|
519
|
+
assert_equal expected, events('["test",]')
|
|
520
|
+
assert_equal expected, events('["test" "test"]')
|
|
521
|
+
assert_equal expected, events('["test" 12]')
|
|
522
|
+
end
|
|
523
|
+
|
|
524
|
+
it 'parses array with value' do
|
|
525
|
+
expected = [:start_document, :start_array, [:value, "test"], :end_array, :end_document]
|
|
526
|
+
assert_equal expected, events('["test"]')
|
|
527
|
+
end
|
|
528
|
+
|
|
529
|
+
it 'parses array with value list' do
|
|
530
|
+
expected = [
|
|
531
|
+
:start_document,
|
|
532
|
+
:start_array,
|
|
533
|
+
[:value, 1],
|
|
534
|
+
[:value, 2],
|
|
535
|
+
[:value, nil],
|
|
536
|
+
[:value, 12.1],
|
|
537
|
+
[:value, "test"],
|
|
538
|
+
:end_array,
|
|
539
|
+
:end_document
|
|
540
|
+
]
|
|
541
|
+
assert_equal expected, events('[1,2, null, 12.1,"test"]')
|
|
542
|
+
end
|
|
543
|
+
end
|
|
544
|
+
|
|
545
|
+
describe 'parsing objects' do
|
|
546
|
+
it 'rejects malformed objects' do
|
|
547
|
+
expected = [:start_document, :start_object, :error]
|
|
548
|
+
assert_equal expected, events('{]')
|
|
549
|
+
assert_equal expected, events('{:}')
|
|
550
|
+
end
|
|
551
|
+
|
|
552
|
+
it 'parses single key object' do
|
|
553
|
+
expected = [:start_document, :start_object, [:key, "key 1"], [:value, 12], :end_object, :end_document]
|
|
554
|
+
assert_equal expected, events('{"key 1" : 12}')
|
|
555
|
+
end
|
|
556
|
+
|
|
557
|
+
it 'parses object key value list' do
|
|
558
|
+
expected = [
|
|
559
|
+
:start_document,
|
|
560
|
+
:start_object,
|
|
561
|
+
[:key, "key 1"], [:value, 12],
|
|
562
|
+
[:key, "key 2"], [:value, "two"],
|
|
563
|
+
:end_object,
|
|
564
|
+
:end_document
|
|
565
|
+
]
|
|
566
|
+
assert_equal expected, events('{"key 1" : 12, "key 2":"two"}')
|
|
567
|
+
end
|
|
568
|
+
|
|
569
|
+
it 'rejects object key with no value' do
|
|
570
|
+
expected = [
|
|
571
|
+
:start_document,
|
|
572
|
+
:start_object,
|
|
573
|
+
[:key, "key"],
|
|
574
|
+
:start_array,
|
|
575
|
+
[:value, nil],
|
|
576
|
+
[:value, false],
|
|
577
|
+
[:value, true],
|
|
578
|
+
:end_array,
|
|
579
|
+
[:key, "key 2"],
|
|
580
|
+
:error
|
|
581
|
+
]
|
|
582
|
+
assert_equal expected, events('{"key": [ null , false , true ] ,"key 2"}')
|
|
583
|
+
end
|
|
584
|
+
|
|
585
|
+
it 'rejects object with trailing comma' do
|
|
586
|
+
expected = [:start_document, :start_object, [:key, "key 1"], [:value, 12], :error]
|
|
587
|
+
assert_equal expected, events('{"key 1" : 12,}')
|
|
588
|
+
end
|
|
589
|
+
end
|
|
590
|
+
|
|
591
|
+
describe 'parsing unicode bytes' do
|
|
592
|
+
it 'parses single byte utf-8' do
|
|
593
|
+
expected = [:start_document, :start_array, [:value, "test"], :end_array, :end_document]
|
|
594
|
+
assert_equal expected, events('["test"]')
|
|
595
|
+
end
|
|
596
|
+
|
|
597
|
+
it 'parses full two byte utf-8' do
|
|
598
|
+
expected = [
|
|
599
|
+
:start_document,
|
|
600
|
+
:start_array,
|
|
601
|
+
[:value, "résumé"],
|
|
602
|
+
[:value, "éé"],
|
|
603
|
+
:end_array,
|
|
604
|
+
:end_document
|
|
605
|
+
]
|
|
606
|
+
assert_equal expected, events("[\"résumé\", \"é\xC3\xA9\"]")
|
|
607
|
+
end
|
|
608
|
+
|
|
609
|
+
# Parser should throw an error when only one byte of a two byte character
|
|
610
|
+
# is available. The \xC3 byte is the first byte of the é character.
|
|
611
|
+
it 'rejects a partial two byte utf-8 string' do
|
|
612
|
+
expected = [:start_document, :start_array, :error]
|
|
613
|
+
assert_equal expected, events('["\xC3"]')
|
|
614
|
+
end
|
|
615
|
+
|
|
616
|
+
it 'parses valid two byte utf-8 string' do
|
|
617
|
+
expected = [:start_document, :start_array, [:value, 'é'], :end_array, :end_document]
|
|
618
|
+
assert_equal expected, events("[\"\xC3\xA9\"]")
|
|
619
|
+
end
|
|
620
|
+
|
|
621
|
+
it 'parses full three byte utf-8 string' do
|
|
622
|
+
expected = [
|
|
623
|
+
:start_document,
|
|
624
|
+
:start_array,
|
|
625
|
+
[:value, "snow\u2603man"],
|
|
626
|
+
[:value, "\u2603\u2603"],
|
|
627
|
+
:end_array,
|
|
628
|
+
:end_document
|
|
629
|
+
]
|
|
630
|
+
assert_equal expected, events("[\"snow\u2603man\", \"\u2603\u2603\"]")
|
|
631
|
+
end
|
|
632
|
+
|
|
633
|
+
it 'rejects one byte of three byte utf-8 string' do
|
|
634
|
+
expected = [:start_document, :start_array, :error]
|
|
635
|
+
assert_equal expected, events('["\xE2"]')
|
|
636
|
+
end
|
|
637
|
+
|
|
638
|
+
it 'rejects two bytes of three byte utf-8 string' do
|
|
639
|
+
expected = [:start_document, :start_array, :error]
|
|
640
|
+
assert_equal expected, events('["\xE2\x98"]')
|
|
641
|
+
end
|
|
642
|
+
|
|
643
|
+
it 'parses full three byte utf-8 string' do
|
|
644
|
+
expected = [:start_document, :start_array, [:value, "\u2603"], :end_array, :end_document]
|
|
645
|
+
assert_equal expected, events("[\"\xE2\x98\x83\"]")
|
|
646
|
+
end
|
|
647
|
+
|
|
648
|
+
it 'parses full four byte utf-8 string' do
|
|
649
|
+
expected = [
|
|
650
|
+
:start_document,
|
|
651
|
+
:start_array,
|
|
652
|
+
[:value, "\u{10102} check mark"],
|
|
653
|
+
:end_array,
|
|
654
|
+
:end_document
|
|
655
|
+
]
|
|
656
|
+
assert_equal expected, events("[\"\u{10102} check mark\"]")
|
|
657
|
+
end
|
|
658
|
+
|
|
659
|
+
it 'rejects one byte of four byte utf-8 string' do
|
|
660
|
+
expected = [:start_document, :start_array, :error]
|
|
661
|
+
assert_equal expected, events('["\xF0"]')
|
|
662
|
+
end
|
|
663
|
+
|
|
664
|
+
it 'rejects two bytes of four byte utf-8 string' do
|
|
665
|
+
expected = [:start_document, :start_array, :error]
|
|
666
|
+
assert_equal expected, events('["\xF0\x90"]')
|
|
667
|
+
end
|
|
668
|
+
|
|
669
|
+
it 'rejects three bytes of four byte utf-8 string' do
|
|
670
|
+
expected = [:start_document, :start_array, :error]
|
|
671
|
+
assert_equal expected, events('["\xF0\x90\x84"]')
|
|
672
|
+
end
|
|
673
|
+
|
|
674
|
+
it 'parses full four byte utf-8 string' do
|
|
675
|
+
expected = [:start_document, :start_array, [:value, "\u{10102}"], :end_array, :end_document]
|
|
676
|
+
assert_equal expected, events("[\"\xF0\x90\x84\x82\"]")
|
|
677
|
+
end
|
|
678
|
+
end
|
|
679
|
+
|
|
680
|
+
describe 'parsing json text from the module' do
|
|
681
|
+
it 'parses an array document' do
|
|
682
|
+
result = Yajl::FFI::Parser.parse('[1,2,3]')
|
|
683
|
+
assert_equal [1, 2, 3], result
|
|
684
|
+
end
|
|
685
|
+
|
|
686
|
+
it 'parses a true keyword literal document' do
|
|
687
|
+
result = Yajl::FFI::Parser.parse('true')
|
|
688
|
+
assert_equal true, result
|
|
689
|
+
end
|
|
690
|
+
|
|
691
|
+
it 'parses a false keyword literal document' do
|
|
692
|
+
result = Yajl::FFI::Parser.parse('false')
|
|
693
|
+
assert_equal false, result
|
|
694
|
+
end
|
|
695
|
+
|
|
696
|
+
it 'parses a null keyword literal document' do
|
|
697
|
+
result = Yajl::FFI::Parser.parse('null')
|
|
698
|
+
assert_equal nil, result
|
|
699
|
+
end
|
|
700
|
+
|
|
701
|
+
it 'parses a string literal document' do
|
|
702
|
+
result = Yajl::FFI::Parser.parse('"hello"')
|
|
703
|
+
assert_equal 'hello', result
|
|
704
|
+
end
|
|
705
|
+
|
|
706
|
+
it 'parses an integer literal document' do
|
|
707
|
+
result = Yajl::FFI::Parser.parse('42')
|
|
708
|
+
assert_equal 42, result
|
|
709
|
+
end
|
|
710
|
+
|
|
711
|
+
it 'parses a float literal document' do
|
|
712
|
+
result = Yajl::FFI::Parser.parse('42.12')
|
|
713
|
+
assert_equal 42.12, result
|
|
714
|
+
end
|
|
715
|
+
|
|
716
|
+
it 'rejects a partial float literal document' do
|
|
717
|
+
assert_raises(Yajl::FFI::ParserError) do
|
|
718
|
+
Yajl::FFI::Parser.parse('42.')
|
|
719
|
+
end
|
|
720
|
+
end
|
|
721
|
+
|
|
722
|
+
it 'rejects a partial document' do
|
|
723
|
+
assert_raises(Yajl::FFI::ParserError) do
|
|
724
|
+
Yajl::FFI::Parser.parse('{')
|
|
725
|
+
end
|
|
726
|
+
end
|
|
727
|
+
|
|
728
|
+
it 'rejects an empty document' do
|
|
729
|
+
assert_raises(Yajl::FFI::ParserError) do
|
|
730
|
+
Yajl::FFI::Parser.parse('')
|
|
731
|
+
end
|
|
732
|
+
end
|
|
733
|
+
end
|
|
734
|
+
|
|
735
|
+
it 'registers observers in initializer block' do
|
|
736
|
+
events = []
|
|
737
|
+
parser = Yajl::FFI::Parser.new do
|
|
738
|
+
start_document { events << :start_document }
|
|
739
|
+
end_document { events << :end_document }
|
|
740
|
+
start_object { events << :start_object }
|
|
741
|
+
end_object { events << :end_object }
|
|
742
|
+
key {|k| events << [:key, k] }
|
|
743
|
+
value {|v| events << [:value, v] }
|
|
744
|
+
end
|
|
745
|
+
parser << '{"key":12}'
|
|
746
|
+
expected = [:start_document, :start_object, [:key, "key"], [:value, 12], :end_object, :end_document]
|
|
747
|
+
assert_equal expected, events
|
|
748
|
+
end
|
|
749
|
+
|
|
750
|
+
private
|
|
751
|
+
|
|
752
|
+
# Run a worst case, one character at a time, parse against the
|
|
753
|
+
# JSON string and return a list of events generated by the parser.
|
|
754
|
+
# A special :error event is included if the parser threw an exception.
|
|
755
|
+
#
|
|
756
|
+
# json - The String to parse.
|
|
757
|
+
# parser - The optional Parser instance to use.
|
|
758
|
+
#
|
|
759
|
+
# Returns an Events instance.
|
|
760
|
+
def events(json, parser = nil)
|
|
761
|
+
parser ||= Yajl::FFI::Parser.new
|
|
762
|
+
collector = Events.new(parser)
|
|
763
|
+
begin
|
|
764
|
+
json.each_char {|ch| parser << ch }
|
|
765
|
+
rescue Yajl::FFI::ParserError
|
|
766
|
+
collector.error
|
|
767
|
+
end
|
|
768
|
+
collector.events
|
|
769
|
+
end
|
|
770
|
+
|
|
771
|
+
# Dynamically map methods in this class to parser callback methods
|
|
772
|
+
# so we can collect parser events for inspection by test cases.
|
|
773
|
+
class Events
|
|
774
|
+
METHODS = %w[start_document end_document start_object end_object start_array end_array key value]
|
|
775
|
+
|
|
776
|
+
attr_reader :events
|
|
777
|
+
|
|
778
|
+
def initialize(parser)
|
|
779
|
+
@events = []
|
|
780
|
+
METHODS.each do |name|
|
|
781
|
+
parser.send(name, &method(name))
|
|
782
|
+
end
|
|
783
|
+
end
|
|
784
|
+
|
|
785
|
+
METHODS.each do |name|
|
|
786
|
+
define_method(name) do |*args|
|
|
787
|
+
@events << (args.empty? ? name.to_sym : [name.to_sym, *args])
|
|
788
|
+
end
|
|
789
|
+
end
|
|
790
|
+
|
|
791
|
+
def error
|
|
792
|
+
@events << :error
|
|
793
|
+
end
|
|
794
|
+
end
|
|
795
|
+
end
|