json-stream 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE +1 -1
- data/README.md +11 -8
- data/Rakefile +2 -2
- data/json-stream.gemspec +2 -2
- data/lib/json/stream/buffer.rb +49 -16
- data/lib/json/stream/builder.rb +29 -48
- data/lib/json/stream/parser.rb +231 -76
- data/lib/json/stream/version.rb +1 -1
- data/spec/buffer_spec.rb +103 -0
- data/spec/builder_spec.rb +157 -0
- data/spec/fixtures/repository.json +107 -0
- data/spec/parser_spec.rb +913 -0
- metadata +19 -17
- data/test/buffer_test.rb +0 -87
- data/test/builder_test.rb +0 -123
- data/test/parser_test.rb +0 -453
data/lib/json/stream/version.rb
CHANGED
data/spec/buffer_spec.rb
ADDED
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
|
|
3
|
+
require 'json/stream'
|
|
4
|
+
require 'minitest/autorun'
|
|
5
|
+
|
|
6
|
+
describe JSON::Stream::Buffer do
|
|
7
|
+
subject { JSON::Stream::Buffer.new }
|
|
8
|
+
|
|
9
|
+
it 'accepts single byte characters' do
|
|
10
|
+
assert_equal "", subject << ""
|
|
11
|
+
assert_equal "abc", subject << "abc"
|
|
12
|
+
assert_equal "\u0000abc", subject << "\u0000abc"
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# The é character can be a single codepoint \u00e9 or two codepoints
|
|
16
|
+
# \u0065\u0301. The first is encoded in 2 bytes, the second in 3 bytes.
|
|
17
|
+
# The json and yajl-ruby gems and CouchDB do not normalize unicode text
|
|
18
|
+
# so neither will we. Although, a good way to normalize is by calling
|
|
19
|
+
# ActiveSupport::Multibyte::Chars.new("é").normalize(:c).
|
|
20
|
+
it 'accepts combined characters' do
|
|
21
|
+
assert_equal "\u0065\u0301", subject << "\u0065\u0301"
|
|
22
|
+
assert_equal 3, (subject << "\u0065\u0301").bytesize
|
|
23
|
+
assert_equal 2, (subject << "\u0065\u0301").size
|
|
24
|
+
|
|
25
|
+
assert_equal "\u00e9", subject << "\u00e9"
|
|
26
|
+
assert_equal 2, (subject << "\u00e9").bytesize
|
|
27
|
+
assert_equal 1, (subject << "\u00e9").size
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
it 'accepts valid two byte characters' do
|
|
31
|
+
assert_equal "abcé", subject << "abcé"
|
|
32
|
+
assert_equal "a", subject << "a\xC3"
|
|
33
|
+
assert_equal "é", subject << "\xA9"
|
|
34
|
+
assert_equal "", subject << "\xC3"
|
|
35
|
+
assert_equal "é", subject << "\xA9"
|
|
36
|
+
assert_equal "é", subject << "\xC3\xA9"
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
it 'accepts valid three byte characters' do
|
|
40
|
+
assert_equal "abcé\u2603", subject << "abcé\u2603"
|
|
41
|
+
assert_equal "a", subject << "a\xE2"
|
|
42
|
+
assert_equal "", subject << "\x98"
|
|
43
|
+
assert_equal "\u2603", subject << "\x83"
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
it 'accepts valid four byte characters' do
|
|
47
|
+
assert_equal "abcé\u2603\u{10102}é", subject << "abcé\u2603\u{10102}é"
|
|
48
|
+
assert_equal "a", subject << "a\xF0"
|
|
49
|
+
assert_equal "", subject << "\x90"
|
|
50
|
+
assert_equal "", subject << "\x84"
|
|
51
|
+
assert_equal "\u{10102}", subject << "\x82"
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
it 'rejects invalid two byte start characters' do
|
|
55
|
+
-> { subject << "\xC3\xC3" }.must_raise JSON::Stream::ParserError
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
it 'rejects invalid three byte start characters' do
|
|
59
|
+
-> { subject << "\xE2\xE2" }.must_raise JSON::Stream::ParserError
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
it 'rejects invalid four byte start characters' do
|
|
63
|
+
-> { subject << "\xF0\xF0" }.must_raise JSON::Stream::ParserError
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
it 'rejects a two byte start with single byte continuation character' do
|
|
67
|
+
-> { subject << "\xC3\u0000" }.must_raise JSON::Stream::ParserError
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
it 'rejects a three byte start with single byte continuation character' do
|
|
71
|
+
-> { subject << "\xE2\u0010" }.must_raise JSON::Stream::ParserError
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
it 'rejects a four byte start with single byte continuation character' do
|
|
75
|
+
-> { subject << "\xF0a" }.must_raise JSON::Stream::ParserError
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
it 'rejects an invalid continuation character' do
|
|
79
|
+
-> { subject << "\xA9" }.must_raise JSON::Stream::ParserError
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
it 'rejects an overlong form' do
|
|
83
|
+
-> { subject << "\xC0\x80" }.must_raise JSON::Stream::ParserError
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
describe 'checking for empty buffers' do
|
|
87
|
+
it 'is initially empty' do
|
|
88
|
+
assert subject.empty?
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
it 'is empty after processing complete characters' do
|
|
92
|
+
subject << 'test'
|
|
93
|
+
assert subject.empty?
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
it 'is not empty after processing partial multi-byte characters' do
|
|
97
|
+
subject << "\xC3"
|
|
98
|
+
refute subject.empty?
|
|
99
|
+
subject << "\xA9"
|
|
100
|
+
assert subject.empty?
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
|
|
3
|
+
require 'json/stream'
|
|
4
|
+
require 'minitest/autorun'
|
|
5
|
+
|
|
6
|
+
describe JSON::Stream::Builder do
|
|
7
|
+
let(:parser) { JSON::Stream::Parser.new }
|
|
8
|
+
subject { JSON::Stream::Builder.new(parser) }
|
|
9
|
+
|
|
10
|
+
it 'builds a false value' do
|
|
11
|
+
assert_nil subject.result
|
|
12
|
+
subject.start_document
|
|
13
|
+
subject.value(false)
|
|
14
|
+
assert_nil subject.result
|
|
15
|
+
subject.end_document
|
|
16
|
+
assert_equal false, subject.result
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
it 'builds a string value' do
|
|
20
|
+
assert_nil subject.result
|
|
21
|
+
subject.start_document
|
|
22
|
+
subject.value("test")
|
|
23
|
+
assert_nil subject.result
|
|
24
|
+
subject.end_document
|
|
25
|
+
assert_equal "test", subject.result
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
it 'builds an empty array' do
|
|
29
|
+
assert_nil subject.result
|
|
30
|
+
subject.start_document
|
|
31
|
+
subject.start_array
|
|
32
|
+
subject.end_array
|
|
33
|
+
assert_nil subject.result
|
|
34
|
+
subject.end_document
|
|
35
|
+
assert_equal [], subject.result
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
it 'builds an array of numbers' do
|
|
39
|
+
subject.start_document
|
|
40
|
+
subject.start_array
|
|
41
|
+
subject.value(1)
|
|
42
|
+
subject.value(2)
|
|
43
|
+
subject.value(3)
|
|
44
|
+
subject.end_array
|
|
45
|
+
subject.end_document
|
|
46
|
+
assert_equal [1, 2, 3], subject.result
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
it 'builds nested empty arrays' do
|
|
50
|
+
subject.start_document
|
|
51
|
+
subject.start_array
|
|
52
|
+
subject.start_array
|
|
53
|
+
subject.end_array
|
|
54
|
+
subject.end_array
|
|
55
|
+
subject.end_document
|
|
56
|
+
assert_equal [[]], subject.result
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
it 'builds nested arrays of numbers' do
|
|
60
|
+
subject.start_document
|
|
61
|
+
subject.start_array
|
|
62
|
+
subject.value(1)
|
|
63
|
+
subject.start_array
|
|
64
|
+
subject.value(2)
|
|
65
|
+
subject.end_array
|
|
66
|
+
subject.value(3)
|
|
67
|
+
subject.end_array
|
|
68
|
+
subject.end_document
|
|
69
|
+
assert_equal [1, [2], 3], subject.result
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
it 'builds an empty object' do
|
|
73
|
+
subject.start_document
|
|
74
|
+
subject.start_object
|
|
75
|
+
subject.end_object
|
|
76
|
+
subject.end_document
|
|
77
|
+
assert_equal({}, subject.result)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
it 'builds a complex object' do
|
|
81
|
+
subject.start_document
|
|
82
|
+
subject.start_object
|
|
83
|
+
subject.key("k1")
|
|
84
|
+
subject.value(1)
|
|
85
|
+
subject.key("k2")
|
|
86
|
+
subject.value(nil)
|
|
87
|
+
subject.key("k3")
|
|
88
|
+
subject.value(true)
|
|
89
|
+
subject.key("k4")
|
|
90
|
+
subject.value(false)
|
|
91
|
+
subject.key("k5")
|
|
92
|
+
subject.value("string value")
|
|
93
|
+
subject.end_object
|
|
94
|
+
subject.end_document
|
|
95
|
+
expected = {
|
|
96
|
+
"k1" => 1,
|
|
97
|
+
"k2" => nil,
|
|
98
|
+
"k3" => true,
|
|
99
|
+
"k4" => false,
|
|
100
|
+
"k5" => "string value"
|
|
101
|
+
}
|
|
102
|
+
assert_equal expected, subject.result
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
it 'builds a nested object' do
|
|
106
|
+
subject.start_document
|
|
107
|
+
subject.start_object
|
|
108
|
+
subject.key("k1")
|
|
109
|
+
subject.value(1)
|
|
110
|
+
|
|
111
|
+
subject.key("k2")
|
|
112
|
+
subject.start_object
|
|
113
|
+
subject.end_object
|
|
114
|
+
|
|
115
|
+
subject.key("k3")
|
|
116
|
+
subject.start_object
|
|
117
|
+
subject.key("sub1")
|
|
118
|
+
subject.start_array
|
|
119
|
+
subject.value(12)
|
|
120
|
+
subject.end_array
|
|
121
|
+
subject.end_object
|
|
122
|
+
|
|
123
|
+
subject.key("k4")
|
|
124
|
+
subject.start_array
|
|
125
|
+
subject.value(1)
|
|
126
|
+
subject.start_object
|
|
127
|
+
subject.key("sub2")
|
|
128
|
+
subject.start_array
|
|
129
|
+
subject.value(nil)
|
|
130
|
+
subject.end_array
|
|
131
|
+
subject.end_object
|
|
132
|
+
subject.end_array
|
|
133
|
+
|
|
134
|
+
subject.key("k5")
|
|
135
|
+
subject.value("string value")
|
|
136
|
+
subject.end_object
|
|
137
|
+
subject.end_document
|
|
138
|
+
expected = {
|
|
139
|
+
"k1" => 1,
|
|
140
|
+
"k2" => {},
|
|
141
|
+
"k3" => {"sub1" => [12]},
|
|
142
|
+
"k4" => [1, {"sub2" => [nil]}],
|
|
143
|
+
"k5" => "string value"
|
|
144
|
+
}
|
|
145
|
+
assert_equal expected, subject.result
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
it 'builds a real document' do
|
|
149
|
+
refute_nil subject
|
|
150
|
+
parser << File.read('spec/fixtures/repository.json')
|
|
151
|
+
refute_nil subject.result
|
|
152
|
+
assert_equal 'rails', subject.result['name']
|
|
153
|
+
assert_equal 4223, subject.result['owner']['id']
|
|
154
|
+
assert_equal false, subject.result['fork']
|
|
155
|
+
assert_equal nil, subject.result['mirror_url']
|
|
156
|
+
end
|
|
157
|
+
end
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": 8514,
|
|
3
|
+
"name": "rails",
|
|
4
|
+
"full_name": "rails/rails",
|
|
5
|
+
"owner": {
|
|
6
|
+
"login": "rails",
|
|
7
|
+
"id": 4223,
|
|
8
|
+
"avatar_url": "https://avatars.githubusercontent.com/u/4223?",
|
|
9
|
+
"gravatar_id": "30f39a09e233e8369dddf6feb4be0308",
|
|
10
|
+
"url": "https://api.github.com/users/rails",
|
|
11
|
+
"html_url": "https://github.com/rails",
|
|
12
|
+
"followers_url": "https://api.github.com/users/rails/followers",
|
|
13
|
+
"following_url": "https://api.github.com/users/rails/following{/other_user}",
|
|
14
|
+
"gists_url": "https://api.github.com/users/rails/gists{/gist_id}",
|
|
15
|
+
"starred_url": "https://api.github.com/users/rails/starred{/owner}{/repo}",
|
|
16
|
+
"subscriptions_url": "https://api.github.com/users/rails/subscriptions",
|
|
17
|
+
"organizations_url": "https://api.github.com/users/rails/orgs",
|
|
18
|
+
"repos_url": "https://api.github.com/users/rails/repos",
|
|
19
|
+
"events_url": "https://api.github.com/users/rails/events{/privacy}",
|
|
20
|
+
"received_events_url": "https://api.github.com/users/rails/received_events",
|
|
21
|
+
"type": "Organization",
|
|
22
|
+
"site_admin": false
|
|
23
|
+
},
|
|
24
|
+
"private": false,
|
|
25
|
+
"html_url": "https://github.com/rails/rails",
|
|
26
|
+
"description": "Ruby on Rails",
|
|
27
|
+
"fork": false,
|
|
28
|
+
"url": "https://api.github.com/repos/rails/rails",
|
|
29
|
+
"forks_url": "https://api.github.com/repos/rails/rails/forks",
|
|
30
|
+
"keys_url": "https://api.github.com/repos/rails/rails/keys{/key_id}",
|
|
31
|
+
"collaborators_url": "https://api.github.com/repos/rails/rails/collaborators{/collaborator}",
|
|
32
|
+
"teams_url": "https://api.github.com/repos/rails/rails/teams",
|
|
33
|
+
"hooks_url": "https://api.github.com/repos/rails/rails/hooks",
|
|
34
|
+
"issue_events_url": "https://api.github.com/repos/rails/rails/issues/events{/number}",
|
|
35
|
+
"events_url": "https://api.github.com/repos/rails/rails/events",
|
|
36
|
+
"assignees_url": "https://api.github.com/repos/rails/rails/assignees{/user}",
|
|
37
|
+
"branches_url": "https://api.github.com/repos/rails/rails/branches{/branch}",
|
|
38
|
+
"tags_url": "https://api.github.com/repos/rails/rails/tags",
|
|
39
|
+
"blobs_url": "https://api.github.com/repos/rails/rails/git/blobs{/sha}",
|
|
40
|
+
"git_tags_url": "https://api.github.com/repos/rails/rails/git/tags{/sha}",
|
|
41
|
+
"git_refs_url": "https://api.github.com/repos/rails/rails/git/refs{/sha}",
|
|
42
|
+
"trees_url": "https://api.github.com/repos/rails/rails/git/trees{/sha}",
|
|
43
|
+
"statuses_url": "https://api.github.com/repos/rails/rails/statuses/{sha}",
|
|
44
|
+
"languages_url": "https://api.github.com/repos/rails/rails/languages",
|
|
45
|
+
"stargazers_url": "https://api.github.com/repos/rails/rails/stargazers",
|
|
46
|
+
"contributors_url": "https://api.github.com/repos/rails/rails/contributors",
|
|
47
|
+
"subscribers_url": "https://api.github.com/repos/rails/rails/subscribers",
|
|
48
|
+
"subscription_url": "https://api.github.com/repos/rails/rails/subscription",
|
|
49
|
+
"commits_url": "https://api.github.com/repos/rails/rails/commits{/sha}",
|
|
50
|
+
"git_commits_url": "https://api.github.com/repos/rails/rails/git/commits{/sha}",
|
|
51
|
+
"comments_url": "https://api.github.com/repos/rails/rails/comments{/number}",
|
|
52
|
+
"issue_comment_url": "https://api.github.com/repos/rails/rails/issues/comments/{number}",
|
|
53
|
+
"contents_url": "https://api.github.com/repos/rails/rails/contents/{+path}",
|
|
54
|
+
"compare_url": "https://api.github.com/repos/rails/rails/compare/{base}...{head}",
|
|
55
|
+
"merges_url": "https://api.github.com/repos/rails/rails/merges",
|
|
56
|
+
"archive_url": "https://api.github.com/repos/rails/rails/{archive_format}{/ref}",
|
|
57
|
+
"downloads_url": "https://api.github.com/repos/rails/rails/downloads",
|
|
58
|
+
"issues_url": "https://api.github.com/repos/rails/rails/issues{/number}",
|
|
59
|
+
"pulls_url": "https://api.github.com/repos/rails/rails/pulls{/number}",
|
|
60
|
+
"milestones_url": "https://api.github.com/repos/rails/rails/milestones{/number}",
|
|
61
|
+
"notifications_url": "https://api.github.com/repos/rails/rails/notifications{?since,all,participating}",
|
|
62
|
+
"labels_url": "https://api.github.com/repos/rails/rails/labels{/name}",
|
|
63
|
+
"releases_url": "https://api.github.com/repos/rails/rails/releases{/id}",
|
|
64
|
+
"created_at": "2008-04-11T02:19:47Z",
|
|
65
|
+
"updated_at": "2014-06-25T21:08:45Z",
|
|
66
|
+
"pushed_at": "2014-06-25T17:47:52Z",
|
|
67
|
+
"git_url": "git://github.com/rails/rails.git",
|
|
68
|
+
"ssh_url": "git@github.com:rails/rails.git",
|
|
69
|
+
"clone_url": "https://github.com/rails/rails.git",
|
|
70
|
+
"svn_url": "https://github.com/rails/rails",
|
|
71
|
+
"homepage": "http://rubyonrails.org",
|
|
72
|
+
"size": 331047,
|
|
73
|
+
"stargazers_count": 22248,
|
|
74
|
+
"watchers_count": 22248,
|
|
75
|
+
"language": "Ruby",
|
|
76
|
+
"has_issues": true,
|
|
77
|
+
"has_downloads": true,
|
|
78
|
+
"has_wiki": false,
|
|
79
|
+
"forks_count": 8278,
|
|
80
|
+
"mirror_url": null,
|
|
81
|
+
"open_issues_count": 625,
|
|
82
|
+
"forks": 8278,
|
|
83
|
+
"open_issues": 625,
|
|
84
|
+
"watchers": 22248,
|
|
85
|
+
"default_branch": "master",
|
|
86
|
+
"organization": {
|
|
87
|
+
"login": "rails",
|
|
88
|
+
"id": 4223,
|
|
89
|
+
"avatar_url": "https://avatars.githubusercontent.com/u/4223?",
|
|
90
|
+
"gravatar_id": "30f39a09e233e8369dddf6feb4be0308",
|
|
91
|
+
"url": "https://api.github.com/users/rails",
|
|
92
|
+
"html_url": "https://github.com/rails",
|
|
93
|
+
"followers_url": "https://api.github.com/users/rails/followers",
|
|
94
|
+
"following_url": "https://api.github.com/users/rails/following{/other_user}",
|
|
95
|
+
"gists_url": "https://api.github.com/users/rails/gists{/gist_id}",
|
|
96
|
+
"starred_url": "https://api.github.com/users/rails/starred{/owner}{/repo}",
|
|
97
|
+
"subscriptions_url": "https://api.github.com/users/rails/subscriptions",
|
|
98
|
+
"organizations_url": "https://api.github.com/users/rails/orgs",
|
|
99
|
+
"repos_url": "https://api.github.com/users/rails/repos",
|
|
100
|
+
"events_url": "https://api.github.com/users/rails/events{/privacy}",
|
|
101
|
+
"received_events_url": "https://api.github.com/users/rails/received_events",
|
|
102
|
+
"type": "Organization",
|
|
103
|
+
"site_admin": false
|
|
104
|
+
},
|
|
105
|
+
"network_count": 8278,
|
|
106
|
+
"subscribers_count": 1521
|
|
107
|
+
}
|
data/spec/parser_spec.rb
ADDED
|
@@ -0,0 +1,913 @@
|
|
|
1
|
+
# encoding: UTF-8
|
|
2
|
+
|
|
3
|
+
require 'json/stream'
|
|
4
|
+
require 'minitest/autorun'
|
|
5
|
+
|
|
6
|
+
describe JSON::Stream::Parser do
|
|
7
|
+
subject { JSON::Stream::Parser.new }
|
|
8
|
+
|
|
9
|
+
describe 'parsing a document' do
|
|
10
|
+
it 'rejects documents containing bad start character' do
|
|
11
|
+
expected = [:error]
|
|
12
|
+
assert_equal expected, events('a')
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
it 'rejects documents starting with period' do
|
|
16
|
+
expected = [:error]
|
|
17
|
+
assert_equal expected, events('.')
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
it 'parses a null value document' do
|
|
21
|
+
expected = [:start_document, [:value, nil], :end_document]
|
|
22
|
+
assert_equal expected, events('null')
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
it 'parses a false value document' do
|
|
26
|
+
expected = [:start_document, [:value, false], :end_document]
|
|
27
|
+
assert_equal expected, events('false')
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
it 'parses a true value document' do
|
|
31
|
+
expected = [:start_document, [:value, true], :end_document]
|
|
32
|
+
assert_equal expected, events('true')
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
it 'parses a string document' do
|
|
36
|
+
expected = [:start_document, [:value, "test"], :end_document]
|
|
37
|
+
assert_equal expected, events('"test"')
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
it 'parses a single digit integer value document' do
|
|
41
|
+
expected = [:start_document, [:value, 2], :end_document]
|
|
42
|
+
events = events('2', subject)
|
|
43
|
+
assert events.empty?
|
|
44
|
+
subject.finish
|
|
45
|
+
assert_equal expected, events
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
it 'parses a multiple digit integer value document' do
|
|
49
|
+
expected = [:start_document, [:value, 12], :end_document]
|
|
50
|
+
events = events('12', subject)
|
|
51
|
+
assert events.empty?
|
|
52
|
+
subject.finish
|
|
53
|
+
assert_equal expected, events
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
it 'parses a zero literal document' do
|
|
57
|
+
expected = [:start_document, [:value, 0], :end_document]
|
|
58
|
+
events = events('0', subject)
|
|
59
|
+
assert events.empty?
|
|
60
|
+
subject.finish
|
|
61
|
+
assert_equal expected, events
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
it 'parses a negative integer document' do
|
|
65
|
+
expected = [:start_document, [:value, -1], :end_document]
|
|
66
|
+
events = events('-1', subject)
|
|
67
|
+
assert events.empty?
|
|
68
|
+
subject.finish
|
|
69
|
+
assert_equal expected, events
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
it 'parses an exponent literal document' do
|
|
73
|
+
expected = [:start_document, [:value, 200.0], :end_document]
|
|
74
|
+
events = events('2e2', subject)
|
|
75
|
+
assert events.empty?
|
|
76
|
+
subject.finish
|
|
77
|
+
assert_equal expected, events
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
it 'parses a float value document' do
|
|
81
|
+
expected = [:start_document, [:value, 12.1], :end_document]
|
|
82
|
+
events = events('12.1', subject)
|
|
83
|
+
assert events.empty?
|
|
84
|
+
subject.finish
|
|
85
|
+
assert_equal expected, events
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
it 'parses a value document with leading whitespace' do
|
|
89
|
+
expected = [:start_document, [:value, false], :end_document]
|
|
90
|
+
assert_equal expected, events(' false ')
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
it 'parses array documents' do
|
|
94
|
+
expected = [:start_document, :start_array, :end_array, :end_document]
|
|
95
|
+
assert_equal expected, events('[]')
|
|
96
|
+
assert_equal expected, events('[ ]')
|
|
97
|
+
assert_equal expected, events(' [] ')
|
|
98
|
+
assert_equal expected, events(' [ ] ')
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
it 'parses object documents' do
|
|
102
|
+
expected = [:start_document, :start_object, :end_object, :end_document]
|
|
103
|
+
assert_equal expected, events('{}')
|
|
104
|
+
assert_equal expected, events('{ }')
|
|
105
|
+
assert_equal expected, events(' {} ')
|
|
106
|
+
assert_equal expected, events(' { } ')
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
it 'rejects documents with trailing characters' do
|
|
110
|
+
expected = [:start_document, :start_object, :end_object, :end_document, :error]
|
|
111
|
+
assert_equal expected, events('{}a')
|
|
112
|
+
assert_equal expected, events('{ } 12')
|
|
113
|
+
assert_equal expected, events(' {} false')
|
|
114
|
+
assert_equal expected, events(' { }, {}')
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
it 'ignores whitespace around tokens, preserves it within strings' do
|
|
118
|
+
json = %Q{
|
|
119
|
+
{ " key 1 " : \t [
|
|
120
|
+
1, 2, " my string ",\r
|
|
121
|
+
false, true, null ]
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
expected = [
|
|
125
|
+
:start_document,
|
|
126
|
+
:start_object,
|
|
127
|
+
[:key, " key 1 "],
|
|
128
|
+
:start_array,
|
|
129
|
+
[:value, 1],
|
|
130
|
+
[:value, 2],
|
|
131
|
+
[:value, " my string "],
|
|
132
|
+
[:value, false],
|
|
133
|
+
[:value, true],
|
|
134
|
+
[:value, nil],
|
|
135
|
+
:end_array,
|
|
136
|
+
:end_object,
|
|
137
|
+
:end_document
|
|
138
|
+
]
|
|
139
|
+
assert_equal expected, events(json)
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
it 'rejects form feed whitespace' do
|
|
143
|
+
json = "[1,\f 2]"
|
|
144
|
+
expected = [:start_document, :start_array, [:value, 1], :error]
|
|
145
|
+
assert_equal expected, events(json)
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
it 'rejects vertical tab whitespace' do
|
|
149
|
+
json = "[1,\v 2]"
|
|
150
|
+
expected = [:start_document, :start_array, [:value, 1], :error]
|
|
151
|
+
assert_equal expected, events(json)
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
it 'rejects partial keyword tokens' do
|
|
155
|
+
expected = [:start_document, :start_array, :error]
|
|
156
|
+
assert_equal expected, events('[tru]')
|
|
157
|
+
assert_equal expected, events('[fal]')
|
|
158
|
+
assert_equal expected, events('[nul,true]')
|
|
159
|
+
assert_equal expected, events('[fals1]')
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
it 'rejects scrambled keyword tokens' do
|
|
163
|
+
expected = [:start_document, :start_array, :error]
|
|
164
|
+
assert_equal expected, events('[ture]')
|
|
165
|
+
assert_equal expected, events('[fales]')
|
|
166
|
+
assert_equal expected, events('[nlul]')
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
it 'parses single keyword tokens' do
|
|
170
|
+
expected = [:start_document, :start_array, [:value, true], :end_array, :end_document]
|
|
171
|
+
assert_equal expected, events('[true]')
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
it 'parses keywords in series' do
|
|
175
|
+
expected = [:start_document, :start_array, [:value, true], [:value, nil], :end_array, :end_document]
|
|
176
|
+
assert_equal expected, events('[true, null]')
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
describe 'finishing the parse' do
|
|
181
|
+
it 'rejects finish with no json data provided' do
|
|
182
|
+
-> { subject.finish }.must_raise JSON::Stream::ParserError
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
it 'rejects partial null keyword' do
|
|
186
|
+
subject << 'nul'
|
|
187
|
+
-> { subject.finish }.must_raise JSON::Stream::ParserError
|
|
188
|
+
end
|
|
189
|
+
|
|
190
|
+
it 'rejects partial true keyword' do
|
|
191
|
+
subject << 'tru'
|
|
192
|
+
-> { subject.finish }.must_raise JSON::Stream::ParserError
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
it 'rejects partial false keyword' do
|
|
196
|
+
subject << 'fals'
|
|
197
|
+
-> { subject.finish }.must_raise JSON::Stream::ParserError
|
|
198
|
+
end
|
|
199
|
+
|
|
200
|
+
it 'rejects partial float literal' do
|
|
201
|
+
subject << '42.'
|
|
202
|
+
-> { subject.finish }.must_raise JSON::Stream::ParserError
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
it 'rejects partial exponent' do
|
|
206
|
+
subject << '42e'
|
|
207
|
+
-> { subject.finish }.must_raise JSON::Stream::ParserError
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
it 'rejects malformed exponent' do
|
|
211
|
+
subject << '42e+'
|
|
212
|
+
-> { subject.finish }.must_raise JSON::Stream::ParserError
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
it 'rejects partial negative number' do
|
|
216
|
+
subject << '-'
|
|
217
|
+
-> { subject.finish }.must_raise JSON::Stream::ParserError
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
it 'rejects partial string literal' do
|
|
221
|
+
subject << '"test'
|
|
222
|
+
-> { subject.finish }.must_raise JSON::Stream::ParserError
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
it 'rejects partial object ending in literal value' do
|
|
226
|
+
subject << '{"test": 42'
|
|
227
|
+
-> { subject.finish }.must_raise JSON::Stream::ParserError
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
it 'rejects partial array ending in literal value' do
|
|
231
|
+
subject << '[42'
|
|
232
|
+
-> { subject.finish }.must_raise JSON::Stream::ParserError
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
it 'does nothing on subsequent finish' do
|
|
236
|
+
begin
|
|
237
|
+
subject << 'false'
|
|
238
|
+
subject.finish
|
|
239
|
+
subject.finish
|
|
240
|
+
rescue
|
|
241
|
+
fail 'raised unexpected error'
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
describe 'parsing number tokens' do
|
|
247
|
+
it 'rejects invalid negative numbers' do
|
|
248
|
+
expected = [:start_document, :start_array, :error]
|
|
249
|
+
assert_equal expected, events('[-]')
|
|
250
|
+
|
|
251
|
+
expected = [:start_document, :start_array, [:value, 1], :error]
|
|
252
|
+
assert_equal expected, events('[1-0]')
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
it 'parses integer zero' do
|
|
256
|
+
expected = [:start_document, :start_array, [:value, 0], :end_array, :end_document]
|
|
257
|
+
assert_equal expected, events('[0]')
|
|
258
|
+
assert_equal expected, events('[-0]')
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
it 'parses float zero' do
|
|
262
|
+
expected = [:start_document, :start_array, [:value, 0.0], :end_array, :end_document]
|
|
263
|
+
assert_equal expected, events('[0.0]')
|
|
264
|
+
assert_equal expected, events('[-0.0]')
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
it 'rejects multi zero' do
|
|
268
|
+
expected = [:start_document, :start_array, [:value, 0], :error]
|
|
269
|
+
assert_equal expected, events('[00]')
|
|
270
|
+
assert_equal expected, events('[-00]')
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
it 'rejects integers that start with zero' do
|
|
274
|
+
expected = [:start_document, :start_array, [:value, 0], :error]
|
|
275
|
+
assert_equal expected, events('[01]')
|
|
276
|
+
assert_equal expected, events('[-01]')
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
it 'parses integer tokens' do
|
|
280
|
+
expected = [:start_document, :start_array, [:value, 1], :end_array, :end_document]
|
|
281
|
+
assert_equal expected, events('[1]')
|
|
282
|
+
|
|
283
|
+
expected = [:start_document, :start_array, [:value, -1], :end_array, :end_document]
|
|
284
|
+
assert_equal expected, events('[-1]')
|
|
285
|
+
|
|
286
|
+
expected = [:start_document, :start_array, [:value, 123], :end_array, :end_document]
|
|
287
|
+
assert_equal expected, events('[123]')
|
|
288
|
+
|
|
289
|
+
expected = [:start_document, :start_array, [:value, -123], :end_array, :end_document]
|
|
290
|
+
assert_equal expected, events('[-123]')
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
it 'parses float tokens' do
|
|
294
|
+
expected = [:start_document, :start_array, [:value, 1.0], :end_array, :end_document]
|
|
295
|
+
assert_equal expected, events('[1.0]')
|
|
296
|
+
assert_equal expected, events('[1.00]')
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
it 'parses negative floats' do
|
|
300
|
+
expected = [:start_document, :start_array, [:value, -1.0], :end_array, :end_document]
|
|
301
|
+
assert_equal expected, events('[-1.0]')
|
|
302
|
+
assert_equal expected, events('[-1.00]')
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
it 'parses multi-digit floats' do
|
|
306
|
+
expected = [:start_document, :start_array, [:value, 123.012], :end_array, :end_document]
|
|
307
|
+
assert_equal expected, events('[123.012]')
|
|
308
|
+
assert_equal expected, events('[123.0120]')
|
|
309
|
+
end
|
|
310
|
+
|
|
311
|
+
it 'parses negative multi-digit floats' do
|
|
312
|
+
expected = [:start_document, :start_array, [:value, -123.012], :end_array, :end_document]
|
|
313
|
+
assert_equal expected, events('[-123.012]')
|
|
314
|
+
assert_equal expected, events('[-123.0120]')
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
it 'rejects floats missing leading zero' do
|
|
318
|
+
expected = [:start_document, :start_array, :error]
|
|
319
|
+
assert_equal expected, events('[.1]')
|
|
320
|
+
assert_equal expected, events('[-.1]')
|
|
321
|
+
assert_equal expected, events('[.01]')
|
|
322
|
+
assert_equal expected, events('[-.01]')
|
|
323
|
+
end
|
|
324
|
+
|
|
325
|
+
it 'rejects float missing fraction' do
|
|
326
|
+
expected = [:start_document, :start_array, :error]
|
|
327
|
+
assert_equal expected, events('[.]')
|
|
328
|
+
assert_equal expected, events('[..]')
|
|
329
|
+
assert_equal expected, events('[0.]')
|
|
330
|
+
assert_equal expected, events('[12.]')
|
|
331
|
+
end
|
|
332
|
+
|
|
333
|
+
it 'parses zero with implicit positive exponent as float' do
|
|
334
|
+
expected = [:start_document, :start_array, [:value, 0.0], :end_array, :end_document]
|
|
335
|
+
events = events('[0e2]')
|
|
336
|
+
assert_equal expected, events
|
|
337
|
+
assert_kind_of Float, events[2][1]
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
it 'parses zero with explicit positive exponent as float' do
|
|
341
|
+
expected = [:start_document, :start_array, [:value, 0.0], :end_array, :end_document]
|
|
342
|
+
events = events('[0e+2]')
|
|
343
|
+
assert_equal expected, events
|
|
344
|
+
assert_kind_of Float, events[2][1]
|
|
345
|
+
end
|
|
346
|
+
|
|
347
|
+
it 'parses zero with negative exponent as float' do
|
|
348
|
+
expected = [:start_document, :start_array, [:value, 0.0], :end_array, :end_document]
|
|
349
|
+
events = events('[0e-2]')
|
|
350
|
+
assert_equal expected, events
|
|
351
|
+
assert_kind_of Float, events[2][1]
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
it 'parses positive exponent integers as floats' do
|
|
355
|
+
expected = [:start_document, :start_array, [:value, 212.0], :end_array, :end_document]
|
|
356
|
+
|
|
357
|
+
events = events('[2.12e2]')
|
|
358
|
+
assert_equal expected, events('[2.12e2]')
|
|
359
|
+
assert_kind_of Float, events[2][1]
|
|
360
|
+
|
|
361
|
+
assert_equal expected, events('[2.12e02]')
|
|
362
|
+
assert_equal expected, events('[2.12e+2]')
|
|
363
|
+
assert_equal expected, events('[2.12e+02]')
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
it 'parses positive exponent floats' do
|
|
367
|
+
expected = [:start_document, :start_array, [:value, 21.2], :end_array, :end_document]
|
|
368
|
+
assert_equal expected, events('[2.12e1]')
|
|
369
|
+
assert_equal expected, events('[2.12e01]')
|
|
370
|
+
assert_equal expected, events('[2.12e+1]')
|
|
371
|
+
assert_equal expected, events('[2.12e+01]')
|
|
372
|
+
end
|
|
373
|
+
|
|
374
|
+
it 'parses negative exponent' do
|
|
375
|
+
expected = [:start_document, :start_array, [:value, 0.0212], :end_array, :end_document]
|
|
376
|
+
assert_equal expected, events('[2.12e-2]')
|
|
377
|
+
assert_equal expected, events('[2.12e-02]')
|
|
378
|
+
assert_equal expected, events('[2.12e-2]')
|
|
379
|
+
assert_equal expected, events('[2.12e-02]')
|
|
380
|
+
end
|
|
381
|
+
|
|
382
|
+
it 'parses zero exponent floats' do
|
|
383
|
+
expected = [:start_document, :start_array, [:value, 2.12], :end_array, :end_document]
|
|
384
|
+
assert_equal expected, events('[2.12e0]')
|
|
385
|
+
assert_equal expected, events('[2.12e00]')
|
|
386
|
+
assert_equal expected, events('[2.12e-0]')
|
|
387
|
+
assert_equal expected, events('[2.12e-00]')
|
|
388
|
+
end
|
|
389
|
+
|
|
390
|
+
it 'parses zero exponent integers' do
|
|
391
|
+
expected = [:start_document, :start_array, [:value, 2.0], :end_array, :end_document]
|
|
392
|
+
assert_equal expected, events('[2e0]')
|
|
393
|
+
assert_equal expected, events('[2e00]')
|
|
394
|
+
assert_equal expected, events('[2e-0]')
|
|
395
|
+
assert_equal expected, events('[2e-00]')
|
|
396
|
+
end
|
|
397
|
+
|
|
398
|
+
it 'rejects missing exponent' do
|
|
399
|
+
expected = [:start_document, :start_array, :error]
|
|
400
|
+
assert_equal expected, events('[e]')
|
|
401
|
+
assert_equal expected, events('[1e]')
|
|
402
|
+
assert_equal expected, events('[1e-]')
|
|
403
|
+
assert_equal expected, events('[1e--]')
|
|
404
|
+
assert_equal expected, events('[1e+]')
|
|
405
|
+
assert_equal expected, events('[1e++]')
|
|
406
|
+
assert_equal expected, events('[0.e]')
|
|
407
|
+
assert_equal expected, events('[10.e]')
|
|
408
|
+
end
|
|
409
|
+
|
|
410
|
+
it 'rejects float with trailing character' do
|
|
411
|
+
expected = [:start_document, :start_array, [:value, 0.0], :error]
|
|
412
|
+
assert_equal expected, events('[0.0q]')
|
|
413
|
+
end
|
|
414
|
+
|
|
415
|
+
it 'rejects integer with trailing character' do
|
|
416
|
+
expected = [:start_document, :start_array, [:value, 1], :error]
|
|
417
|
+
assert_equal expected, events('[1q]')
|
|
418
|
+
end
|
|
419
|
+
end
|
|
420
|
+
|
|
421
|
+
describe 'parsing string tokens' do
|
|
422
|
+
describe 'parsing two-character escapes' do
|
|
423
|
+
it 'rejects invalid escape characters' do
|
|
424
|
+
expected = [:start_document, :start_array, :error]
|
|
425
|
+
assert_equal expected, events('["\\a"]')
|
|
426
|
+
end
|
|
427
|
+
|
|
428
|
+
it 'parses quotation mark' do
|
|
429
|
+
expected = [:start_document, :start_array, [:value, "\""], :end_array, :end_document]
|
|
430
|
+
assert_equal expected, events('["\""]')
|
|
431
|
+
end
|
|
432
|
+
|
|
433
|
+
it 'parses reverse solidus' do
|
|
434
|
+
expected = [:start_document, :start_array, [:value, "\\"], :end_array, :end_document]
|
|
435
|
+
assert_equal expected, events('["\\\"]')
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
it 'parses solidus' do
|
|
439
|
+
expected = [:start_document, :start_array, [:value, "/"], :end_array, :end_document]
|
|
440
|
+
assert_equal expected, events('["\/"]')
|
|
441
|
+
end
|
|
442
|
+
|
|
443
|
+
it 'parses backspace' do
|
|
444
|
+
expected = [:start_document, :start_array, [:value, "\b"], :end_array, :end_document]
|
|
445
|
+
assert_equal expected, events('["\b"]')
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
it 'parses form feed' do
|
|
449
|
+
expected = [:start_document, :start_array, [:value, "\f"], :end_array, :end_document]
|
|
450
|
+
assert_equal expected, events('["\f"]')
|
|
451
|
+
end
|
|
452
|
+
|
|
453
|
+
it 'parses line feed' do
|
|
454
|
+
expected = [:start_document, :start_array, [:value, "\n"], :end_array, :end_document]
|
|
455
|
+
assert_equal expected, events('["\n"]')
|
|
456
|
+
end
|
|
457
|
+
|
|
458
|
+
it 'parses carriage return' do
|
|
459
|
+
expected = [:start_document, :start_array, [:value, "\r"], :end_array, :end_document]
|
|
460
|
+
assert_equal expected, events('["\r"]')
|
|
461
|
+
end
|
|
462
|
+
|
|
463
|
+
it 'parses tab' do
|
|
464
|
+
expected = [:start_document, :start_array, [:value, "\t"], :end_array, :end_document]
|
|
465
|
+
assert_equal expected, events('["\t"]')
|
|
466
|
+
end
|
|
467
|
+
|
|
468
|
+
it 'parses a series of escapes with whitespace' do
|
|
469
|
+
expected = [:start_document, :start_array, [:value, "\" \\ / \b \f \n \r \t"], :end_array, :end_document]
|
|
470
|
+
assert_equal expected, events('["\" \\\ \/ \b \f \n \r \t"]')
|
|
471
|
+
end
|
|
472
|
+
|
|
473
|
+
it 'parses a series of escapes without whitespace' do
|
|
474
|
+
expected = [:start_document, :start_array, [:value, "\"\\/\b\f\n\r\t"], :end_array, :end_document]
|
|
475
|
+
assert_equal expected, events('["\"\\\\/\b\f\n\r\t"]')
|
|
476
|
+
end
|
|
477
|
+
|
|
478
|
+
it 'parses a series of escapes with duplicate characters between them' do
|
|
479
|
+
expected = [:start_document, :start_array, [:value, "\"t\\b/f\bn\f/\nn\rr\t"], :end_array, :end_document]
|
|
480
|
+
assert_equal expected, events('["\"t\\\b\/f\bn\f/\nn\rr\t"]')
|
|
481
|
+
end
|
|
482
|
+
end
|
|
483
|
+
|
|
484
|
+
describe 'parsing control characters' do
|
|
485
|
+
it 'rejects control character in array' do
|
|
486
|
+
expected = [:start_document, :start_array, :error]
|
|
487
|
+
assert_equal expected, events("[\" \u0000 \"]")
|
|
488
|
+
end
|
|
489
|
+
|
|
490
|
+
it 'rejects control character in object' do
|
|
491
|
+
expected = [:start_document, :start_object, :error]
|
|
492
|
+
assert_equal expected, events("{\" \u0000 \":12}")
|
|
493
|
+
end
|
|
494
|
+
|
|
495
|
+
it 'parses escaped control character' do
|
|
496
|
+
expected = [:start_document, :start_array, [:value, "\u0000"], :end_array, :end_document]
|
|
497
|
+
assert_equal expected, events('["\\u0000"]')
|
|
498
|
+
end
|
|
499
|
+
|
|
500
|
+
it 'parses escaped control character in object key' do
|
|
501
|
+
expected = [:start_document, :start_object, [:key, "\u0000"], [:value, 12], :end_object, :end_document]
|
|
502
|
+
assert_equal expected, events('{"\\u0000": 12}')
|
|
503
|
+
end
|
|
504
|
+
|
|
505
|
+
it 'parses non-control character' do
|
|
506
|
+
# del ascii 127 is allowed unescaped in json
|
|
507
|
+
expected = [:start_document, :start_array, [:value, " \u007F "], :end_array, :end_document]
|
|
508
|
+
assert_equal expected, events("[\" \u007f \"]")
|
|
509
|
+
end
|
|
510
|
+
end
|
|
511
|
+
|
|
512
|
+
describe 'parsing unicode escape sequences' do
|
|
513
|
+
it 'parses escaped ascii character' do
|
|
514
|
+
a = "\x61"
|
|
515
|
+
escaped = '\u0061'
|
|
516
|
+
expected = [:start_document, :start_array, [:value, a], :end_array, :end_document]
|
|
517
|
+
assert_equal expected, events('["' + escaped + '"]')
|
|
518
|
+
end
|
|
519
|
+
|
|
520
|
+
it 'parses un-escaped raw unicode' do
|
|
521
|
+
# U+1F602 face with tears of joy
|
|
522
|
+
face = "\xf0\x9f\x98\x82"
|
|
523
|
+
expected = [:start_document, :start_array, [:value, face], :end_array, :end_document]
|
|
524
|
+
assert_equal expected, events('["' + face + '"]')
|
|
525
|
+
end
|
|
526
|
+
|
|
527
|
+
it 'parses escaped unicode surrogate pairs' do
|
|
528
|
+
# U+1F602 face with tears of joy
|
|
529
|
+
face = "\xf0\x9f\x98\x82"
|
|
530
|
+
escaped = '\uD83D\uDE02'
|
|
531
|
+
expected = [:start_document, :start_array, [:value, face], :end_array, :end_document]
|
|
532
|
+
assert_equal expected, events('["' + escaped + '"]')
|
|
533
|
+
end
|
|
534
|
+
|
|
535
|
+
it 'rejects partial unicode escapes' do
|
|
536
|
+
expected = [:start_document, :start_array, :error]
|
|
537
|
+
assert_equal expected, events('[" \\u "]')
|
|
538
|
+
assert_equal expected, events('[" \\u2 "]')
|
|
539
|
+
assert_equal expected, events('[" \\u26 "]')
|
|
540
|
+
assert_equal expected, events('[" \\u260 "]')
|
|
541
|
+
end
|
|
542
|
+
|
|
543
|
+
it 'parses unicode escapes' do
|
|
544
|
+
# U+2603 snowman
|
|
545
|
+
snowman = "\xe2\x98\x83"
|
|
546
|
+
escaped = '\u2603'
|
|
547
|
+
|
|
548
|
+
expected = [:start_document, :start_array, [:value, snowman], :end_array, :end_document]
|
|
549
|
+
assert_equal expected, events('["' + escaped + '"]')
|
|
550
|
+
|
|
551
|
+
expected = [:start_document, :start_array, [:value, 'snow' + snowman + ' man'], :end_array, :end_document]
|
|
552
|
+
assert_equal expected, events('["snow' + escaped + ' man"]')
|
|
553
|
+
|
|
554
|
+
expected = [:start_document, :start_array, [:value, 'snow' + snowman + '3 man'], :end_array, :end_document]
|
|
555
|
+
assert_equal expected, events('["snow' + escaped + '3 man"]')
|
|
556
|
+
|
|
557
|
+
expected = [:start_document, :start_object, [:key, 'snow' + snowman + '3 man'], [:value, 1], :end_object, :end_document]
|
|
558
|
+
assert_equal expected, events('{"snow\\u26033 man": 1}')
|
|
559
|
+
end
|
|
560
|
+
end
|
|
561
|
+
|
|
562
|
+
describe 'parsing unicode escapes with surrogate pairs' do
|
|
563
|
+
it 'rejects missing second pair' do
|
|
564
|
+
expected = [:start_document, :start_array, :error]
|
|
565
|
+
assert_equal expected, events('["\uD834"]')
|
|
566
|
+
end
|
|
567
|
+
|
|
568
|
+
it 'rejects missing first pair' do
|
|
569
|
+
expected = [:start_document, :start_array, :error]
|
|
570
|
+
assert_equal expected, events('["\uDD1E"]')
|
|
571
|
+
end
|
|
572
|
+
|
|
573
|
+
it 'rejects double first pair' do
|
|
574
|
+
expected = [:start_document, :start_array, :error]
|
|
575
|
+
assert_equal expected, events('["\uD834\uD834"]')
|
|
576
|
+
end
|
|
577
|
+
|
|
578
|
+
it 'rejects double second pair' do
|
|
579
|
+
expected = [:start_document, :start_array, :error]
|
|
580
|
+
assert_equal expected, events('["\uDD1E\uDD1E"]')
|
|
581
|
+
end
|
|
582
|
+
|
|
583
|
+
it 'rejects reversed pair' do
|
|
584
|
+
expected = [:start_document, :start_array, :error]
|
|
585
|
+
assert_equal expected, events('["\uDD1E\uD834"]')
|
|
586
|
+
end
|
|
587
|
+
|
|
588
|
+
it 'parses correct pairs in object keys and values' do
|
|
589
|
+
# U+1D11E G-Clef
|
|
590
|
+
clef = "\xf0\x9d\x84\x9e"
|
|
591
|
+
expected = [
|
|
592
|
+
:start_document,
|
|
593
|
+
:start_object,
|
|
594
|
+
[:key, clef],
|
|
595
|
+
[:value, "g\u{1D11E}clef"],
|
|
596
|
+
:end_object,
|
|
597
|
+
:end_document
|
|
598
|
+
]
|
|
599
|
+
assert_equal expected, events(%q{ {"\uD834\uDD1E": "g\uD834\uDD1Eclef"} })
|
|
600
|
+
end
|
|
601
|
+
end
|
|
602
|
+
end
|
|
603
|
+
|
|
604
|
+
describe 'parsing arrays' do
|
|
605
|
+
it 'rejects trailing comma' do
|
|
606
|
+
expected = [:start_document, :start_array, [:value, 12], :error]
|
|
607
|
+
assert_equal expected, events('[12, ]')
|
|
608
|
+
end
|
|
609
|
+
|
|
610
|
+
it 'parses nested empty array' do
|
|
611
|
+
expected = [:start_document, :start_array, :start_array, :end_array, :end_array, :end_document]
|
|
612
|
+
assert_equal expected, events('[[]]')
|
|
613
|
+
end
|
|
614
|
+
|
|
615
|
+
it 'parses nested array with value' do
|
|
616
|
+
expected = [:start_document, :start_array, :start_array, [:value, 2.1], :end_array, :end_array, :end_document]
|
|
617
|
+
assert_equal expected, events('[[ 2.10 ]]')
|
|
618
|
+
end
|
|
619
|
+
|
|
620
|
+
it 'rejects malformed arrays' do
|
|
621
|
+
expected = [:start_document, :start_array, :error]
|
|
622
|
+
assert_equal expected, events('[}')
|
|
623
|
+
assert_equal expected, events('[,]')
|
|
624
|
+
assert_equal expected, events('[, 12]')
|
|
625
|
+
end
|
|
626
|
+
|
|
627
|
+
it 'rejects malformed nested arrays' do
|
|
628
|
+
expected = [:start_document, :start_array, :start_array, :error]
|
|
629
|
+
assert_equal(expected, events('[[}]'))
|
|
630
|
+
assert_equal expected, events('[[}]')
|
|
631
|
+
assert_equal expected, events('[[,]]')
|
|
632
|
+
end
|
|
633
|
+
|
|
634
|
+
it 'rejects malformed array value lists' do
|
|
635
|
+
expected = [:start_document, :start_array, [:value, "test"], :error]
|
|
636
|
+
assert_equal expected, events('["test"}')
|
|
637
|
+
assert_equal expected, events('["test",]')
|
|
638
|
+
assert_equal expected, events('["test" "test"]')
|
|
639
|
+
assert_equal expected, events('["test" 12]')
|
|
640
|
+
end
|
|
641
|
+
|
|
642
|
+
it 'parses array with value' do
|
|
643
|
+
expected = [:start_document, :start_array, [:value, "test"], :end_array, :end_document]
|
|
644
|
+
assert_equal expected, events('["test"]')
|
|
645
|
+
end
|
|
646
|
+
|
|
647
|
+
it 'parses array with value list' do
|
|
648
|
+
expected = [
|
|
649
|
+
:start_document,
|
|
650
|
+
:start_array,
|
|
651
|
+
[:value, 1],
|
|
652
|
+
[:value, 2],
|
|
653
|
+
[:value, nil],
|
|
654
|
+
[:value, 12.1],
|
|
655
|
+
[:value, "test"],
|
|
656
|
+
:end_array,
|
|
657
|
+
:end_document
|
|
658
|
+
]
|
|
659
|
+
assert_equal expected, events('[1,2, null, 12.1,"test"]')
|
|
660
|
+
end
|
|
661
|
+
end
|
|
662
|
+
|
|
663
|
+
describe 'parsing objects' do
|
|
664
|
+
it 'rejects malformed objects' do
|
|
665
|
+
expected = [:start_document, :start_object, :error]
|
|
666
|
+
assert_equal expected, events('{]')
|
|
667
|
+
assert_equal expected, events('{:}')
|
|
668
|
+
end
|
|
669
|
+
|
|
670
|
+
it 'parses single key object' do
|
|
671
|
+
expected = [:start_document, :start_object, [:key, "key 1"], [:value, 12], :end_object, :end_document]
|
|
672
|
+
assert_equal expected, events('{"key 1" : 12}')
|
|
673
|
+
end
|
|
674
|
+
|
|
675
|
+
it 'parses object key value list' do
|
|
676
|
+
expected = [
|
|
677
|
+
:start_document,
|
|
678
|
+
:start_object,
|
|
679
|
+
[:key, "key 1"], [:value, 12],
|
|
680
|
+
[:key, "key 2"], [:value, "two"],
|
|
681
|
+
:end_object,
|
|
682
|
+
:end_document
|
|
683
|
+
]
|
|
684
|
+
assert_equal expected, events('{"key 1" : 12, "key 2":"two"}')
|
|
685
|
+
end
|
|
686
|
+
|
|
687
|
+
it 'rejects object key with no value' do
|
|
688
|
+
expected = [
|
|
689
|
+
:start_document,
|
|
690
|
+
:start_object,
|
|
691
|
+
[:key, "key"],
|
|
692
|
+
:start_array,
|
|
693
|
+
[:value, nil],
|
|
694
|
+
[:value, false],
|
|
695
|
+
[:value, true],
|
|
696
|
+
:end_array,
|
|
697
|
+
[:key, "key 2"],
|
|
698
|
+
:error
|
|
699
|
+
]
|
|
700
|
+
assert_equal expected, events('{"key": [ null , false , true ] ,"key 2"}')
|
|
701
|
+
end
|
|
702
|
+
|
|
703
|
+
it 'rejects object with trailing comma' do
|
|
704
|
+
expected = [:start_document, :start_object, [:key, "key 1"], [:value, 12], :error]
|
|
705
|
+
assert_equal expected, events('{"key 1" : 12,}')
|
|
706
|
+
end
|
|
707
|
+
end
|
|
708
|
+
|
|
709
|
+
describe 'parsing unicode bytes' do
|
|
710
|
+
it 'parses single byte utf-8' do
|
|
711
|
+
expected = [:start_document, :start_array, [:value, "test"], :end_array, :end_document]
|
|
712
|
+
assert_equal expected, events('["test"]')
|
|
713
|
+
end
|
|
714
|
+
|
|
715
|
+
it 'parses full two byte utf-8' do
|
|
716
|
+
expected = [
|
|
717
|
+
:start_document,
|
|
718
|
+
:start_array,
|
|
719
|
+
[:value, "résumé"],
|
|
720
|
+
[:value, "éé"],
|
|
721
|
+
:end_array,
|
|
722
|
+
:end_document
|
|
723
|
+
]
|
|
724
|
+
assert_equal expected, events("[\"résumé\", \"é\xC3\xA9\"]")
|
|
725
|
+
end
|
|
726
|
+
|
|
727
|
+
# Parser should throw an error when only one byte of a two byte character
|
|
728
|
+
# is available. The \xC3 byte is the first byte of the é character.
|
|
729
|
+
it 'rejects a partial two byte utf-8 string' do
|
|
730
|
+
expected = [:start_document, :start_array, :error]
|
|
731
|
+
assert_equal expected, events("[\"\xC3\"]")
|
|
732
|
+
end
|
|
733
|
+
|
|
734
|
+
it 'parses valid two byte utf-8 string' do
|
|
735
|
+
expected = [:start_document, :start_array, [:value, 'é'], :end_array, :end_document]
|
|
736
|
+
assert_equal expected, events("[\"\xC3\xA9\"]")
|
|
737
|
+
end
|
|
738
|
+
|
|
739
|
+
it 'parses full three byte utf-8 string' do
|
|
740
|
+
expected = [
|
|
741
|
+
:start_document,
|
|
742
|
+
:start_array,
|
|
743
|
+
[:value, "snow\u2603man"],
|
|
744
|
+
[:value, "\u2603\u2603"],
|
|
745
|
+
:end_array,
|
|
746
|
+
:end_document
|
|
747
|
+
]
|
|
748
|
+
assert_equal expected, events("[\"snow\u2603man\", \"\u2603\u2603\"]")
|
|
749
|
+
end
|
|
750
|
+
|
|
751
|
+
it 'rejects one byte of three byte utf-8 string' do
|
|
752
|
+
expected = [:start_document, :start_array, :error]
|
|
753
|
+
assert_equal expected, events("[\"\xE2\"]")
|
|
754
|
+
end
|
|
755
|
+
|
|
756
|
+
it 'rejects two bytes of three byte utf-8 string' do
|
|
757
|
+
expected = [:start_document, :start_array, :error]
|
|
758
|
+
assert_equal expected, events("[\"\xE2\x98\"]")
|
|
759
|
+
end
|
|
760
|
+
|
|
761
|
+
it 'parses full three byte utf-8 string' do
|
|
762
|
+
expected = [:start_document, :start_array, [:value, "\u2603"], :end_array, :end_document]
|
|
763
|
+
assert_equal expected, events("[\"\xE2\x98\x83\"]")
|
|
764
|
+
end
|
|
765
|
+
|
|
766
|
+
it 'parses full four byte utf-8 string' do
|
|
767
|
+
expected = [
|
|
768
|
+
:start_document,
|
|
769
|
+
:start_array,
|
|
770
|
+
[:value, "\u{10102} check mark"],
|
|
771
|
+
:end_array,
|
|
772
|
+
:end_document
|
|
773
|
+
]
|
|
774
|
+
assert_equal expected, events("[\"\u{10102} check mark\"]")
|
|
775
|
+
end
|
|
776
|
+
|
|
777
|
+
it 'rejects one byte of four byte utf-8 string' do
|
|
778
|
+
expected = [:start_document, :start_array, :error]
|
|
779
|
+
assert_equal expected, events("[\"\xF0\"]")
|
|
780
|
+
end
|
|
781
|
+
|
|
782
|
+
it 'rejects two bytes of four byte utf-8 string' do
|
|
783
|
+
expected = [:start_document, :start_array, :error]
|
|
784
|
+
assert_equal expected, events("[\"\xF0\x90\"]")
|
|
785
|
+
end
|
|
786
|
+
|
|
787
|
+
it 'rejects three bytes of four byte utf-8 string' do
|
|
788
|
+
expected = [:start_document, :start_array, :error]
|
|
789
|
+
assert_equal expected, events("[\"\xF0\x90\x84\"]")
|
|
790
|
+
end
|
|
791
|
+
|
|
792
|
+
it 'parses full four byte utf-8 string' do
|
|
793
|
+
expected = [:start_document, :start_array, [:value, "\u{10102}"], :end_array, :end_document]
|
|
794
|
+
assert_equal expected, events("[\"\xF0\x90\x84\x82\"]")
|
|
795
|
+
end
|
|
796
|
+
end
|
|
797
|
+
|
|
798
|
+
describe 'parsing json text from the module' do
|
|
799
|
+
it 'parses an array document' do
|
|
800
|
+
result = JSON::Stream::Parser.parse('[1,2,3]')
|
|
801
|
+
assert_equal [1, 2, 3], result
|
|
802
|
+
end
|
|
803
|
+
|
|
804
|
+
it 'parses a true keyword literal document' do
|
|
805
|
+
result = JSON::Stream::Parser.parse('true')
|
|
806
|
+
assert_equal true, result
|
|
807
|
+
end
|
|
808
|
+
|
|
809
|
+
it 'parses a false keyword literal document' do
|
|
810
|
+
result = JSON::Stream::Parser.parse('false')
|
|
811
|
+
assert_equal false, result
|
|
812
|
+
end
|
|
813
|
+
|
|
814
|
+
it 'parses a null keyword literal document' do
|
|
815
|
+
result = JSON::Stream::Parser.parse('null')
|
|
816
|
+
assert_equal nil, result
|
|
817
|
+
end
|
|
818
|
+
|
|
819
|
+
it 'parses a string literal document' do
|
|
820
|
+
result = JSON::Stream::Parser.parse('"hello"')
|
|
821
|
+
assert_equal 'hello', result
|
|
822
|
+
end
|
|
823
|
+
|
|
824
|
+
it 'parses an integer literal document' do
|
|
825
|
+
result = JSON::Stream::Parser.parse('42')
|
|
826
|
+
assert_equal 42, result
|
|
827
|
+
end
|
|
828
|
+
|
|
829
|
+
it 'parses a float literal document' do
|
|
830
|
+
result = JSON::Stream::Parser.parse('42.12')
|
|
831
|
+
assert_equal 42.12, result
|
|
832
|
+
end
|
|
833
|
+
|
|
834
|
+
it 'rejects a partial float literal document' do
|
|
835
|
+
assert_raises(JSON::Stream::ParserError) do
|
|
836
|
+
JSON::Stream::Parser.parse('42.')
|
|
837
|
+
end
|
|
838
|
+
end
|
|
839
|
+
|
|
840
|
+
it 'rejects a partial document' do
|
|
841
|
+
assert_raises(JSON::Stream::ParserError) do
|
|
842
|
+
JSON::Stream::Parser.parse('{')
|
|
843
|
+
end
|
|
844
|
+
end
|
|
845
|
+
|
|
846
|
+
it 'rejects an empty document' do
|
|
847
|
+
assert_raises(JSON::Stream::ParserError) do
|
|
848
|
+
JSON::Stream::Parser.parse('')
|
|
849
|
+
end
|
|
850
|
+
end
|
|
851
|
+
end
|
|
852
|
+
|
|
853
|
+
it 'registers observers in initializer block' do
|
|
854
|
+
events = []
|
|
855
|
+
parser = JSON::Stream::Parser.new do
|
|
856
|
+
start_document { events << :start_document }
|
|
857
|
+
end_document { events << :end_document }
|
|
858
|
+
start_object { events << :start_object }
|
|
859
|
+
end_object { events << :end_object }
|
|
860
|
+
key {|k| events << [:key, k] }
|
|
861
|
+
value {|v| events << [:value, v] }
|
|
862
|
+
end
|
|
863
|
+
parser << '{"key":12}'
|
|
864
|
+
expected = [:start_document, :start_object, [:key, "key"], [:value, 12], :end_object, :end_document]
|
|
865
|
+
assert_equal expected, events
|
|
866
|
+
end
|
|
867
|
+
|
|
868
|
+
private
|
|
869
|
+
|
|
870
|
+
# Run a worst case, one byte at a time, parse against the JSON string and
|
|
871
|
+
# return a list of events generated by the parser. A special :error event is
|
|
872
|
+
# included if the parser threw an exception.
|
|
873
|
+
#
|
|
874
|
+
# json - The String to parse.
|
|
875
|
+
# parser - The optional Parser instance to use.
|
|
876
|
+
#
|
|
877
|
+
# Returns an Events instance.
|
|
878
|
+
def events(json, parser = nil)
|
|
879
|
+
parser ||= JSON::Stream::Parser.new
|
|
880
|
+
collector = Events.new(parser)
|
|
881
|
+
begin
|
|
882
|
+
json.each_byte {|byte| parser << [byte].pack('C') }
|
|
883
|
+
rescue JSON::Stream::ParserError
|
|
884
|
+
collector.error
|
|
885
|
+
end
|
|
886
|
+
collector.events
|
|
887
|
+
end
|
|
888
|
+
|
|
889
|
+
# Dynamically map methods in this class to parser callback methods
|
|
890
|
+
# so we can collect parser events for inspection by test cases.
|
|
891
|
+
class Events
|
|
892
|
+
METHODS = %w[start_document end_document start_object end_object start_array end_array key value]
|
|
893
|
+
|
|
894
|
+
attr_reader :events
|
|
895
|
+
|
|
896
|
+
def initialize(parser)
|
|
897
|
+
@events = []
|
|
898
|
+
METHODS.each do |name|
|
|
899
|
+
parser.send(name, &method(name))
|
|
900
|
+
end
|
|
901
|
+
end
|
|
902
|
+
|
|
903
|
+
METHODS.each do |name|
|
|
904
|
+
define_method(name) do |*args|
|
|
905
|
+
@events << (args.empty? ? name.to_sym : [name.to_sym, *args])
|
|
906
|
+
end
|
|
907
|
+
end
|
|
908
|
+
|
|
909
|
+
def error
|
|
910
|
+
@events << :error
|
|
911
|
+
end
|
|
912
|
+
end
|
|
913
|
+
end
|