nosj 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +112 -0
- data/Cargo.lock +3 -3
- data/README.md +220 -19
- data/ext/nosj/Cargo.toml +2 -2
- data/ext/nosj/fuzz/Cargo.toml +41 -0
- data/ext/nosj/fuzz/fuzz_targets/lines.rs +9 -0
- data/ext/nosj/fuzz/fuzz_targets/patch.rs +10 -0
- data/ext/nosj/fuzz/fuzz_targets/reformat.rs +10 -0
- data/ext/nosj/fuzz/src/lib.rs +48 -0
- data/ext/nosj/fuzz/src/prelude.rb +400 -0
- data/ext/nosj/src/errors.rs +171 -0
- data/ext/nosj/src/files.rs +29 -7
- data/ext/nosj/src/gen/errors.rs +2 -9
- data/ext/nosj/src/gen/mod.rs +156 -12
- data/ext/nosj/src/gen/opts.rs +88 -10
- data/ext/nosj/src/gen/ruby.rs +51 -0
- data/ext/nosj/src/gen/walker.rs +97 -59
- data/ext/nosj/src/lazy.rs +42 -22
- data/ext/nosj/src/lib.rs +30 -0
- data/ext/nosj/src/lines.rs +90 -0
- data/ext/nosj/src/parse.rs +55 -14
- data/ext/nosj/src/patch.rs +547 -0
- data/ext/nosj/src/pointer.rs +12 -5
- data/ext/nosj/src/reformat.rs +320 -0
- data/ext/nosj/src/sink.rs +11 -0
- data/ext/nosj/src/stats.rs +301 -0
- data/lib/nosj/json.rb +30 -6
- data/lib/nosj/multi_json.rb +1 -1
- data/lib/nosj/rails.rb +76 -0
- data/lib/nosj/version.rb +1 -1
- data/lib/nosj.rb +351 -5
- data/sig/nosj.rbs +66 -0
- metadata +16 -2
|
@@ -0,0 +1,400 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Differential checks for the extension fuzz targets. Only the native
|
|
4
|
+
# layer is loaded (Init_nosj, no lib/nosj.rb), so the error classes the
|
|
5
|
+
# extension looks up by name are defined here first.
|
|
6
|
+
module NOSJ
|
|
7
|
+
class Error < StandardError; end
|
|
8
|
+
class ParserError < Error; end
|
|
9
|
+
class GeneratorError < Error; end
|
|
10
|
+
class NestingError < Error; end
|
|
11
|
+
class PatchError < Error; end
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
module NOSJFuzz
|
|
15
|
+
# A reference-side rejection: the case must fail natively too.
|
|
16
|
+
RefFail = Class.new(StandardError)
|
|
17
|
+
|
|
18
|
+
PARSE_FAIL = [NOSJ::ParserError, NOSJ::NestingError].freeze
|
|
19
|
+
EDIT_FAIL = (PARSE_FAIL + [NOSJ::PatchError, NOSJ::GeneratorError,
|
|
20
|
+
KeyError, ArgumentError, TypeError]).freeze
|
|
21
|
+
PRETTY = {indent: " ", space: " ", object_nl: "\n", array_nl: "\n"}.freeze
|
|
22
|
+
# A copy op can double the document (copy root into a child), so a
|
|
23
|
+
# long op list can grow it exponentially; keep runs tractable.
|
|
24
|
+
MAX_PATCH_OPS = 12
|
|
25
|
+
|
|
26
|
+
module_function
|
|
27
|
+
|
|
28
|
+
def utf8(bytes) = bytes.dup.force_encoding(Encoding::UTF_8)
|
|
29
|
+
|
|
30
|
+
def try_parse(s, opts = nil)
|
|
31
|
+
[:ok, NOSJ.parse_native(s, opts)]
|
|
32
|
+
rescue *PARSE_FAIL
|
|
33
|
+
[:err, nil]
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# --- reformat: minify/reformat vs parse-then-generate ---------------
|
|
37
|
+
|
|
38
|
+
def reformat_case(bytes)
|
|
39
|
+
s = utf8(bytes)
|
|
40
|
+
status, value = try_parse(s)
|
|
41
|
+
|
|
42
|
+
unless NOSJ.valid_native(s, nil) == (status == :ok)
|
|
43
|
+
raise "valid? disagrees with parse on acceptance"
|
|
44
|
+
end
|
|
45
|
+
stats_ok = begin
|
|
46
|
+
NOSJ.stats_native(s, nil)
|
|
47
|
+
true
|
|
48
|
+
rescue *PARSE_FAIL
|
|
49
|
+
false
|
|
50
|
+
end
|
|
51
|
+
raise "stats disagrees with parse on acceptance" unless stats_ok == (status == :ok)
|
|
52
|
+
|
|
53
|
+
min_status, min = begin
|
|
54
|
+
[:ok, NOSJ.reformat_native(s, nil)]
|
|
55
|
+
rescue *PARSE_FAIL
|
|
56
|
+
[:err, nil]
|
|
57
|
+
rescue NOSJ::GeneratorError
|
|
58
|
+
[:generator, nil]
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
if status == :err
|
|
62
|
+
# The pipe may abort with GeneratorError (lone-surrogate key,
|
|
63
|
+
# non-finite float) before the parser reaches whatever made the
|
|
64
|
+
# whole document unparseable; any refusal is a refusal.
|
|
65
|
+
raise "reformat accepted what parse refused" unless [:err, :generator].include?(min_status)
|
|
66
|
+
return
|
|
67
|
+
end
|
|
68
|
+
if min_status == :generator
|
|
69
|
+
# On a document parse accepts, only an overflow-to-Infinity
|
|
70
|
+
# float (1e999, a 300-digit integer with a small exponent, maybe
|
|
71
|
+
# hidden behind a duplicate key parse would discard) or a
|
|
72
|
+
# lone-surrogate object key may abort the pipe. Rerunning with
|
|
73
|
+
# allow_nan separates them: it lifts the float refusal but not
|
|
74
|
+
# the key refusal, and the key requires a \u escape.
|
|
75
|
+
begin
|
|
76
|
+
NOSJ.reformat_native(s, {allow_nan: true})
|
|
77
|
+
rescue NOSJ::GeneratorError
|
|
78
|
+
raise "GeneratorError without a \\u escape in source" unless s.include?("\\u")
|
|
79
|
+
end
|
|
80
|
+
return
|
|
81
|
+
end
|
|
82
|
+
raise "reformat refused what parse accepted" unless min_status == :ok
|
|
83
|
+
|
|
84
|
+
reparsed = stage("reparse minify output #{min.inspect[0, 200]}") do
|
|
85
|
+
NOSJ.parse_native(min, nil)
|
|
86
|
+
end
|
|
87
|
+
raise "minify does not round-trip: #{min.inspect[0, 200]}" unless reparsed.eql?(value)
|
|
88
|
+
raise "minify is not idempotent" unless stage("re-minify") {
|
|
89
|
+
NOSJ.reformat_native(min, nil)
|
|
90
|
+
} == min
|
|
91
|
+
pretty = stage("pretty") { NOSJ.reformat_native(s, PRETTY) }
|
|
92
|
+
raise "pretty does not round-trip" unless stage("reparse pretty") {
|
|
93
|
+
NOSJ.parse_native(pretty, nil)
|
|
94
|
+
}.eql?(value)
|
|
95
|
+
nil
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Rebrand an exception from a must-succeed stage with its context;
|
|
99
|
+
# these are harness violations, not expected refusals.
|
|
100
|
+
def stage(name)
|
|
101
|
+
yield
|
|
102
|
+
rescue => e
|
|
103
|
+
raise "#{name} failed: #{e.class}: #{e.message}"
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# --- lines: each_line vs one parse per line -------------------------
|
|
107
|
+
|
|
108
|
+
def lines_case(bytes)
|
|
109
|
+
s = utf8(bytes)
|
|
110
|
+
yielded = []
|
|
111
|
+
status = begin
|
|
112
|
+
NOSJ.each_line_native(s, nil) { |v| yielded << v }
|
|
113
|
+
:ok
|
|
114
|
+
rescue *PARSE_FAIL
|
|
115
|
+
:err
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
unless s.valid_encoding?
|
|
119
|
+
raise "each_line accepted invalid UTF-8" unless status == :err
|
|
120
|
+
raise "each_line yielded before the encoding gate" unless yielded.empty?
|
|
121
|
+
return
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
reference = []
|
|
125
|
+
ref_status = :ok
|
|
126
|
+
s.split("\n", -1).each do |line|
|
|
127
|
+
next if line.match?(/\A[ \t\r]*\z/)
|
|
128
|
+
line_status, value = try_parse(line)
|
|
129
|
+
if line_status == :err
|
|
130
|
+
ref_status = :err
|
|
131
|
+
break
|
|
132
|
+
end
|
|
133
|
+
reference << value
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
unless status == ref_status
|
|
137
|
+
raise "acceptance: each_line #{status}, per-line parse #{ref_status}"
|
|
138
|
+
end
|
|
139
|
+
# On error the values yielded before the bad line must still match.
|
|
140
|
+
raise "yielded values diverge from per-line parses" unless yielded.eql?(reference)
|
|
141
|
+
|
|
142
|
+
if status == :ok && !yielded.empty?
|
|
143
|
+
begin
|
|
144
|
+
ndjson = NOSJ.generate_lines_native(yielded, nil)
|
|
145
|
+
rescue NOSJ::GeneratorError
|
|
146
|
+
return # WTF-8 strings (lone surrogates) are not generable
|
|
147
|
+
end
|
|
148
|
+
back = []
|
|
149
|
+
NOSJ.each_line_native(ndjson, nil) { |v| back << v }
|
|
150
|
+
raise "generate_lines does not round-trip" unless back.eql?(yielded)
|
|
151
|
+
end
|
|
152
|
+
nil
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# --- patch/splice: raw-byte editing vs tree editing -----------------
|
|
156
|
+
|
|
157
|
+
# Input is "document \0 spec"; a spec parsing to an object fuzzes
|
|
158
|
+
# splice (pointer => value), an array fuzzes RFC 6902 patch.
|
|
159
|
+
#
|
|
160
|
+
# The reference tree parses under the default nesting cap, so
|
|
161
|
+
# deeper-than-100 documents are exercised for crashes only, like
|
|
162
|
+
# malformed ones: resolution scans just the bytes some pointer needs
|
|
163
|
+
# (documented crate semantics), so the native side may legitimately
|
|
164
|
+
# succeed where whole-document parsing refuses.
|
|
165
|
+
def patch_case(bytes)
|
|
166
|
+
doc_bytes, sep, spec_bytes = bytes.partition("\0")
|
|
167
|
+
return if sep.empty?
|
|
168
|
+
doc = utf8(doc_bytes)
|
|
169
|
+
spec_status, spec = try_parse(utf8(spec_bytes))
|
|
170
|
+
return unless spec_status == :ok
|
|
171
|
+
# Replacement values with broken encoding (lone surrogates) raise
|
|
172
|
+
# GeneratorError on insertion; the reference cannot mirror that.
|
|
173
|
+
return unless deep_valid_encoding?(spec)
|
|
174
|
+
|
|
175
|
+
tree_status, tree = try_parse(doc)
|
|
176
|
+
# Duplicate keys: raw-byte resolution sees the first occurrence,
|
|
177
|
+
# tree materialization keeps the last; the two sides cannot agree.
|
|
178
|
+
if tree_status == :ok && NOSJ.stats_native(doc, nil)[:keys] != count_keys(tree)
|
|
179
|
+
return
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
case spec
|
|
183
|
+
when Hash
|
|
184
|
+
splice_case(doc, tree_status, tree, spec)
|
|
185
|
+
when Array
|
|
186
|
+
return if spec.length > MAX_PATCH_OPS
|
|
187
|
+
rfc6902_case(doc, tree_status, tree, spec)
|
|
188
|
+
end
|
|
189
|
+
nil
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def splice_case(doc, tree_status, tree, edits)
|
|
193
|
+
native_status, result = begin
|
|
194
|
+
[:ok, NOSJ.splice_native(doc, edits, nil)]
|
|
195
|
+
rescue *EDIT_FAIL
|
|
196
|
+
[:err, nil]
|
|
197
|
+
end
|
|
198
|
+
return unless tree_status == :ok
|
|
199
|
+
|
|
200
|
+
ref_status, ref = begin
|
|
201
|
+
[:ok, ref_splice(tree, edits)]
|
|
202
|
+
rescue RefFail
|
|
203
|
+
[:err, nil]
|
|
204
|
+
end
|
|
205
|
+
unless native_status == ref_status
|
|
206
|
+
raise "splice acceptance: native #{native_status}, reference #{ref_status}"
|
|
207
|
+
end
|
|
208
|
+
return unless native_status == :ok
|
|
209
|
+
got = NOSJ.parse_native(result, {max_nesting: false})
|
|
210
|
+
raise "splice result diverges from reference" unless got.eql?(ref)
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def rfc6902_case(doc, tree_status, tree, ops)
|
|
214
|
+
native_status, result = begin
|
|
215
|
+
[:ok, NOSJ.patch_native(doc, ops, nil)]
|
|
216
|
+
rescue *EDIT_FAIL
|
|
217
|
+
[:err, nil]
|
|
218
|
+
end
|
|
219
|
+
return unless tree_status == :ok
|
|
220
|
+
|
|
221
|
+
ref_status, ref = begin
|
|
222
|
+
[:ok, ref_patch(tree, ops)]
|
|
223
|
+
rescue RefFail
|
|
224
|
+
[:err, nil]
|
|
225
|
+
end
|
|
226
|
+
unless native_status == ref_status
|
|
227
|
+
raise "patch acceptance: native #{native_status}, reference #{ref_status}"
|
|
228
|
+
end
|
|
229
|
+
return unless native_status == :ok
|
|
230
|
+
got = NOSJ.parse_native(result, {max_nesting: false})
|
|
231
|
+
raise "patch result diverges from reference" unless got.eql?(ref)
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
# --- the pure-Ruby reference implementations ------------------------
|
|
235
|
+
|
|
236
|
+
def ref_splice(tree, edits)
|
|
237
|
+
token_lists = edits.keys.map { |ptr| ptr_tokens(ptr) }
|
|
238
|
+
token_lists.combination(2) do |a, b|
|
|
239
|
+
short, long = (a.length <= b.length) ? [a, b] : [b, a]
|
|
240
|
+
raise RefFail if long[0, short.length] == short
|
|
241
|
+
end
|
|
242
|
+
box = [tree]
|
|
243
|
+
edits.each_value.with_index do |value, i|
|
|
244
|
+
toks = token_lists[i]
|
|
245
|
+
if toks.empty?
|
|
246
|
+
box[0] = value
|
|
247
|
+
else
|
|
248
|
+
parent = ref_resolve(box[0], toks[0..-2])
|
|
249
|
+
case parent
|
|
250
|
+
when Hash
|
|
251
|
+
raise RefFail unless parent.key?(toks[-1])
|
|
252
|
+
parent[toks[-1]] = value
|
|
253
|
+
when Array
|
|
254
|
+
parent[strict_index(toks[-1], parent.size)] = value
|
|
255
|
+
else
|
|
256
|
+
raise RefFail
|
|
257
|
+
end
|
|
258
|
+
end
|
|
259
|
+
end
|
|
260
|
+
box[0]
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
def ref_patch(tree, ops)
|
|
264
|
+
box = [tree]
|
|
265
|
+
ops.each do |op|
|
|
266
|
+
raise RefFail unless op.is_a?(Hash)
|
|
267
|
+
name = op["op"]
|
|
268
|
+
raise RefFail unless name.is_a?(String)
|
|
269
|
+
path = op["path"]
|
|
270
|
+
raise RefFail unless path.is_a?(String)
|
|
271
|
+
case name
|
|
272
|
+
when "add" then ref_add(box, ptr_tokens(path), fetch_value(op))
|
|
273
|
+
when "replace" then ref_replace(box, ptr_tokens(path), fetch_value(op))
|
|
274
|
+
when "remove" then ref_remove(box, ptr_tokens(path))
|
|
275
|
+
when "move"
|
|
276
|
+
from = op["from"]
|
|
277
|
+
raise RefFail unless from.is_a?(String)
|
|
278
|
+
# The native side short-circuits these before validating the
|
|
279
|
+
# pointers, in this order; mirror it exactly.
|
|
280
|
+
next if path == from
|
|
281
|
+
raise RefFail if path.start_with?("#{from}/") || from.empty?
|
|
282
|
+
from_toks = ptr_tokens(from)
|
|
283
|
+
moved = ref_resolve(box[0], from_toks)
|
|
284
|
+
ref_remove(box, from_toks)
|
|
285
|
+
ref_add(box, ptr_tokens(path), moved)
|
|
286
|
+
when "copy"
|
|
287
|
+
from = op["from"]
|
|
288
|
+
raise RefFail unless from.is_a?(String)
|
|
289
|
+
ref_add(box, ptr_tokens(path), deep_dup(ref_resolve(box[0], ptr_tokens(from))))
|
|
290
|
+
when "test"
|
|
291
|
+
raise RefFail unless ref_resolve(box[0], ptr_tokens(path)) == fetch_value(op)
|
|
292
|
+
else
|
|
293
|
+
raise RefFail
|
|
294
|
+
end
|
|
295
|
+
end
|
|
296
|
+
box[0]
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
def ptr_tokens(ptr)
|
|
300
|
+
return [] if ptr.empty?
|
|
301
|
+
raise RefFail unless ptr.start_with?("/")
|
|
302
|
+
# RFC 6901 unescape order: ~1 before ~0, matching the native side.
|
|
303
|
+
ptr.split("/", -1).drop(1).map { |t| t.gsub("~1", "/").gsub("~0", "~") }
|
|
304
|
+
end
|
|
305
|
+
|
|
306
|
+
def strict_index(token, size, insert: false)
|
|
307
|
+
raise RefFail unless token.match?(/\A(?:0|[1-9][0-9]*)\z/)
|
|
308
|
+
i = token.to_i
|
|
309
|
+
in_range = i < size || (insert && i == size)
|
|
310
|
+
raise RefFail unless in_range
|
|
311
|
+
i
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
def ref_resolve(node, toks)
|
|
315
|
+
toks.each do |t|
|
|
316
|
+
case node
|
|
317
|
+
when Hash
|
|
318
|
+
raise RefFail unless node.key?(t)
|
|
319
|
+
node = node[t]
|
|
320
|
+
when Array
|
|
321
|
+
node = node[strict_index(t, node.size)]
|
|
322
|
+
else
|
|
323
|
+
raise RefFail
|
|
324
|
+
end
|
|
325
|
+
end
|
|
326
|
+
node
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
def ref_add(box, toks, value)
|
|
330
|
+
return box[0] = value if toks.empty?
|
|
331
|
+
parent = ref_resolve(box[0], toks[0..-2])
|
|
332
|
+
case parent
|
|
333
|
+
when Hash
|
|
334
|
+
parent[toks[-1]] = value
|
|
335
|
+
when Array
|
|
336
|
+
i = (toks[-1] == "-") ? parent.size : strict_index(toks[-1], parent.size, insert: true)
|
|
337
|
+
parent.insert(i, value)
|
|
338
|
+
else
|
|
339
|
+
raise RefFail
|
|
340
|
+
end
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
def ref_replace(box, toks, value)
|
|
344
|
+
return box[0] = value if toks.empty?
|
|
345
|
+
parent = ref_resolve(box[0], toks[0..-2])
|
|
346
|
+
case parent
|
|
347
|
+
when Hash
|
|
348
|
+
raise RefFail unless parent.key?(toks[-1])
|
|
349
|
+
parent[toks[-1]] = value
|
|
350
|
+
when Array
|
|
351
|
+
parent[strict_index(toks[-1], parent.size)] = value
|
|
352
|
+
else
|
|
353
|
+
raise RefFail
|
|
354
|
+
end
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
def ref_remove(box, toks)
|
|
358
|
+
raise RefFail if toks.empty?
|
|
359
|
+
parent = ref_resolve(box[0], toks[0..-2])
|
|
360
|
+
case parent
|
|
361
|
+
when Hash
|
|
362
|
+
raise RefFail unless parent.key?(toks[-1])
|
|
363
|
+
parent.delete(toks[-1])
|
|
364
|
+
when Array
|
|
365
|
+
parent.delete_at(strict_index(toks[-1], parent.size))
|
|
366
|
+
else
|
|
367
|
+
raise RefFail
|
|
368
|
+
end
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
def fetch_value(op)
|
|
372
|
+
raise RefFail unless op.key?("value")
|
|
373
|
+
op["value"]
|
|
374
|
+
end
|
|
375
|
+
|
|
376
|
+
def deep_dup(v)
|
|
377
|
+
case v
|
|
378
|
+
when Hash then v.transform_values { |e| deep_dup(e) }
|
|
379
|
+
when Array then v.map { |e| deep_dup(e) }
|
|
380
|
+
else v
|
|
381
|
+
end
|
|
382
|
+
end
|
|
383
|
+
|
|
384
|
+
def deep_valid_encoding?(v)
|
|
385
|
+
case v
|
|
386
|
+
when String then v.valid_encoding?
|
|
387
|
+
when Array then v.all? { |e| deep_valid_encoding?(e) }
|
|
388
|
+
when Hash then v.all? { |k, e| deep_valid_encoding?(k) && deep_valid_encoding?(e) }
|
|
389
|
+
else true
|
|
390
|
+
end
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
def count_keys(v)
|
|
394
|
+
case v
|
|
395
|
+
when Hash then v.size + v.sum { |_, e| count_keys(e) }
|
|
396
|
+
when Array then v.sum { |e| count_keys(e) }
|
|
397
|
+
else 0
|
|
398
|
+
end
|
|
399
|
+
end
|
|
400
|
+
end
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
//! Parse-side exception raising. Malformed documents raise
|
|
2
|
+
//! `NOSJ::ParserError` carrying the failure position (`#byte_offset`,
|
|
3
|
+
//! `#line`, `#column`) and a caret snippet (`#snippet`); too-deep
|
|
4
|
+
//! nesting raises `NOSJ::NestingError`, matching gem json's classes.
|
|
5
|
+
//!
|
|
6
|
+
//! Everything here runs only when a parse has already failed: the
|
|
7
|
+
//! position pass over the source and the exception construction cost
|
|
8
|
+
//! nothing on the happy path, and the accessors on the Ruby side are
|
|
9
|
+
//! plain ivar reads.
|
|
10
|
+
|
|
11
|
+
use magnus::value::ReprValue;
|
|
12
|
+
use magnus::{Class, Error, ExceptionClass, Module, Object, RModule, RObject, Ruby};
|
|
13
|
+
|
|
14
|
+
/// Look up an exception class under NOSJ, falling back to RuntimeError
|
|
15
|
+
/// if the Ruby layer has not defined it (only possible when the native
|
|
16
|
+
/// extension is loaded without lib/nosj.rb).
|
|
17
|
+
pub(crate) fn nosj_exception(ruby: &Ruby, name: &str) -> ExceptionClass {
|
|
18
|
+
let lookup = || -> Result<ExceptionClass, Error> {
|
|
19
|
+
let m: RModule = ruby.define_module("NOSJ")?;
|
|
20
|
+
m.const_get(name)
|
|
21
|
+
};
|
|
22
|
+
lookup().unwrap_or_else(|_| ruby.exception_runtime_error())
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/// NOSJ::ParserError without position info (encoding refusals and
|
|
26
|
+
/// other failures that have no meaningful offset).
|
|
27
|
+
#[cold]
|
|
28
|
+
pub(crate) fn parser_error(ruby: &Ruby, msg: String) -> Error {
|
|
29
|
+
Error::new(nosj_exception(ruby, "ParserError"), msg)
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/// NOSJ::NestingError (parse side: message parity with gem json, which
|
|
33
|
+
/// raises JSON::NestingError when max_nesting is exceeded).
|
|
34
|
+
#[cold]
|
|
35
|
+
pub(crate) fn nesting_error(ruby: &Ruby, msg: String) -> Error {
|
|
36
|
+
Error::new(nosj_exception(ruby, "NestingError"), msg)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/// Plain RuntimeError, for failures that are not parse errors (the
|
|
40
|
+
/// unmappable-I/O fallback).
|
|
41
|
+
#[cold]
|
|
42
|
+
pub(crate) fn runtime_error(ruby: &Ruby, msg: String) -> Error {
|
|
43
|
+
Error::new(ruby.exception_runtime_error(), msg)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/// NOSJ::ParserError enriched with the failure position: `offset` is a
|
|
47
|
+
/// byte offset into `source` (the full document, so positions stay
|
|
48
|
+
/// absolute even when the failing parse ran over a subtree slice).
|
|
49
|
+
#[cold]
|
|
50
|
+
pub(crate) fn parser_error_at(ruby: &Ruby, source: &[u8], offset: usize, msg: String) -> Error {
|
|
51
|
+
let class = nosj_exception(ruby, "ParserError");
|
|
52
|
+
let loc = locate(source, offset);
|
|
53
|
+
let Ok(exc) = class.new_instance((msg.as_str(),)) else {
|
|
54
|
+
return Error::new(class, msg);
|
|
55
|
+
};
|
|
56
|
+
// Exception instances are plain T_OBJECTs; magnus exposes ivar_set
|
|
57
|
+
// through RObject, not through its Exception wrapper.
|
|
58
|
+
if let Some(obj) = RObject::from_value(exc.as_value()) {
|
|
59
|
+
let set = |name: &str, v: usize| {
|
|
60
|
+
let _ = obj.ivar_set(name, v);
|
|
61
|
+
};
|
|
62
|
+
set("@byte_offset", offset.min(source.len()));
|
|
63
|
+
set("@line", loc.line);
|
|
64
|
+
set("@column", loc.column);
|
|
65
|
+
if let Some(snippet) = &loc.snippet {
|
|
66
|
+
let _ = obj.ivar_set("@snippet", snippet.as_str());
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
Error::from(exc)
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/// A resolved source position: 1-based line, 1-based character column
|
|
73
|
+
/// within that line, and a two-line caret snippet (`None` when the
|
|
74
|
+
/// offending line is empty or not valid UTF-8).
|
|
75
|
+
struct Location {
|
|
76
|
+
line: usize,
|
|
77
|
+
column: usize,
|
|
78
|
+
snippet: Option<String>,
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/// UTF-8 continuation bytes are 0b10xxxxxx; every other byte starts a
|
|
82
|
+
/// character, so counting non-continuation bytes counts characters.
|
|
83
|
+
const UTF8_CONTINUATION_MASK: u8 = 0b1100_0000;
|
|
84
|
+
const UTF8_CONTINUATION_BITS: u8 = 0b1000_0000;
|
|
85
|
+
|
|
86
|
+
fn count_chars(bytes: &[u8]) -> usize {
|
|
87
|
+
bytes
|
|
88
|
+
.iter()
|
|
89
|
+
.filter(|&&b| b & UTF8_CONTINUATION_MASK != UTF8_CONTINUATION_BITS)
|
|
90
|
+
.count()
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
fn locate(source: &[u8], offset: usize) -> Location {
|
|
94
|
+
let off = offset.min(source.len());
|
|
95
|
+
let line = 1 + count_newlines(&source[..off]);
|
|
96
|
+
let line_start = source[..off]
|
|
97
|
+
.iter()
|
|
98
|
+
.rposition(|&b| b == b'\n')
|
|
99
|
+
.map_or(0, |p| p + 1);
|
|
100
|
+
let mut line_end = source[off..]
|
|
101
|
+
.iter()
|
|
102
|
+
.position(|&b| b == b'\n')
|
|
103
|
+
.map_or(source.len(), |p| off + p);
|
|
104
|
+
if line_end > line_start && source[line_end - 1] == b'\r' {
|
|
105
|
+
line_end -= 1;
|
|
106
|
+
}
|
|
107
|
+
// An offset sitting on the line terminator itself carets one past
|
|
108
|
+
// the last character of the line's content.
|
|
109
|
+
let caret = off.clamp(line_start, line_end);
|
|
110
|
+
let column = 1 + count_chars(&source[line_start..caret]);
|
|
111
|
+
let snippet = std::str::from_utf8(&source[line_start..line_end])
|
|
112
|
+
.ok()
|
|
113
|
+
.and_then(|l| build_snippet(l, caret - line_start));
|
|
114
|
+
Location {
|
|
115
|
+
line,
|
|
116
|
+
column,
|
|
117
|
+
snippet,
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
fn count_newlines(bytes: &[u8]) -> usize {
|
|
122
|
+
bytes.iter().filter(|&&b| b == b'\n').count()
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/// Characters kept around the caret. Minified JSON is routinely one
|
|
126
|
+
/// multi-kilobyte line, so the snippet shows a window, not the line.
|
|
127
|
+
const SNIPPET_CHARS_BEFORE: usize = 50;
|
|
128
|
+
const SNIPPET_CHARS_AFTER: usize = 30;
|
|
129
|
+
const SNIPPET_ELLIPSIS: char = '…';
|
|
130
|
+
|
|
131
|
+
/// Render the offending line plus a caret line underneath, windowed
|
|
132
|
+
/// around `caret_byte` (a byte offset into `line`).
|
|
133
|
+
fn build_snippet(line: &str, caret_byte: usize) -> Option<String> {
|
|
134
|
+
if line.is_empty() {
|
|
135
|
+
return None;
|
|
136
|
+
}
|
|
137
|
+
let mut caret_byte = caret_byte.min(line.len());
|
|
138
|
+
while !line.is_char_boundary(caret_byte) {
|
|
139
|
+
caret_byte -= 1;
|
|
140
|
+
}
|
|
141
|
+
let caret_chars = line[..caret_byte].chars().count();
|
|
142
|
+
let total_chars = caret_chars + line[caret_byte..].chars().count();
|
|
143
|
+
|
|
144
|
+
let window_first = caret_chars.saturating_sub(SNIPPET_CHARS_BEFORE);
|
|
145
|
+
let window_last = (caret_chars + SNIPPET_CHARS_AFTER).min(total_chars);
|
|
146
|
+
let mut caret_column = caret_chars - window_first;
|
|
147
|
+
|
|
148
|
+
let mut out = String::new();
|
|
149
|
+
if window_first > 0 {
|
|
150
|
+
out.push(SNIPPET_ELLIPSIS);
|
|
151
|
+
caret_column += 1;
|
|
152
|
+
}
|
|
153
|
+
for ch in line
|
|
154
|
+
.chars()
|
|
155
|
+
.skip(window_first)
|
|
156
|
+
.take(window_last - window_first)
|
|
157
|
+
{
|
|
158
|
+
// Control characters (tabs included) render as one space so
|
|
159
|
+
// the caret column stays aligned with what is printed.
|
|
160
|
+
out.push(if ch.is_control() { ' ' } else { ch });
|
|
161
|
+
}
|
|
162
|
+
if window_last < total_chars {
|
|
163
|
+
out.push(SNIPPET_ELLIPSIS);
|
|
164
|
+
}
|
|
165
|
+
out.push('\n');
|
|
166
|
+
for _ in 0..caret_column {
|
|
167
|
+
out.push(' ');
|
|
168
|
+
}
|
|
169
|
+
out.push('^');
|
|
170
|
+
Some(out)
|
|
171
|
+
}
|
data/ext/nosj/src/files.rs
CHANGED
|
@@ -17,9 +17,10 @@ use std::io::Read;
|
|
|
17
17
|
use magnus::value::ReprValue;
|
|
18
18
|
use magnus::{Error, RArray, RString, Ruby, Value};
|
|
19
19
|
|
|
20
|
+
use crate::errors::{parser_error_at, runtime_error};
|
|
20
21
|
use crate::gen;
|
|
21
22
|
use crate::lazy::{self, DocBytes};
|
|
22
|
-
use crate::parse::{err, materialize, parse_native_opts, ParseNativeOpts};
|
|
23
|
+
use crate::parse::{err, materialize, materialize_at, parse_native_opts, span_of, ParseNativeOpts};
|
|
23
24
|
use crate::pointer::path_to_pointer;
|
|
24
25
|
use crate::state::PULL_STATE;
|
|
25
26
|
|
|
@@ -60,16 +61,16 @@ fn errno_of(e: &std::io::Error) -> Option<i32> {
|
|
|
60
61
|
fn io_error(ruby: &Ruby, path: &str, e: &std::io::Error) -> Error {
|
|
61
62
|
use magnus::rb_sys::FromRawValue;
|
|
62
63
|
let Some(errno) = errno_of(e) else {
|
|
63
|
-
return
|
|
64
|
+
return runtime_error(ruby, format!("{e} - {path}"));
|
|
64
65
|
};
|
|
65
66
|
let Ok(cpath) = std::ffi::CString::new(path) else {
|
|
66
|
-
return
|
|
67
|
+
return runtime_error(ruby, format!("{e} - {path}"));
|
|
67
68
|
};
|
|
68
69
|
// SAFETY: rb_syserr_new returns a live Errno exception instance.
|
|
69
70
|
let exc = unsafe { Value::from_raw(rb_sys::rb_syserr_new(errno, cpath.as_ptr())) };
|
|
70
71
|
match magnus::Exception::from_value(exc) {
|
|
71
72
|
Some(exc) => exc.into(),
|
|
72
|
-
None =>
|
|
73
|
+
None => runtime_error(ruby, format!("{e} - {path}")),
|
|
73
74
|
}
|
|
74
75
|
}
|
|
75
76
|
|
|
@@ -128,8 +129,26 @@ pub fn write_file_native(
|
|
|
128
129
|
})
|
|
129
130
|
}
|
|
130
131
|
|
|
132
|
+
/// `NOSJ.write_lines(path, values, opts)`: generate NDJSON (one
|
|
133
|
+
/// document per element, newline-terminated) and write the bytes
|
|
134
|
+
/// straight from the generator's pooled buffer. Returns the byte
|
|
135
|
+
/// count, like `File.write`.
|
|
136
|
+
pub fn write_lines_native(
|
|
137
|
+
ruby: &Ruby,
|
|
138
|
+
_rb_self: Value,
|
|
139
|
+
path: RString,
|
|
140
|
+
values: magnus::RArray,
|
|
141
|
+
opts: Value,
|
|
142
|
+
) -> Result<usize, Error> {
|
|
143
|
+
let p = path.to_string()?;
|
|
144
|
+
gen::generate_lines_bytes_into(ruby, values, opts, |ruby, bytes| {
|
|
145
|
+
fs::write(&p, bytes).map_err(|e| io_error(ruby, &p, &e))?;
|
|
146
|
+
Ok(bytes.len())
|
|
147
|
+
})
|
|
148
|
+
}
|
|
149
|
+
|
|
131
150
|
/// Map `path` read-only and hand a UTF-8-checked view to `f`.
|
|
132
|
-
fn with_mapped_file<R>(
|
|
151
|
+
pub(crate) fn with_mapped_file<R>(
|
|
133
152
|
ruby: &Ruby,
|
|
134
153
|
path: &str,
|
|
135
154
|
f: impl FnOnce(memmap2::Mmap) -> Result<R, Error>,
|
|
@@ -178,11 +197,14 @@ fn resolve_file_pointer(
|
|
|
178
197
|
});
|
|
179
198
|
match resolved {
|
|
180
199
|
Ok(None) => Ok(ruby.qnil().as_value()),
|
|
181
|
-
Ok(Some(slice)) =>
|
|
200
|
+
Ok(Some(slice)) => {
|
|
201
|
+
let (start, end) = span_of(&map, slice.as_bytes());
|
|
202
|
+
materialize_at(ruby, &map, start, end, o)
|
|
203
|
+
}
|
|
182
204
|
Err(e) if matches!(e.kind, nosj::ErrorKind::InvalidPointer) => {
|
|
183
205
|
Err(Error::new(ruby.exception_arg_error(), e.to_string()))
|
|
184
206
|
}
|
|
185
|
-
Err(e) => Err(
|
|
207
|
+
Err(e) => Err(parser_error_at(ruby, &map, e.offset, e.to_string())),
|
|
186
208
|
}
|
|
187
209
|
})
|
|
188
210
|
}
|
data/ext/nosj/src/gen/errors.rs
CHANGED
|
@@ -5,9 +5,10 @@
|
|
|
5
5
|
use magnus::error::ErrorType;
|
|
6
6
|
use magnus::rb_sys::AsRawValue;
|
|
7
7
|
use magnus::value::ReprValue;
|
|
8
|
-
use magnus::{Error,
|
|
8
|
+
use magnus::{Error, Ruby};
|
|
9
9
|
|
|
10
10
|
use super::ruby::rstring_bytes;
|
|
11
|
+
use crate::errors::nosj_exception;
|
|
11
12
|
|
|
12
13
|
pub(super) enum GenFail {
|
|
13
14
|
/// Re-raise a Ruby exception captured by a protected call (user
|
|
@@ -22,14 +23,6 @@ pub(super) enum GenFail {
|
|
|
22
23
|
Nesting(usize),
|
|
23
24
|
}
|
|
24
25
|
|
|
25
|
-
fn nosj_exception(ruby: &Ruby, name: &str) -> ExceptionClass {
|
|
26
|
-
let lookup = || -> Result<ExceptionClass, Error> {
|
|
27
|
-
let m: RModule = ruby.define_module("NOSJ")?;
|
|
28
|
-
m.const_get(name)
|
|
29
|
-
};
|
|
30
|
-
lookup().unwrap_or_else(|_| ruby.exception_runtime_error())
|
|
31
|
-
}
|
|
32
|
-
|
|
33
26
|
/// The exception's `to_s` (its message), matching what the gem embeds
|
|
34
27
|
/// when it wraps a secondary exception.
|
|
35
28
|
fn error_message(err: &Error) -> String {
|