nosj 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +110 -1
- data/Cargo.lock +13 -3
- data/README.md +272 -24
- data/ext/nosj/Cargo.toml +5 -2
- data/ext/nosj/src/errors.rs +171 -0
- data/ext/nosj/src/files.rs +240 -0
- data/ext/nosj/src/gen/errors.rs +2 -9
- data/ext/nosj/src/gen/mod.rs +198 -20
- data/ext/nosj/src/gen/opts.rs +87 -9
- data/ext/nosj/src/gen/ruby.rs +51 -0
- data/ext/nosj/src/gen/walker.rs +97 -59
- data/ext/nosj/src/lazy.rs +422 -0
- data/ext/nosj/src/lib.rs +60 -0
- data/ext/nosj/src/lines.rs +90 -0
- data/ext/nosj/src/parse.rs +50 -14
- data/ext/nosj/src/patch.rs +547 -0
- data/ext/nosj/src/pointer.rs +26 -17
- data/ext/nosj/src/reformat.rs +301 -0
- data/ext/nosj/src/sink.rs +6 -0
- data/ext/nosj/src/stats.rs +301 -0
- data/lib/nosj/json.rb +30 -6
- data/lib/nosj/lazy.rb +165 -0
- data/lib/nosj/multi_json.rb +1 -1
- data/lib/nosj/rails.rb +76 -0
- data/lib/nosj/version.rb +1 -1
- data/lib/nosj.rb +434 -5
- data/sig/nosj.rbs +120 -0
- metadata +16 -3
data/lib/nosj.rb
CHANGED
|
@@ -2,13 +2,15 @@
|
|
|
2
2
|
|
|
3
3
|
require_relative "nosj/version"
|
|
4
4
|
require_relative "nosj/native"
|
|
5
|
+
require_relative "nosj/lazy"
|
|
5
6
|
|
|
6
7
|
# nosj is the evil twin of the +json+ gem: the same API, output bytes,
|
|
7
8
|
# option names, and error messages, backed by the Rust
|
|
8
9
|
# {https://github.com/yaroslav/nosj nosj} crate with SIMD-accelerated
|
|
9
10
|
# parsing and generation. Beyond the +json+ gem's surface it adds
|
|
10
|
-
# zero-allocation validation ({.valid?})
|
|
11
|
-
# ({.dig}, {.at_pointer}, and their batch forms)
|
|
11
|
+
# zero-allocation validation ({.valid?}), partial parsing
|
|
12
|
+
# ({.dig}, {.at_pointer}, and their batch forms), and lazy documents
|
|
13
|
+
# ({.lazy}).
|
|
12
14
|
#
|
|
13
15
|
# Options arrive as a positional Hash (the +json+ gem's own calling
|
|
14
16
|
# convention); an explicit +**kwargs+ would allocate per call.
|
|
@@ -24,15 +26,58 @@ module NOSJ
|
|
|
24
26
|
# Base class for nosj errors.
|
|
25
27
|
class Error < StandardError; end
|
|
26
28
|
|
|
29
|
+
# Raised when a document cannot be parsed. Carries the failure
|
|
30
|
+
# position, computed once when the parse fails (successful parses
|
|
31
|
+
# never pay for it): {#byte_offset}, 1-based {#line}, character-based
|
|
32
|
+
# {#column}, and a caret {#snippet} pointing at the offending spot.
|
|
33
|
+
# Positions are absolute within the document you passed, including
|
|
34
|
+
# through partial parsing ({NOSJ.dig}, {NOSJ.at_pointer}, lazy
|
|
35
|
+
# documents) and the file APIs. All four are +nil+ for failures
|
|
36
|
+
# without a position (encoding refusals).
|
|
37
|
+
#
|
|
38
|
+
# @example
|
|
39
|
+
# NOSJ.parse(%({\n "a": 1,\n "b": }))
|
|
40
|
+
# # => NOSJ::ParserError, with
|
|
41
|
+
# # e.line #=> 3
|
|
42
|
+
# # e.column #=> 8
|
|
43
|
+
# # e.snippet #=> " \"b\": }\n ^"
|
|
44
|
+
class ParserError < Error
|
|
45
|
+
# @return [Integer, nil] byte offset of the failure in the source
|
|
46
|
+
attr_reader :byte_offset
|
|
47
|
+
# @return [Integer, nil] 1-based line of the failure
|
|
48
|
+
attr_reader :line
|
|
49
|
+
# @return [Integer, nil] 1-based character (not byte) column within
|
|
50
|
+
# {#line}
|
|
51
|
+
attr_reader :column
|
|
52
|
+
# @return [String, nil] the offending line (windowed when long)
|
|
53
|
+
# with a caret line underneath
|
|
54
|
+
attr_reader :snippet
|
|
55
|
+
|
|
56
|
+
# The default message plus {#snippet}: Ruby prints
|
|
57
|
+
# +detailed_message+ when an exception reaches the top level, so an
|
|
58
|
+
# unrescued parse error shows where the document broke.
|
|
59
|
+
def detailed_message(highlight: false, **opts)
|
|
60
|
+
base = super
|
|
61
|
+
snippet ? "#{base}\n#{snippet}" : base
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
27
65
|
# Raised when a value cannot be generated (non-finite floats without
|
|
28
66
|
# +allow_nan+, unsupported objects under +strict+, broken encodings).
|
|
29
67
|
# Message-compatible with +JSON::GeneratorError+.
|
|
30
68
|
class GeneratorError < Error; end
|
|
31
69
|
|
|
32
|
-
# Raised when generation exceeds +max_nesting+.
|
|
33
|
-
# with +JSON::NestingError+.
|
|
70
|
+
# Raised when parsing or generation exceeds +max_nesting+.
|
|
71
|
+
# Message-compatible with +JSON::NestingError+.
|
|
34
72
|
class NestingError < Error; end
|
|
35
73
|
|
|
74
|
+
# Raised when an RFC 6902 patch cannot be applied: a +test+ operation
|
|
75
|
+
# failed, a target or source path does not exist, an array index is
|
|
76
|
+
# out of range, or a +move+ targets its own child. Structurally
|
|
77
|
+
# malformed patch documents (not an op Hash, unknown op, missing
|
|
78
|
+
# fields) raise ArgumentError instead.
|
|
79
|
+
class PatchError < Error; end
|
|
80
|
+
|
|
36
81
|
PRETTY_GENERATE_OPTS = {
|
|
37
82
|
indent: " ", space: " ", object_nl: "\n", array_nl: "\n"
|
|
38
83
|
}.freeze
|
|
@@ -55,7 +100,9 @@ module NOSJ
|
|
|
55
100
|
# (Integer or +false+ for unlimited), +allow_nan+,
|
|
56
101
|
# +allow_trailing_comma+
|
|
57
102
|
# @return [Object] the parsed value tree
|
|
58
|
-
# @raise [
|
|
103
|
+
# @raise [ParserError] when the document is malformed or not UTF-8;
|
|
104
|
+
# carries the failure position ({ParserError#line} and friends)
|
|
105
|
+
# @raise [NestingError] when nesting exceeds +max_nesting+
|
|
59
106
|
# @raise [ArgumentError] for unsupported options
|
|
60
107
|
def self.parse(source, opts = nil)
|
|
61
108
|
parse_native(source, opts)
|
|
@@ -190,4 +237,386 @@ module NOSJ
|
|
|
190
237
|
def self.at_pointers(source, pointers, opts = nil)
|
|
191
238
|
at_pointers_native(source, pointers, opts)
|
|
192
239
|
end
|
|
240
|
+
|
|
241
|
+
# Parses a JSON file, like +JSON.load_file+—except the file is read
|
|
242
|
+
# natively into a reused buffer, so no file-sized Ruby String is ever
|
|
243
|
+
# created (or garbage-collected).
|
|
244
|
+
#
|
|
245
|
+
# @example
|
|
246
|
+
# NOSJ.load_file("config.json", symbolize_names: true)
|
|
247
|
+
#
|
|
248
|
+
# @param path [String] the file to parse (UTF-8)
|
|
249
|
+
# @param opts [Hash, nil] the same options as {.parse}
|
|
250
|
+
# @return [Object] the parsed value tree
|
|
251
|
+
# @raise [SystemCallError] +Errno::ENOENT+ and friends, like File.read
|
|
252
|
+
# @raise [ParserError] when the document is malformed or not UTF-8
|
|
253
|
+
def self.load_file(path, opts = nil)
|
|
254
|
+
load_file_native(path, opts)
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
# Generates +obj+ as JSON and writes it to +path+, streaming the
|
|
258
|
+
# generator's buffer straight to disk—no intermediate Ruby String.
|
|
259
|
+
#
|
|
260
|
+
# @example
|
|
261
|
+
# NOSJ.write_file("out.json", {"a" => [1, true]}) #=> 14
|
|
262
|
+
# NOSJ.write_file("pretty.json", obj, indent: " ", object_nl: "\n")
|
|
263
|
+
#
|
|
264
|
+
# @param path [String] the file to (over)write
|
|
265
|
+
# @param obj [Object] the value tree to generate
|
|
266
|
+
# @param opts [Hash, nil] the same options as {.generate}
|
|
267
|
+
# @return [Integer] the number of bytes written, like File.write
|
|
268
|
+
# @raise [SystemCallError] +Errno::ENOENT+ and friends, like File.write
|
|
269
|
+
# @raise [GeneratorError] like {.generate}
|
|
270
|
+
def self.write_file(path, obj, opts = nil)
|
|
271
|
+
write_file_native(path, obj, opts)
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
# Wraps a JSON file as a lazy document ({.lazy} for files): the file
|
|
275
|
+
# is memory-mapped read-only, so beyond one sequential UTF-8 check,
|
|
276
|
+
# pages you never read are never loaded from disk. The mapping lives
|
|
277
|
+
# as long as any node on it; the file must not be modified while it
|
|
278
|
+
# is in use.
|
|
279
|
+
#
|
|
280
|
+
# @example
|
|
281
|
+
# doc = NOSJ.load_lazy_file("huge.json")
|
|
282
|
+
# doc["users"][3]["name"] # touches only these pages
|
|
283
|
+
#
|
|
284
|
+
# @param path [String] the file to wrap (UTF-8)
|
|
285
|
+
# @param opts [Hash, nil] {.parse} options applied on materialization
|
|
286
|
+
# @return [NOSJ::Lazy, Object]
|
|
287
|
+
# @raise [SystemCallError] +Errno::ENOENT+ and friends
|
|
288
|
+
# @raise [ParserError] when the file is not UTF-8 or the root is malformed
|
|
289
|
+
def self.load_lazy_file(path, opts = nil)
|
|
290
|
+
load_lazy_file_native(path, opts)
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
# {.at_pointer} against a file: memory-maps it, resolves the pointer,
|
|
294
|
+
# materializes only the matched subtree, and never reads the rest
|
|
295
|
+
# into Ruby.
|
|
296
|
+
#
|
|
297
|
+
# @example
|
|
298
|
+
# NOSJ.at_pointer_file("huge.json", "/users/3/name")
|
|
299
|
+
#
|
|
300
|
+
# @param path [String] the file to query (UTF-8)
|
|
301
|
+
# @param pointer [String] an RFC 6901 JSON Pointer
|
|
302
|
+
# @param opts [Hash, nil] materialization options
|
|
303
|
+
# @return [Object, nil] nil when the pointer misses
|
|
304
|
+
# @raise [ArgumentError] for malformed pointers
|
|
305
|
+
def self.at_pointer_file(path, pointer, opts = nil)
|
|
306
|
+
at_pointer_file_native(path, pointer, opts)
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
# {.dig} against a file: the Hash#dig-shaped counterpart of
|
|
310
|
+
# {.at_pointer_file}. Negative indices resolve to nil.
|
|
311
|
+
#
|
|
312
|
+
# @example
|
|
313
|
+
# NOSJ.dig_file("huge.json", "users", 3, "name")
|
|
314
|
+
#
|
|
315
|
+
# @param path [String] the file to query (UTF-8)
|
|
316
|
+
# @param path_elements [Array<String, Symbol, Integer>]
|
|
317
|
+
# @return [Object, nil]
|
|
318
|
+
def self.dig_file(path, *path_elements)
|
|
319
|
+
dig_file_native(path, path_elements)
|
|
320
|
+
end
|
|
321
|
+
|
|
322
|
+
# Minifies a document without building any Ruby values: the parser's
|
|
323
|
+
# events pipe straight into the emission kernels, SIMD in and SIMD
|
|
324
|
+
# out. Output is exactly what <code>generate(parse(json))</code>
|
|
325
|
+
# would produce, except duplicate object keys pass through instead of
|
|
326
|
+
# being collapsed (a reformatter must not silently drop data).
|
|
327
|
+
# Numbers come out in the canonical spelling (+1.50+ becomes +1.5+)
|
|
328
|
+
# and string escapes are normalized.
|
|
329
|
+
#
|
|
330
|
+
# @example
|
|
331
|
+
# NOSJ.minify(%({ "a": [1, 2],\n "b": "x" })) #=> '{"a":[1,2],"b":"x"}'
|
|
332
|
+
#
|
|
333
|
+
# @param json [String] the document (UTF-8 or US-ASCII)
|
|
334
|
+
# @param opts [Hash, nil] acceptance options (+allow_nan+,
|
|
335
|
+
# +allow_trailing_comma+, +max_nesting+); trailing commas are
|
|
336
|
+
# normalized away when accepted
|
|
337
|
+
# @return [String] the minified document
|
|
338
|
+
# @raise [ParserError] when the document is malformed
|
|
339
|
+
# @raise [NestingError] past +max_nesting+
|
|
340
|
+
def self.minify(json, opts = nil)
|
|
341
|
+
reformat_native(json, opts)
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
# Reformats a document without building any Ruby values: {.minify}'s
|
|
345
|
+
# pipe with formatting. <code>pretty: true</code> is a shorthand for
|
|
346
|
+
# {.pretty_generate}'s layout; the individual {.generate} formatting
|
|
347
|
+
# and escape options (+indent+, +space+, +object_nl+, +ascii_only+,
|
|
348
|
+
# +script_safe+, ...) compose with it and win over it.
|
|
349
|
+
#
|
|
350
|
+
# @example
|
|
351
|
+
# NOSJ.reformat(json, pretty: true)
|
|
352
|
+
# NOSJ.reformat(json, ascii_only: true) # escape-transcode, compact
|
|
353
|
+
#
|
|
354
|
+
# @param json [String] the document (UTF-8 or US-ASCII)
|
|
355
|
+
# @param opts [Hash, nil] +pretty+, {.generate} formatting/escape
|
|
356
|
+
# options, and {.minify}'s acceptance options
|
|
357
|
+
# @return [String] the reformatted document
|
|
358
|
+
# @raise [ParserError] when the document is malformed
|
|
359
|
+
# @raise [NestingError] past +max_nesting+
|
|
360
|
+
# @raise [GeneratorError] when +ascii_only+ meets a lone-surrogate
|
|
361
|
+
# string it cannot represent
|
|
362
|
+
def self.reformat(json, opts = nil)
|
|
363
|
+
if opts&.key?(:pretty)
|
|
364
|
+
pretty = opts[:pretty]
|
|
365
|
+
opts = opts.except(:pretty)
|
|
366
|
+
opts = PRETTY_GENERATE_OPTS.merge(opts) if pretty
|
|
367
|
+
end
|
|
368
|
+
reformat_native(json, opts)
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
# {.reformat} against a file: the pipe runs over a read-only memory
|
|
372
|
+
# map, so the input document never becomes a Ruby String; only the
|
|
373
|
+
# result does.
|
|
374
|
+
#
|
|
375
|
+
# @example
|
|
376
|
+
# compact = NOSJ.reformat_file("big.json") # minify
|
|
377
|
+
# pretty = NOSJ.reformat_file("big.json", pretty: true)
|
|
378
|
+
#
|
|
379
|
+
# @param path [String] the file to reformat (UTF-8)
|
|
380
|
+
# @param opts [Hash, nil] same options as {.reformat}
|
|
381
|
+
# @return [String] the reformatted document
|
|
382
|
+
# @raise [SystemCallError] +Errno::ENOENT+ and friends
|
|
383
|
+
# @raise [ParserError] when the file is malformed or not UTF-8
|
|
384
|
+
def self.reformat_file(path, opts = nil)
|
|
385
|
+
if opts&.key?(:pretty)
|
|
386
|
+
pretty = opts[:pretty]
|
|
387
|
+
opts = opts.except(:pretty)
|
|
388
|
+
opts = PRETTY_GENERATE_OPTS.merge(opts) if pretty
|
|
389
|
+
end
|
|
390
|
+
reformat_file_native(path, opts)
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
# Byte-splicing edits: replaces the values at the given JSON Pointers
|
|
394
|
+
# directly in the text. Every target resolves in ONE forward pass
|
|
395
|
+
# (skipping, not parsing), and the result is built in one sweep:
|
|
396
|
+
# every byte outside the target spans is copied untouched, so
|
|
397
|
+
# formatting, key order, and number spellings elsewhere are preserved
|
|
398
|
+
# exactly. For tweaking a field in a passing payload this replaces
|
|
399
|
+
# the whole parse → mutate → generate cycle.
|
|
400
|
+
#
|
|
401
|
+
# @example
|
|
402
|
+
# NOSJ.splice(json, "/config/timeout" => 30)
|
|
403
|
+
# NOSJ.splice(json, "/a" => 1, "/b/c" => [true]) # batch, one pass
|
|
404
|
+
#
|
|
405
|
+
# @param json [String] the document (UTF-8 or US-ASCII)
|
|
406
|
+
# @param edits [Hash{String => Object}] JSON Pointer => replacement
|
|
407
|
+
# value (generated compactly, byte-identical to {.generate})
|
|
408
|
+
# @param opts [Hash, nil] {.generate} options for the inserted values
|
|
409
|
+
# @return [String] the edited document
|
|
410
|
+
# @raise [KeyError] when a pointer does not resolve (splice replaces;
|
|
411
|
+
# use {.patch} +add+ to insert)
|
|
412
|
+
# @raise [ArgumentError] for malformed pointers or overlapping targets
|
|
413
|
+
# @raise [ParserError] when the document is malformed
|
|
414
|
+
def self.splice(json, edits, opts = nil)
|
|
415
|
+
splice_native(json, edits, opts)
|
|
416
|
+
end
|
|
417
|
+
|
|
418
|
+
# Applies an RFC 6902 JSON Patch to the raw document: +add+,
|
|
419
|
+
# +remove+, +replace+, +move+, +copy+, and +test+, applied
|
|
420
|
+
# sequentially, each as a byte-splice (structural ops walk only the
|
|
421
|
+
# parent container's span). Op hashes accept String or Symbol keys.
|
|
422
|
+
#
|
|
423
|
+
# @example
|
|
424
|
+
# NOSJ.patch(json, [
|
|
425
|
+
# {"op" => "test", "path" => "/a", "value" => 1},
|
|
426
|
+
# {"op" => "replace", "path" => "/a", "value" => 2},
|
|
427
|
+
# {"op" => "add", "path" => "/list/-", "value" => "x"},
|
|
428
|
+
# {"op" => "move", "from" => "/tmp", "path" => "/kept"}
|
|
429
|
+
# ])
|
|
430
|
+
#
|
|
431
|
+
# @param json [String] the document (UTF-8 or US-ASCII)
|
|
432
|
+
# @param ops [Array<Hash>] RFC 6902 operations
|
|
433
|
+
# @param opts [Hash, nil] {.generate} options for inserted values
|
|
434
|
+
# @return [String] the patched document
|
|
435
|
+
# @raise [PatchError] when application fails (failed +test+, missing
|
|
436
|
+
# target, index out of range, move into own child)
|
|
437
|
+
# @raise [ArgumentError] for structurally malformed patch documents
|
|
438
|
+
# @raise [ParserError] when the document is malformed
|
|
439
|
+
def self.patch(json, ops, opts = nil)
|
|
440
|
+
patch_native(json, ops, opts)
|
|
441
|
+
end
|
|
442
|
+
|
|
443
|
+
# Applies an RFC 7386 JSON Merge Patch: +nil+ values remove keys,
|
|
444
|
+
# nested Hashes merge recursively, everything else replaces. This is
|
|
445
|
+
# the semantic form (parse, merge, generate); Symbol keys in +patch+
|
|
446
|
+
# match String keys in the document.
|
|
447
|
+
#
|
|
448
|
+
# @example
|
|
449
|
+
# NOSJ.merge_patch(%({"a":{"b":1,"c":2}}), {a: {b: nil, d: 3}})
|
|
450
|
+
# #=> '{"a":{"c":2,"d":3}}'
|
|
451
|
+
#
|
|
452
|
+
# @param json [String] the document (UTF-8 or US-ASCII)
|
|
453
|
+
# @param patch [Object] the merge patch (a non-Hash replaces the
|
|
454
|
+
# whole document)
|
|
455
|
+
# @param opts [Hash, nil] {.generate} options for the result
|
|
456
|
+
# @return [String] the merged document
|
|
457
|
+
# @raise [ParserError] when the document is malformed
|
|
458
|
+
def self.merge_patch(json, patch, opts = nil)
|
|
459
|
+
return generate(patch, opts) unless patch.is_a?(Hash)
|
|
460
|
+
generate(merge_patch_value(parse(json), patch), opts)
|
|
461
|
+
end
|
|
462
|
+
|
|
463
|
+
# RFC 7386, applied to parsed values.
|
|
464
|
+
def self.merge_patch_value(target, patch)
|
|
465
|
+
return patch unless patch.is_a?(Hash)
|
|
466
|
+
target = {} unless target.is_a?(Hash)
|
|
467
|
+
out = target.dup
|
|
468
|
+
patch.each do |key, value|
|
|
469
|
+
key = key.to_s
|
|
470
|
+
if value.nil?
|
|
471
|
+
out.delete(key)
|
|
472
|
+
else
|
|
473
|
+
out[key] = merge_patch_value(out[key], value)
|
|
474
|
+
end
|
|
475
|
+
end
|
|
476
|
+
out
|
|
477
|
+
end
|
|
478
|
+
private_class_method :merge_patch_value
|
|
479
|
+
|
|
480
|
+
# NDJSON / JSON Lines: yields one parsed value per line of +source+.
|
|
481
|
+
# Framing is exact because a raw newline can never occur inside a
|
|
482
|
+
# JSON value; blank lines are skipped (the NDJSON convention). One
|
|
483
|
+
# value per line is enforced: a second value on a line raises, and a
|
|
484
|
+
# malformed line raises {ParserError} whose {ParserError#line} is the
|
|
485
|
+
# physical line number in +source+.
|
|
486
|
+
#
|
|
487
|
+
# Pass a frozen string for zero-copy iteration (an unfrozen source is
|
|
488
|
+
# walked over a private copy, like {.lazy}). The block may itself
|
|
489
|
+
# call any NOSJ method.
|
|
490
|
+
#
|
|
491
|
+
# @example
|
|
492
|
+
# NOSJ.each_line(log) { |event| ingest(event) }
|
|
493
|
+
# NOSJ.each_line(log).first(10) # Enumerator when blockless
|
|
494
|
+
#
|
|
495
|
+
# @param source [String] newline-delimited JSON (UTF-8 or US-ASCII)
|
|
496
|
+
# @param opts [Hash, nil] the same options as {.parse}, applied per line
|
|
497
|
+
# @yieldparam value [Object] one parsed document per non-blank line
|
|
498
|
+
# @return [Enumerator] when no block is given, else +nil+
|
|
499
|
+
# @raise [ParserError] on the first malformed line
|
|
500
|
+
def self.each_line(source, opts = nil, &block)
|
|
501
|
+
return enum_for(:each_line, source, opts) unless block
|
|
502
|
+
each_line_native(source, opts, &block)
|
|
503
|
+
end
|
|
504
|
+
|
|
505
|
+
# {.each_line} against a file: the NDJSON stream is walked over a
|
|
506
|
+
# read-only memory map, so the file never becomes a Ruby String.
|
|
507
|
+
#
|
|
508
|
+
# @example
|
|
509
|
+
# NOSJ.each_line_file("events.ndjson") { |event| ingest(event) }
|
|
510
|
+
#
|
|
511
|
+
# @param path [String] the NDJSON file (UTF-8)
|
|
512
|
+
# @param opts [Hash, nil] the same options as {.parse}, applied per line
|
|
513
|
+
# @yieldparam value [Object] one parsed document per non-blank line
|
|
514
|
+
# @return [Enumerator] when no block is given, else +nil+
|
|
515
|
+
# @raise [SystemCallError] +Errno::ENOENT+ and friends
|
|
516
|
+
# @raise [ParserError] on the first malformed line
|
|
517
|
+
def self.each_line_file(path, opts = nil, &block)
|
|
518
|
+
return enum_for(:each_line_file, path, opts) unless block
|
|
519
|
+
each_line_file_native(path, opts, &block)
|
|
520
|
+
end
|
|
521
|
+
|
|
522
|
+
# Generates NDJSON / JSON Lines: one compact document per element,
|
|
523
|
+
# each terminated with a newline, built in a single pass into one
|
|
524
|
+
# buffer. Formatting options containing newlines raise ArgumentError
|
|
525
|
+
# (they would break the line framing); everything else from
|
|
526
|
+
# {.generate} applies.
|
|
527
|
+
#
|
|
528
|
+
# @example
|
|
529
|
+
# NOSJ.generate_lines([{a: 1}, {b: 2}]) #=> %({"a":1}\n{"b":2}\n)
|
|
530
|
+
#
|
|
531
|
+
# @param values [Array, Enumerable] one document per element
|
|
532
|
+
# @param opts [Hash, nil] the same options as {.generate}
|
|
533
|
+
# @return [String] the NDJSON document (empty when +values+ is empty)
|
|
534
|
+
# @raise [ArgumentError] for formatting options that contain newlines
|
|
535
|
+
# @raise [GeneratorError] (see {.generate})
|
|
536
|
+
def self.generate_lines(values, opts = nil)
|
|
537
|
+
generate_lines_native(lines_array(values), opts)
|
|
538
|
+
end
|
|
539
|
+
|
|
540
|
+
# {.generate_lines} straight to a file, streaming the generator's
|
|
541
|
+
# buffer to disk like {.write_file}.
|
|
542
|
+
#
|
|
543
|
+
# @example
|
|
544
|
+
# NOSJ.write_lines("out.ndjson", events) #=> bytes written
|
|
545
|
+
#
|
|
546
|
+
# @param path [String] the file to (over)write
|
|
547
|
+
# @param values [Array, Enumerable] one document per line
|
|
548
|
+
# @param opts [Hash, nil] the same options as {.generate}
|
|
549
|
+
# @return [Integer] the number of bytes written, like File.write
|
|
550
|
+
# @raise [SystemCallError] +Errno::ENOENT+ and friends
|
|
551
|
+
# @raise [ArgumentError] for formatting options that contain newlines
|
|
552
|
+
# @raise [GeneratorError] (see {.generate})
|
|
553
|
+
def self.write_lines(path, values, opts = nil)
|
|
554
|
+
write_lines_native(path, lines_array(values), opts)
|
|
555
|
+
end
|
|
556
|
+
|
|
557
|
+
# One document per element: Arrays pass through, other Enumerables
|
|
558
|
+
# convert, anything else (nil included, whose to_a would silently
|
|
559
|
+
# yield []) is a TypeError.
|
|
560
|
+
def self.lines_array(values)
|
|
561
|
+
return values if values.is_a?(Array)
|
|
562
|
+
unless values.is_a?(Enumerable)
|
|
563
|
+
raise TypeError, "no implicit conversion of #{values.class} into Array"
|
|
564
|
+
end
|
|
565
|
+
values.to_a
|
|
566
|
+
end
|
|
567
|
+
private_class_method :lines_array
|
|
568
|
+
|
|
569
|
+
# Document statistics from one full-parser pass into a counting sink:
|
|
570
|
+
# no Ruby value is built for the document itself, only the small
|
|
571
|
+
# result Hash. A debugging endpoint for "what is this 40 MB blob".
|
|
572
|
+
#
|
|
573
|
+
# The result (Symbol keys):
|
|
574
|
+
#
|
|
575
|
+
# {
|
|
576
|
+
# byte_size: 631, # of the document
|
|
577
|
+
# root: :object, # :object/:array/:string/:integer/
|
|
578
|
+
# # :float/:boolean/:null
|
|
579
|
+
# max_depth: 4, # container nesting, counted like
|
|
580
|
+
# # max_nesting (root container = 1)
|
|
581
|
+
# values: {total:, objects:, arrays:, strings:, integers:,
|
|
582
|
+
# floats:, booleans:, nulls:},
|
|
583
|
+
# keys: {total:, unique:},
|
|
584
|
+
# key_histogram: {"name" => 128, ...}, # sorted by count desc,
|
|
585
|
+
# # so .first(10) = top 10
|
|
586
|
+
# containers: {max_object_entries:, max_array_length:},
|
|
587
|
+
# strings: {bytes:, max_bytes:} # decoded UTF-8 bytes
|
|
588
|
+
# }
|
|
589
|
+
#
|
|
590
|
+
# Unlike {.parse}, nesting is UNLIMITED by default (a deep blob is
|
|
591
|
+
# exactly what a diagnostic should describe); pass +max_nesting+ to
|
|
592
|
+
# enforce a limit. Histogram memory is proportional to the number of
|
|
593
|
+
# unique keys.
|
|
594
|
+
#
|
|
595
|
+
# @example Top ten keys of a mystery blob
|
|
596
|
+
# NOSJ.stats(blob)[:key_histogram].first(10)
|
|
597
|
+
#
|
|
598
|
+
# @param source [String] the JSON document (UTF-8 or US-ASCII)
|
|
599
|
+
# @param opts [Hash, nil] +max_nesting+, +allow_nan+,
|
|
600
|
+
# +allow_trailing_comma+ (acceptance options only)
|
|
601
|
+
# @return [Hash] the statistics described above
|
|
602
|
+
# @raise [ParserError] when the document is malformed or not UTF-8
|
|
603
|
+
def self.stats(source, opts = nil)
|
|
604
|
+
stats_native(source, opts)
|
|
605
|
+
end
|
|
606
|
+
|
|
607
|
+
# {.stats} against a file: memory-maps it and runs the counting pass
|
|
608
|
+
# without reading the document into Ruby. +byte_size+ is the file
|
|
609
|
+
# size.
|
|
610
|
+
#
|
|
611
|
+
# @example
|
|
612
|
+
# NOSJ.stats_file("huge.json") => {byte_size: 41_943_040, ...}
|
|
613
|
+
#
|
|
614
|
+
# @param path [String] the file to inspect (UTF-8)
|
|
615
|
+
# @param opts [Hash, nil] same options as {.stats}
|
|
616
|
+
# @return [Hash] the statistics described on {.stats}
|
|
617
|
+
# @raise [SystemCallError] +Errno::ENOENT+ and friends
|
|
618
|
+
# @raise [ParserError] when the file is malformed or not UTF-8
|
|
619
|
+
def self.stats_file(path, opts = nil)
|
|
620
|
+
stats_file_native(path, opts)
|
|
621
|
+
end
|
|
193
622
|
end
|
data/sig/nosj.rbs
CHANGED
|
@@ -25,12 +25,32 @@ module NOSJ
|
|
|
25
25
|
class Error < StandardError
|
|
26
26
|
end
|
|
27
27
|
|
|
28
|
+
# Parse failures. Position accessors are nil for failures without
|
|
29
|
+
# one (encoding refusals); positions are byte-absolute within the
|
|
30
|
+
# document, lines and (character) columns 1-based.
|
|
31
|
+
class ParserError < Error
|
|
32
|
+
def byte_offset: () -> Integer?
|
|
33
|
+
|
|
34
|
+
def line: () -> Integer?
|
|
35
|
+
|
|
36
|
+
def column: () -> Integer?
|
|
37
|
+
|
|
38
|
+
def snippet: () -> String?
|
|
39
|
+
|
|
40
|
+
def detailed_message: (?highlight: bool, **untyped) -> String
|
|
41
|
+
end
|
|
42
|
+
|
|
28
43
|
class GeneratorError < Error
|
|
29
44
|
end
|
|
30
45
|
|
|
31
46
|
class NestingError < Error
|
|
32
47
|
end
|
|
33
48
|
|
|
49
|
+
# RFC 6902 application failures (failed test, missing target, index
|
|
50
|
+
# out of range, move into own child).
|
|
51
|
+
class PatchError < Error
|
|
52
|
+
end
|
|
53
|
+
|
|
34
54
|
def self.parse: (String source, ?opts opts) -> value
|
|
35
55
|
|
|
36
56
|
def self.generate: (untyped obj, ?opts opts) -> String
|
|
@@ -39,6 +59,12 @@ module NOSJ
|
|
|
39
59
|
|
|
40
60
|
def self.valid?: (String source, ?opts opts) -> bool
|
|
41
61
|
|
|
62
|
+
# Document statistics (byte_size, root, max_depth, values, keys,
|
|
63
|
+
# key_histogram, containers, strings) from one counting-sink pass.
|
|
64
|
+
def self.stats: (String source, ?opts opts) -> Hash[Symbol, untyped]
|
|
65
|
+
|
|
66
|
+
def self.stats_file: (String path, ?opts opts) -> Hash[Symbol, untyped]
|
|
67
|
+
|
|
42
68
|
def self.dig: (String source, *path_element path) -> value
|
|
43
69
|
|
|
44
70
|
def self.dig_many: (String source, Array[Array[path_element]] paths, ?opts opts) -> Array[value]
|
|
@@ -47,6 +73,100 @@ module NOSJ
|
|
|
47
73
|
|
|
48
74
|
def self.at_pointers: (String source, Array[String] pointers, ?opts opts) -> Array[value]
|
|
49
75
|
|
|
76
|
+
def self.lazy: (String source, ?opts opts) -> (Lazy | value)
|
|
77
|
+
|
|
78
|
+
# Reformat without building values: parser events pipe straight into
|
|
79
|
+
# the emission kernels. minify is compact; reformat takes pretty: and
|
|
80
|
+
# the generate formatting/escape options.
|
|
81
|
+
def self.minify: (String json, ?opts opts) -> String
|
|
82
|
+
|
|
83
|
+
def self.reformat: (String json, ?opts opts) -> String
|
|
84
|
+
|
|
85
|
+
def self.reformat_file: (String path, ?opts opts) -> String
|
|
86
|
+
|
|
87
|
+
# Byte-splicing edits: JSON Pointer => replacement value, resolved
|
|
88
|
+
# in one pass; bytes outside the target spans are copied untouched.
|
|
89
|
+
def self.splice: (String json, Hash[String, untyped] edits, ?opts opts) -> String
|
|
90
|
+
|
|
91
|
+
# RFC 6902 JSON Patch over the raw document.
|
|
92
|
+
def self.patch: (String json, Array[Hash[untyped, untyped]] ops, ?opts opts) -> String
|
|
93
|
+
|
|
94
|
+
# RFC 7386 JSON Merge Patch (semantic: parse, merge, generate).
|
|
95
|
+
def self.merge_patch: (String json, untyped patch, ?opts opts) -> String
|
|
96
|
+
|
|
97
|
+
# NDJSON / JSON Lines: one parsed value per non-blank line.
|
|
98
|
+
def self.each_line: (String source, ?opts opts) { (value) -> void } -> nil
|
|
99
|
+
| (String source, ?opts opts) -> Enumerator[value, nil]
|
|
100
|
+
|
|
101
|
+
def self.each_line_file: (String path, ?opts opts) { (value) -> void } -> nil
|
|
102
|
+
| (String path, ?opts opts) -> Enumerator[value, nil]
|
|
103
|
+
|
|
104
|
+
def self.generate_lines: (_Each[untyped] values, ?opts opts) -> String
|
|
105
|
+
|
|
106
|
+
def self.write_lines: (String path, _Each[untyped] values, ?opts opts) -> Integer
|
|
107
|
+
|
|
108
|
+
def self.load_file: (String path, ?opts opts) -> value
|
|
109
|
+
|
|
110
|
+
def self.write_file: (String path, untyped obj, ?opts opts) -> Integer
|
|
111
|
+
|
|
112
|
+
def self.load_lazy_file: (String path, ?opts opts) -> (Lazy | value)
|
|
113
|
+
|
|
114
|
+
def self.at_pointer_file: (String path, String pointer, ?opts opts) -> value
|
|
115
|
+
|
|
116
|
+
def self.dig_file: (String path, *path_element path_elements) -> value
|
|
117
|
+
|
|
118
|
+
# A lazy view of one JSON container: children resolve on first
|
|
119
|
+
# access (containers as further Lazy nodes, scalars as values,
|
|
120
|
+
# misses as nil) and are cached on the node.
|
|
121
|
+
class Lazy
|
|
122
|
+
include Enumerable[untyped]
|
|
123
|
+
|
|
124
|
+
def []: (path_element token) -> (Lazy | value)
|
|
125
|
+
|
|
126
|
+
def dig: (path_element first, *path_element rest) -> (Lazy | value)
|
|
127
|
+
|
|
128
|
+
def at_pointer: (String pointer) -> (Lazy | value)
|
|
129
|
+
|
|
130
|
+
def value: () -> value
|
|
131
|
+
|
|
132
|
+
alias materialize value
|
|
133
|
+
|
|
134
|
+
def to_h: () -> Hash[String | Symbol, value]
|
|
135
|
+
|
|
136
|
+
def to_a: () -> Array[value]
|
|
137
|
+
|
|
138
|
+
def object?: () -> bool
|
|
139
|
+
|
|
140
|
+
def array?: () -> bool
|
|
141
|
+
|
|
142
|
+
def keys: () -> Array[String]
|
|
143
|
+
|
|
144
|
+
def size: () -> Integer
|
|
145
|
+
|
|
146
|
+
alias length size
|
|
147
|
+
|
|
148
|
+
alias count size
|
|
149
|
+
|
|
150
|
+
def empty?: () -> bool
|
|
151
|
+
|
|
152
|
+
def each: () { (untyped) -> void } -> self
|
|
153
|
+
| () -> Enumerator[untyped, self]
|
|
154
|
+
|
|
155
|
+
def inspect: () -> String
|
|
156
|
+
|
|
157
|
+
alias to_s inspect
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# Defined by `require "nosj/rails"`, which also installs it as
|
|
161
|
+
# ActiveSupport::JSON::Encoding.json_encoder.
|
|
162
|
+
class RailsEncoder
|
|
163
|
+
attr_reader options: Hash[Symbol, untyped]
|
|
164
|
+
|
|
165
|
+
def initialize: (?Hash[Symbol, untyped]? options) -> void
|
|
166
|
+
|
|
167
|
+
def encode: (untyped value) -> String
|
|
168
|
+
end
|
|
169
|
+
|
|
50
170
|
# Defined by `require "nosj/multi_json"`. The runtime superclass is
|
|
51
171
|
# multi_json's Adapter (whose namespace differs across multi_json
|
|
52
172
|
# versions), so it is not declared here.
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: nosj
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Yaroslav Markin
|
|
@@ -25,8 +25,12 @@ dependencies:
|
|
|
25
25
|
version: 0.9.91
|
|
26
26
|
description: 'gem nosj is an extremely fast, json-gem-compatible JSON parser and generator
|
|
27
27
|
for Ruby: Rust and SIMD via the first-party nosj crate, precompiled platform gems
|
|
28
|
-
with per-platform PGO, partial parsing (JSON Pointer, single and batch),
|
|
29
|
-
|
|
28
|
+
with per-platform PGO, partial parsing (JSON Pointer, single and batch), lazy documents
|
|
29
|
+
that parse a value only when you touch it, file APIs that parse, generate, and query
|
|
30
|
+
files natively (memory-mapped, so unread pages never leave the disk), allocation-free
|
|
31
|
+
validation, document statistics from one counting pass, parse errors that carry
|
|
32
|
+
line, column, and a caret snippet, a one-line JSON module drop-in, and a Rails mode
|
|
33
|
+
that plugs into ActiveSupport''s encoder seam.'
|
|
30
34
|
email:
|
|
31
35
|
- yaroslav@markin.net
|
|
32
36
|
executables: []
|
|
@@ -42,6 +46,8 @@ files:
|
|
|
42
46
|
- README.md
|
|
43
47
|
- ext/nosj/Cargo.toml
|
|
44
48
|
- ext/nosj/extconf.rb
|
|
49
|
+
- ext/nosj/src/errors.rs
|
|
50
|
+
- ext/nosj/src/files.rs
|
|
45
51
|
- ext/nosj/src/gen/errors.rs
|
|
46
52
|
- ext/nosj/src/gen/hash_iter.rs
|
|
47
53
|
- ext/nosj/src/gen/keys.rs
|
|
@@ -49,15 +55,22 @@ files:
|
|
|
49
55
|
- ext/nosj/src/gen/opts.rs
|
|
50
56
|
- ext/nosj/src/gen/ruby.rs
|
|
51
57
|
- ext/nosj/src/gen/walker.rs
|
|
58
|
+
- ext/nosj/src/lazy.rs
|
|
52
59
|
- ext/nosj/src/lib.rs
|
|
60
|
+
- ext/nosj/src/lines.rs
|
|
53
61
|
- ext/nosj/src/parse.rs
|
|
62
|
+
- ext/nosj/src/patch.rs
|
|
54
63
|
- ext/nosj/src/pointer.rs
|
|
64
|
+
- ext/nosj/src/reformat.rs
|
|
55
65
|
- ext/nosj/src/sink.rs
|
|
56
66
|
- ext/nosj/src/state.rs
|
|
67
|
+
- ext/nosj/src/stats.rs
|
|
57
68
|
- lib/nosj.rb
|
|
58
69
|
- lib/nosj/json.rb
|
|
70
|
+
- lib/nosj/lazy.rb
|
|
59
71
|
- lib/nosj/multi_json.rb
|
|
60
72
|
- lib/nosj/native.rb
|
|
73
|
+
- lib/nosj/rails.rb
|
|
61
74
|
- lib/nosj/version.rb
|
|
62
75
|
- sig/nosj.rbs
|
|
63
76
|
homepage: https://github.com/yaroslav/nosj-ruby
|