nosj 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +112 -0
- data/Cargo.lock +3 -3
- data/README.md +220 -19
- data/ext/nosj/Cargo.toml +2 -2
- data/ext/nosj/fuzz/Cargo.toml +41 -0
- data/ext/nosj/fuzz/fuzz_targets/lines.rs +9 -0
- data/ext/nosj/fuzz/fuzz_targets/patch.rs +10 -0
- data/ext/nosj/fuzz/fuzz_targets/reformat.rs +10 -0
- data/ext/nosj/fuzz/src/lib.rs +48 -0
- data/ext/nosj/fuzz/src/prelude.rb +400 -0
- data/ext/nosj/src/errors.rs +171 -0
- data/ext/nosj/src/files.rs +29 -7
- data/ext/nosj/src/gen/errors.rs +2 -9
- data/ext/nosj/src/gen/mod.rs +156 -12
- data/ext/nosj/src/gen/opts.rs +88 -10
- data/ext/nosj/src/gen/ruby.rs +51 -0
- data/ext/nosj/src/gen/walker.rs +97 -59
- data/ext/nosj/src/lazy.rs +42 -22
- data/ext/nosj/src/lib.rs +30 -0
- data/ext/nosj/src/lines.rs +90 -0
- data/ext/nosj/src/parse.rs +55 -14
- data/ext/nosj/src/patch.rs +547 -0
- data/ext/nosj/src/pointer.rs +12 -5
- data/ext/nosj/src/reformat.rs +320 -0
- data/ext/nosj/src/sink.rs +11 -0
- data/ext/nosj/src/stats.rs +301 -0
- data/lib/nosj/json.rb +30 -6
- data/lib/nosj/multi_json.rb +1 -1
- data/lib/nosj/rails.rb +76 -0
- data/lib/nosj/version.rb +1 -1
- data/lib/nosj.rb +351 -5
- data/sig/nosj.rbs +66 -0
- metadata +16 -2
data/lib/nosj/json.rb
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
#
|
|
7
7
|
# reroutes JSON.parse, JSON.generate, JSON.pretty_generate and JSON.dump
|
|
8
8
|
# through NOSJ whenever the requested options fall within NOSJ's
|
|
9
|
-
# supported set, and falls back to
|
|
9
|
+
# supported set, and falls back to gem json's own implementation for
|
|
10
10
|
# everything else (create_additions, object_class/array_class,
|
|
11
11
|
# decimal_class, on_load procs, JSON::State instances, IO arguments).
|
|
12
12
|
# Entry points built on JSON.parse (JSON.load, JSON.parse!,
|
|
@@ -30,8 +30,11 @@ module NOSJ
|
|
|
30
30
|
# Implementation detail of `require "nosj/json"`.
|
|
31
31
|
# @private
|
|
32
32
|
module JSONDropIn
|
|
33
|
+
# quirks_mode rides the fast path because NOSJ.parse is always
|
|
34
|
+
# quirks-mode (top-level scalars parse) and ignores the key; Rails
|
|
35
|
+
# 7.x passes it from ActiveSupport::JSON.decode.
|
|
33
36
|
PARSE_OPTS = %i[symbolize_names freeze max_nesting allow_nan
|
|
34
|
-
allow_trailing_comma].freeze
|
|
37
|
+
allow_trailing_comma quirks_mode].freeze
|
|
35
38
|
GENERATE_OPTS = %i[indent space space_before object_nl array_nl
|
|
36
39
|
max_nesting allow_nan ascii_only script_safe
|
|
37
40
|
escape_slash strict depth
|
|
@@ -41,7 +44,7 @@ module NOSJ
|
|
|
41
44
|
|
|
42
45
|
# The fast path handles nil or a plain Hash whose every key NOSJ
|
|
43
46
|
# implements; anything else (JSON::State, exotic options, string
|
|
44
|
-
# keys) belongs to
|
|
47
|
+
# keys) belongs to gem json.
|
|
45
48
|
def supported?(opts, allowed)
|
|
46
49
|
return true if opts.nil?
|
|
47
50
|
return false unless opts.instance_of?(Hash)
|
|
@@ -50,8 +53,29 @@ module NOSJ
|
|
|
50
53
|
end
|
|
51
54
|
|
|
52
55
|
def parse(source, opts)
|
|
56
|
+
# NOSJ.parse is deliberately strict about encodings (json-3.0
|
|
57
|
+
# semantics), but the drop-in must match the installed gem, which
|
|
58
|
+
# accepts more. BINARY strings holding valid UTF-8 are the big
|
|
59
|
+
# real-world case: Rack delivers request bodies as BINARY, so
|
|
60
|
+
# Rails JSON params come through here. Retagging a dup is cheap
|
|
61
|
+
# (copy-on-write bytes), and the validity scan is memoized
|
|
62
|
+
# coderange the parse would compute anyway. Anything else
|
|
63
|
+
# non-UTF-8 (UTF-16, ...) belongs to gem json, which transcodes.
|
|
64
|
+
if source.is_a?(String)
|
|
65
|
+
case source.encoding
|
|
66
|
+
when Encoding::UTF_8, Encoding::US_ASCII
|
|
67
|
+
# the fast path as-is
|
|
68
|
+
when Encoding::BINARY
|
|
69
|
+
utf8 = source.dup.force_encoding(Encoding::UTF_8)
|
|
70
|
+
source = utf8 if utf8.valid_encoding?
|
|
71
|
+
else
|
|
72
|
+
return ::JSON.nosj_original_parse(source, **(opts || {}))
|
|
73
|
+
end
|
|
74
|
+
end
|
|
53
75
|
NOSJ.parse(source, opts)
|
|
54
|
-
rescue
|
|
76
|
+
rescue NOSJ::NestingError => e
|
|
77
|
+
raise ::JSON::NestingError, e.message
|
|
78
|
+
rescue NOSJ::ParserError => e
|
|
55
79
|
raise ::JSON::ParserError, e.message
|
|
56
80
|
end
|
|
57
81
|
|
|
@@ -103,9 +127,9 @@ module JSON
|
|
|
103
127
|
|
|
104
128
|
def dump(obj, an_io = nil, limit = nil, kwargs = nil)
|
|
105
129
|
# Fast path for the common shapes, dump(obj) and dump(obj, opts
|
|
106
|
-
# hash), mirroring
|
|
130
|
+
# hash), mirroring gem json: dump defaults merged under the
|
|
107
131
|
# user's options, NestingError surfaced as ArgumentError. IO and
|
|
108
|
-
# limit arguments take
|
|
132
|
+
# limit arguments take gem json's own dump.
|
|
109
133
|
if limit.nil? && kwargs.nil? && (an_io.nil? || an_io.instance_of?(Hash))
|
|
110
134
|
opts = _dump_default_options
|
|
111
135
|
opts = opts.merge(an_io) if an_io
|
data/lib/nosj/multi_json.rb
CHANGED
|
@@ -36,7 +36,7 @@ module NOSJ
|
|
|
36
36
|
# @raise [JSON::ParserError] when the document is malformed
|
|
37
37
|
def load(string, options = {})
|
|
38
38
|
::NOSJ.parse(string, options[:symbolize_names] ? SYMBOLIZE : nil)
|
|
39
|
-
rescue
|
|
39
|
+
rescue ::NOSJ::ParserError, ::NOSJ::NestingError => e
|
|
40
40
|
raise ParseError, e.message
|
|
41
41
|
end
|
|
42
42
|
|
data/lib/nosj/rails.rb
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Rails mode: `require "nosj/rails"` accelerates a Rails application in
|
|
4
|
+
# both directions.
|
|
5
|
+
#
|
|
6
|
+
# - It installs {NOSJ::RailsEncoder} as ActiveSupport's JSON encoder
|
|
7
|
+
# (the official +ActiveSupport::JSON::Encoding.json_encoder+ seam, the
|
|
8
|
+
# same one Oj's Rails mode uses). That captures every encode a Rails
|
|
9
|
+
# app performs through +obj.to_json+, +render json:+, and
|
|
10
|
+
# +ActiveSupport::JSON.encode+: the object tree is walked natively,
|
|
11
|
+
# values that are not JSON-native recurse through +as_json+ exactly
|
|
12
|
+
# like ActiveSupport's own encoder, and non-finite floats encode as
|
|
13
|
+
# +null+ (+Float#as_json+ parity).
|
|
14
|
+
# - It loads the `nosj/json` drop-in, so +JSON.parse+ (and with it
|
|
15
|
+
# +ActiveSupport::JSON.decode+ and JSON request-body parsing) takes
|
|
16
|
+
# the fast path.
|
|
17
|
+
#
|
|
18
|
+
# In a Rails Gemfile:
|
|
19
|
+
#
|
|
20
|
+
# gem "nosj", require: "nosj/rails"
|
|
21
|
+
#
|
|
22
|
+
# +JSON::Fragment+ values splice their pre-rendered JSON, like current
|
|
23
|
+
# ActiveSupport. On older ActiveSupport (before its encoder learned
|
|
24
|
+
# about fragments) stock encoding dumps the fragment's instance
|
|
25
|
+
# variables instead; there this encoder deliberately diverges in favor
|
|
26
|
+
# of real splicing.
|
|
27
|
+
|
|
28
|
+
require "nosj/json"
|
|
29
|
+
require "active_support"
|
|
30
|
+
require "active_support/json"
|
|
31
|
+
|
|
32
|
+
module NOSJ
|
|
33
|
+
# ActiveSupport JSON encoder backed by nosj: the interface of
|
|
34
|
+
# ActiveSupport's +JSONGemEncoder+ (accepts the options hash, encodes
|
|
35
|
+
# one value), with the tree walk, generation, AND the HTML/JS-safety
|
|
36
|
+
# escape pass running natively (a byte scan instead of ActiveSupport's
|
|
37
|
+
# Ruby regex post-pass), honoring +escape_html_entities_in_json+ (and,
|
|
38
|
+
# where present, +escape_js_separators_in_json+ and the per-call
|
|
39
|
+
# +escape:+ / +escape_html_entities:+ options).
|
|
40
|
+
class RailsEncoder
|
|
41
|
+
# The escape_js_separators_in_json knob is newer than the Rails
|
|
42
|
+
# versions we support; its presence is fixed at load time (only its
|
|
43
|
+
# value can change at runtime). Absent, ActiveSupport always
|
|
44
|
+
# escapes the JS separators.
|
|
45
|
+
HAS_JS_SEPARATORS_KNOB =
|
|
46
|
+
ActiveSupport::JSON::Encoding.respond_to?(:escape_js_separators_in_json)
|
|
47
|
+
private_constant :HAS_JS_SEPARATORS_KNOB
|
|
48
|
+
|
|
49
|
+
attr_reader :options
|
|
50
|
+
|
|
51
|
+
def initialize(options = nil)
|
|
52
|
+
@options = options || {}
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def encode(value)
|
|
56
|
+
value = value.as_json(@options.dup) unless @options.empty?
|
|
57
|
+
if @options.fetch(:escape, true)
|
|
58
|
+
escape_html = @options.fetch(:escape_html_entities) do
|
|
59
|
+
ActiveSupport::JSON::Encoding.escape_html_entities_in_json
|
|
60
|
+
end
|
|
61
|
+
NOSJ.generate_rails_native(value, !!escape_html, escape_js_separators?)
|
|
62
|
+
else
|
|
63
|
+
NOSJ.generate_rails_native(value, false, false)
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
private
|
|
68
|
+
|
|
69
|
+
def escape_js_separators?
|
|
70
|
+
!HAS_JS_SEPARATORS_KNOB ||
|
|
71
|
+
ActiveSupport::JSON::Encoding.escape_js_separators_in_json
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
ActiveSupport::JSON::Encoding.json_encoder = RailsEncoder
|
|
76
|
+
end
|
data/lib/nosj/version.rb
CHANGED
data/lib/nosj.rb
CHANGED
|
@@ -26,15 +26,58 @@ module NOSJ
|
|
|
26
26
|
# Base class for nosj errors.
|
|
27
27
|
class Error < StandardError; end
|
|
28
28
|
|
|
29
|
+
# Raised when a document cannot be parsed. Carries the failure
|
|
30
|
+
# position, computed once when the parse fails (successful parses
|
|
31
|
+
# never pay for it): {#byte_offset}, 1-based {#line}, character-based
|
|
32
|
+
# {#column}, and a caret {#snippet} pointing at the offending spot.
|
|
33
|
+
# Positions are absolute within the document you passed, including
|
|
34
|
+
# through partial parsing ({NOSJ.dig}, {NOSJ.at_pointer}, lazy
|
|
35
|
+
# documents) and the file APIs. All four are +nil+ for failures
|
|
36
|
+
# without a position (encoding refusals).
|
|
37
|
+
#
|
|
38
|
+
# @example
|
|
39
|
+
# NOSJ.parse(%({\n "a": 1,\n "b": }))
|
|
40
|
+
# # => NOSJ::ParserError, with
|
|
41
|
+
# # e.line #=> 3
|
|
42
|
+
# # e.column #=> 8
|
|
43
|
+
# # e.snippet #=> " \"b\": }\n ^"
|
|
44
|
+
class ParserError < Error
|
|
45
|
+
# @return [Integer, nil] byte offset of the failure in the source
|
|
46
|
+
attr_reader :byte_offset
|
|
47
|
+
# @return [Integer, nil] 1-based line of the failure
|
|
48
|
+
attr_reader :line
|
|
49
|
+
# @return [Integer, nil] 1-based character (not byte) column within
|
|
50
|
+
# {#line}
|
|
51
|
+
attr_reader :column
|
|
52
|
+
# @return [String, nil] the offending line (windowed when long)
|
|
53
|
+
# with a caret line underneath
|
|
54
|
+
attr_reader :snippet
|
|
55
|
+
|
|
56
|
+
# The default message plus {#snippet}: Ruby prints
|
|
57
|
+
# +detailed_message+ when an exception reaches the top level, so an
|
|
58
|
+
# unrescued parse error shows where the document broke.
|
|
59
|
+
def detailed_message(highlight: false, **opts)
|
|
60
|
+
base = super
|
|
61
|
+
snippet ? "#{base}\n#{snippet}" : base
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
29
65
|
# Raised when a value cannot be generated (non-finite floats without
|
|
30
66
|
# +allow_nan+, unsupported objects under +strict+, broken encodings).
|
|
31
67
|
# Message-compatible with +JSON::GeneratorError+.
|
|
32
68
|
class GeneratorError < Error; end
|
|
33
69
|
|
|
34
|
-
# Raised when generation exceeds +max_nesting+.
|
|
35
|
-
# with +JSON::NestingError+.
|
|
70
|
+
# Raised when parsing or generation exceeds +max_nesting+.
|
|
71
|
+
# Message-compatible with +JSON::NestingError+.
|
|
36
72
|
class NestingError < Error; end
|
|
37
73
|
|
|
74
|
+
# Raised when an RFC 6902 patch cannot be applied: a +test+ operation
|
|
75
|
+
# failed, a target or source path does not exist, an array index is
|
|
76
|
+
# out of range, or a +move+ targets its own child. Structurally
|
|
77
|
+
# malformed patch documents (not an op Hash, unknown op, missing
|
|
78
|
+
# fields) raise ArgumentError instead.
|
|
79
|
+
class PatchError < Error; end
|
|
80
|
+
|
|
38
81
|
PRETTY_GENERATE_OPTS = {
|
|
39
82
|
indent: " ", space: " ", object_nl: "\n", array_nl: "\n"
|
|
40
83
|
}.freeze
|
|
@@ -57,7 +100,9 @@ module NOSJ
|
|
|
57
100
|
# (Integer or +false+ for unlimited), +allow_nan+,
|
|
58
101
|
# +allow_trailing_comma+
|
|
59
102
|
# @return [Object] the parsed value tree
|
|
60
|
-
# @raise [
|
|
103
|
+
# @raise [ParserError] when the document is malformed or not UTF-8;
|
|
104
|
+
# carries the failure position ({ParserError#line} and friends)
|
|
105
|
+
# @raise [NestingError] when nesting exceeds +max_nesting+
|
|
61
106
|
# @raise [ArgumentError] for unsupported options
|
|
62
107
|
def self.parse(source, opts = nil)
|
|
63
108
|
parse_native(source, opts)
|
|
@@ -204,7 +249,7 @@ module NOSJ
|
|
|
204
249
|
# @param opts [Hash, nil] the same options as {.parse}
|
|
205
250
|
# @return [Object] the parsed value tree
|
|
206
251
|
# @raise [SystemCallError] +Errno::ENOENT+ and friends, like File.read
|
|
207
|
-
# @raise [
|
|
252
|
+
# @raise [ParserError] when the document is malformed or not UTF-8
|
|
208
253
|
def self.load_file(path, opts = nil)
|
|
209
254
|
load_file_native(path, opts)
|
|
210
255
|
end
|
|
@@ -240,7 +285,7 @@ module NOSJ
|
|
|
240
285
|
# @param opts [Hash, nil] {.parse} options applied on materialization
|
|
241
286
|
# @return [NOSJ::Lazy, Object]
|
|
242
287
|
# @raise [SystemCallError] +Errno::ENOENT+ and friends
|
|
243
|
-
# @raise [
|
|
288
|
+
# @raise [ParserError] when the file is not UTF-8 or the root is malformed
|
|
244
289
|
def self.load_lazy_file(path, opts = nil)
|
|
245
290
|
load_lazy_file_native(path, opts)
|
|
246
291
|
end
|
|
@@ -273,4 +318,305 @@ module NOSJ
|
|
|
273
318
|
def self.dig_file(path, *path_elements)
|
|
274
319
|
dig_file_native(path, path_elements)
|
|
275
320
|
end
|
|
321
|
+
|
|
322
|
+
# Minifies a document without building any Ruby values: the parser's
|
|
323
|
+
# events pipe straight into the emission kernels, SIMD in and SIMD
|
|
324
|
+
# out. Output is exactly what <code>generate(parse(json))</code>
|
|
325
|
+
# would produce, except duplicate object keys pass through instead of
|
|
326
|
+
# being collapsed (a reformatter must not silently drop data).
|
|
327
|
+
# Numbers come out in the canonical spelling (+1.50+ becomes +1.5+)
|
|
328
|
+
# and string escapes are normalized.
|
|
329
|
+
#
|
|
330
|
+
# @example
|
|
331
|
+
# NOSJ.minify(%({ "a": [1, 2],\n "b": "x" })) #=> '{"a":[1,2],"b":"x"}'
|
|
332
|
+
#
|
|
333
|
+
# @param json [String] the document (UTF-8 or US-ASCII)
|
|
334
|
+
# @param opts [Hash, nil] acceptance options (+allow_nan+,
|
|
335
|
+
# +allow_trailing_comma+, +max_nesting+); trailing commas are
|
|
336
|
+
# normalized away when accepted
|
|
337
|
+
# @return [String] the minified document
|
|
338
|
+
# @raise [ParserError] when the document is malformed
|
|
339
|
+
# @raise [NestingError] past +max_nesting+
|
|
340
|
+
def self.minify(json, opts = nil)
|
|
341
|
+
reformat_native(json, opts)
|
|
342
|
+
end
|
|
343
|
+
|
|
344
|
+
# Reformats a document without building any Ruby values: {.minify}'s
|
|
345
|
+
# pipe with formatting. <code>pretty: true</code> is a shorthand for
|
|
346
|
+
# {.pretty_generate}'s layout; the individual {.generate} formatting
|
|
347
|
+
# and escape options (+indent+, +space+, +object_nl+, +ascii_only+,
|
|
348
|
+
# +script_safe+, ...) compose with it and win over it.
|
|
349
|
+
#
|
|
350
|
+
# @example
|
|
351
|
+
# NOSJ.reformat(json, pretty: true)
|
|
352
|
+
# NOSJ.reformat(json, ascii_only: true) # escape-transcode, compact
|
|
353
|
+
#
|
|
354
|
+
# @param json [String] the document (UTF-8 or US-ASCII)
|
|
355
|
+
# @param opts [Hash, nil] +pretty+, {.generate} formatting/escape
|
|
356
|
+
# options, and {.minify}'s acceptance options
|
|
357
|
+
# @return [String] the reformatted document
|
|
358
|
+
# @raise [ParserError] when the document is malformed
|
|
359
|
+
# @raise [NestingError] past +max_nesting+
|
|
360
|
+
# @raise [GeneratorError] when +ascii_only+ meets a lone-surrogate
|
|
361
|
+
# string it cannot represent
|
|
362
|
+
def self.reformat(json, opts = nil)
|
|
363
|
+
if opts&.key?(:pretty)
|
|
364
|
+
pretty = opts[:pretty]
|
|
365
|
+
opts = opts.except(:pretty)
|
|
366
|
+
opts = PRETTY_GENERATE_OPTS.merge(opts) if pretty
|
|
367
|
+
end
|
|
368
|
+
reformat_native(json, opts)
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
# {.reformat} against a file: the pipe runs over a read-only memory
|
|
372
|
+
# map, so the input document never becomes a Ruby String; only the
|
|
373
|
+
# result does.
|
|
374
|
+
#
|
|
375
|
+
# @example
|
|
376
|
+
# compact = NOSJ.reformat_file("big.json") # minify
|
|
377
|
+
# pretty = NOSJ.reformat_file("big.json", pretty: true)
|
|
378
|
+
#
|
|
379
|
+
# @param path [String] the file to reformat (UTF-8)
|
|
380
|
+
# @param opts [Hash, nil] same options as {.reformat}
|
|
381
|
+
# @return [String] the reformatted document
|
|
382
|
+
# @raise [SystemCallError] +Errno::ENOENT+ and friends
|
|
383
|
+
# @raise [ParserError] when the file is malformed or not UTF-8
|
|
384
|
+
def self.reformat_file(path, opts = nil)
|
|
385
|
+
if opts&.key?(:pretty)
|
|
386
|
+
pretty = opts[:pretty]
|
|
387
|
+
opts = opts.except(:pretty)
|
|
388
|
+
opts = PRETTY_GENERATE_OPTS.merge(opts) if pretty
|
|
389
|
+
end
|
|
390
|
+
reformat_file_native(path, opts)
|
|
391
|
+
end
|
|
392
|
+
|
|
393
|
+
# Byte-splicing edits: replaces the values at the given JSON Pointers
|
|
394
|
+
# directly in the text. Every target resolves in ONE forward pass
|
|
395
|
+
# (skipping, not parsing), and the result is built in one sweep:
|
|
396
|
+
# every byte outside the target spans is copied untouched, so
|
|
397
|
+
# formatting, key order, and number spellings elsewhere are preserved
|
|
398
|
+
# exactly. For tweaking a field in a passing payload this replaces
|
|
399
|
+
# the whole parse → mutate → generate cycle.
|
|
400
|
+
#
|
|
401
|
+
# @example
|
|
402
|
+
# NOSJ.splice(json, "/config/timeout" => 30)
|
|
403
|
+
# NOSJ.splice(json, "/a" => 1, "/b/c" => [true]) # batch, one pass
|
|
404
|
+
#
|
|
405
|
+
# @param json [String] the document (UTF-8 or US-ASCII)
|
|
406
|
+
# @param edits [Hash{String => Object}] JSON Pointer => replacement
|
|
407
|
+
# value (generated compactly, byte-identical to {.generate})
|
|
408
|
+
# @param opts [Hash, nil] {.generate} options for the inserted values
|
|
409
|
+
# @return [String] the edited document
|
|
410
|
+
# @raise [KeyError] when a pointer does not resolve (splice replaces;
|
|
411
|
+
# use {.patch} +add+ to insert)
|
|
412
|
+
# @raise [ArgumentError] for malformed pointers or overlapping targets
|
|
413
|
+
# @raise [ParserError] when the document is malformed
|
|
414
|
+
def self.splice(json, edits, opts = nil)
|
|
415
|
+
splice_native(json, edits, opts)
|
|
416
|
+
end
|
|
417
|
+
|
|
418
|
+
# Applies an RFC 6902 JSON Patch to the raw document: +add+,
|
|
419
|
+
# +remove+, +replace+, +move+, +copy+, and +test+, applied
|
|
420
|
+
# sequentially, each as a byte-splice (structural ops walk only the
|
|
421
|
+
# parent container's span). Op hashes accept String or Symbol keys.
|
|
422
|
+
#
|
|
423
|
+
# @example
|
|
424
|
+
# NOSJ.patch(json, [
|
|
425
|
+
# {"op" => "test", "path" => "/a", "value" => 1},
|
|
426
|
+
# {"op" => "replace", "path" => "/a", "value" => 2},
|
|
427
|
+
# {"op" => "add", "path" => "/list/-", "value" => "x"},
|
|
428
|
+
# {"op" => "move", "from" => "/tmp", "path" => "/kept"}
|
|
429
|
+
# ])
|
|
430
|
+
#
|
|
431
|
+
# @param json [String] the document (UTF-8 or US-ASCII)
|
|
432
|
+
# @param ops [Array<Hash>] RFC 6902 operations
|
|
433
|
+
# @param opts [Hash, nil] {.generate} options for inserted values
|
|
434
|
+
# @return [String] the patched document
|
|
435
|
+
# @raise [PatchError] when application fails (failed +test+, missing
|
|
436
|
+
# target, index out of range, move into own child)
|
|
437
|
+
# @raise [ArgumentError] for structurally malformed patch documents
|
|
438
|
+
# @raise [ParserError] when the document is malformed
|
|
439
|
+
def self.patch(json, ops, opts = nil)
|
|
440
|
+
patch_native(json, ops, opts)
|
|
441
|
+
end
|
|
442
|
+
|
|
443
|
+
# Applies an RFC 7386 JSON Merge Patch: +nil+ values remove keys,
|
|
444
|
+
# nested Hashes merge recursively, everything else replaces. This is
|
|
445
|
+
# the semantic form (parse, merge, generate); Symbol keys in +patch+
|
|
446
|
+
# match String keys in the document.
|
|
447
|
+
#
|
|
448
|
+
# @example
|
|
449
|
+
# NOSJ.merge_patch(%({"a":{"b":1,"c":2}}), {a: {b: nil, d: 3}})
|
|
450
|
+
# #=> '{"a":{"c":2,"d":3}}'
|
|
451
|
+
#
|
|
452
|
+
# @param json [String] the document (UTF-8 or US-ASCII)
|
|
453
|
+
# @param patch [Object] the merge patch (a non-Hash replaces the
|
|
454
|
+
# whole document)
|
|
455
|
+
# @param opts [Hash, nil] {.generate} options for the result
|
|
456
|
+
# @return [String] the merged document
|
|
457
|
+
# @raise [ParserError] when the document is malformed
|
|
458
|
+
def self.merge_patch(json, patch, opts = nil)
|
|
459
|
+
return generate(patch, opts) unless patch.is_a?(Hash)
|
|
460
|
+
generate(merge_patch_value(parse(json), patch), opts)
|
|
461
|
+
end
|
|
462
|
+
|
|
463
|
+
# RFC 7386, applied to parsed values.
|
|
464
|
+
def self.merge_patch_value(target, patch)
|
|
465
|
+
return patch unless patch.is_a?(Hash)
|
|
466
|
+
target = {} unless target.is_a?(Hash)
|
|
467
|
+
out = target.dup
|
|
468
|
+
patch.each do |key, value|
|
|
469
|
+
key = key.to_s
|
|
470
|
+
if value.nil?
|
|
471
|
+
out.delete(key)
|
|
472
|
+
else
|
|
473
|
+
out[key] = merge_patch_value(out[key], value)
|
|
474
|
+
end
|
|
475
|
+
end
|
|
476
|
+
out
|
|
477
|
+
end
|
|
478
|
+
private_class_method :merge_patch_value
|
|
479
|
+
|
|
480
|
+
# NDJSON / JSON Lines: yields one parsed value per line of +source+.
|
|
481
|
+
# Framing is exact because a raw newline can never occur inside a
|
|
482
|
+
# JSON value; blank lines are skipped (the NDJSON convention). One
|
|
483
|
+
# value per line is enforced: a second value on a line raises, and a
|
|
484
|
+
# malformed line raises {ParserError} whose {ParserError#line} is the
|
|
485
|
+
# physical line number in +source+.
|
|
486
|
+
#
|
|
487
|
+
# Pass a frozen string for zero-copy iteration (an unfrozen source is
|
|
488
|
+
# walked over a private copy, like {.lazy}). The block may itself
|
|
489
|
+
# call any NOSJ method.
|
|
490
|
+
#
|
|
491
|
+
# @example
|
|
492
|
+
# NOSJ.each_line(log) { |event| ingest(event) }
|
|
493
|
+
# NOSJ.each_line(log).first(10) # Enumerator when blockless
|
|
494
|
+
#
|
|
495
|
+
# @param source [String] newline-delimited JSON (UTF-8 or US-ASCII)
|
|
496
|
+
# @param opts [Hash, nil] the same options as {.parse}, applied per line
|
|
497
|
+
# @yieldparam value [Object] one parsed document per non-blank line
|
|
498
|
+
# @return [Enumerator] when no block is given, else +nil+
|
|
499
|
+
# @raise [ParserError] on the first malformed line
|
|
500
|
+
def self.each_line(source, opts = nil, &block)
|
|
501
|
+
return enum_for(:each_line, source, opts) unless block
|
|
502
|
+
each_line_native(source, opts, &block)
|
|
503
|
+
end
|
|
504
|
+
|
|
505
|
+
# {.each_line} against a file: the NDJSON stream is walked over a
|
|
506
|
+
# read-only memory map, so the file never becomes a Ruby String.
|
|
507
|
+
#
|
|
508
|
+
# @example
|
|
509
|
+
# NOSJ.each_line_file("events.ndjson") { |event| ingest(event) }
|
|
510
|
+
#
|
|
511
|
+
# @param path [String] the NDJSON file (UTF-8)
|
|
512
|
+
# @param opts [Hash, nil] the same options as {.parse}, applied per line
|
|
513
|
+
# @yieldparam value [Object] one parsed document per non-blank line
|
|
514
|
+
# @return [Enumerator] when no block is given, else +nil+
|
|
515
|
+
# @raise [SystemCallError] +Errno::ENOENT+ and friends
|
|
516
|
+
# @raise [ParserError] on the first malformed line
|
|
517
|
+
def self.each_line_file(path, opts = nil, &block)
|
|
518
|
+
return enum_for(:each_line_file, path, opts) unless block
|
|
519
|
+
each_line_file_native(path, opts, &block)
|
|
520
|
+
end
|
|
521
|
+
|
|
522
|
+
# Generates NDJSON / JSON Lines: one compact document per element,
|
|
523
|
+
# each terminated with a newline, built in a single pass into one
|
|
524
|
+
# buffer. Formatting options containing newlines raise ArgumentError
|
|
525
|
+
# (they would break the line framing); everything else from
|
|
526
|
+
# {.generate} applies.
|
|
527
|
+
#
|
|
528
|
+
# @example
|
|
529
|
+
# NOSJ.generate_lines([{a: 1}, {b: 2}]) #=> %({"a":1}\n{"b":2}\n)
|
|
530
|
+
#
|
|
531
|
+
# @param values [Array, Enumerable] one document per element
|
|
532
|
+
# @param opts [Hash, nil] the same options as {.generate}
|
|
533
|
+
# @return [String] the NDJSON document (empty when +values+ is empty)
|
|
534
|
+
# @raise [ArgumentError] for formatting options that contain newlines
|
|
535
|
+
# @raise [GeneratorError] (see {.generate})
|
|
536
|
+
def self.generate_lines(values, opts = nil)
|
|
537
|
+
generate_lines_native(lines_array(values), opts)
|
|
538
|
+
end
|
|
539
|
+
|
|
540
|
+
# {.generate_lines} straight to a file, streaming the generator's
|
|
541
|
+
# buffer to disk like {.write_file}.
|
|
542
|
+
#
|
|
543
|
+
# @example
|
|
544
|
+
# NOSJ.write_lines("out.ndjson", events) #=> bytes written
|
|
545
|
+
#
|
|
546
|
+
# @param path [String] the file to (over)write
|
|
547
|
+
# @param values [Array, Enumerable] one document per line
|
|
548
|
+
# @param opts [Hash, nil] the same options as {.generate}
|
|
549
|
+
# @return [Integer] the number of bytes written, like File.write
|
|
550
|
+
# @raise [SystemCallError] +Errno::ENOENT+ and friends
|
|
551
|
+
# @raise [ArgumentError] for formatting options that contain newlines
|
|
552
|
+
# @raise [GeneratorError] (see {.generate})
|
|
553
|
+
def self.write_lines(path, values, opts = nil)
|
|
554
|
+
write_lines_native(path, lines_array(values), opts)
|
|
555
|
+
end
|
|
556
|
+
|
|
557
|
+
# One document per element: Arrays pass through, other Enumerables
|
|
558
|
+
# convert, anything else (nil included, whose to_a would silently
|
|
559
|
+
# yield []) is a TypeError.
|
|
560
|
+
def self.lines_array(values)
|
|
561
|
+
return values if values.is_a?(Array)
|
|
562
|
+
unless values.is_a?(Enumerable)
|
|
563
|
+
raise TypeError, "no implicit conversion of #{values.class} into Array"
|
|
564
|
+
end
|
|
565
|
+
values.to_a
|
|
566
|
+
end
|
|
567
|
+
private_class_method :lines_array
|
|
568
|
+
|
|
569
|
+
# Document statistics from one full-parser pass into a counting sink:
|
|
570
|
+
# no Ruby value is built for the document itself, only the small
|
|
571
|
+
# result Hash. A debugging endpoint for "what is this 40 MB blob".
|
|
572
|
+
#
|
|
573
|
+
# The result (Symbol keys):
|
|
574
|
+
#
|
|
575
|
+
# {
|
|
576
|
+
# byte_size: 631, # of the document
|
|
577
|
+
# root: :object, # :object/:array/:string/:integer/
|
|
578
|
+
# # :float/:boolean/:null
|
|
579
|
+
# max_depth: 4, # container nesting, counted like
|
|
580
|
+
# # max_nesting (root container = 1)
|
|
581
|
+
# values: {total:, objects:, arrays:, strings:, integers:,
|
|
582
|
+
# floats:, booleans:, nulls:},
|
|
583
|
+
# keys: {total:, unique:},
|
|
584
|
+
# key_histogram: {"name" => 128, ...}, # sorted by count desc,
|
|
585
|
+
# # so .first(10) = top 10
|
|
586
|
+
# containers: {max_object_entries:, max_array_length:},
|
|
587
|
+
# strings: {bytes:, max_bytes:} # decoded UTF-8 bytes
|
|
588
|
+
# }
|
|
589
|
+
#
|
|
590
|
+
# Unlike {.parse}, nesting is UNLIMITED by default (a deep blob is
|
|
591
|
+
# exactly what a diagnostic should describe); pass +max_nesting+ to
|
|
592
|
+
# enforce a limit. Histogram memory is proportional to the number of
|
|
593
|
+
# unique keys.
|
|
594
|
+
#
|
|
595
|
+
# @example Top ten keys of a mystery blob
|
|
596
|
+
# NOSJ.stats(blob)[:key_histogram].first(10)
|
|
597
|
+
#
|
|
598
|
+
# @param source [String] the JSON document (UTF-8 or US-ASCII)
|
|
599
|
+
# @param opts [Hash, nil] +max_nesting+, +allow_nan+,
|
|
600
|
+
# +allow_trailing_comma+ (acceptance options only)
|
|
601
|
+
# @return [Hash] the statistics described above
|
|
602
|
+
# @raise [ParserError] when the document is malformed or not UTF-8
|
|
603
|
+
def self.stats(source, opts = nil)
|
|
604
|
+
stats_native(source, opts)
|
|
605
|
+
end
|
|
606
|
+
|
|
607
|
+
# {.stats} against a file: memory-maps it and runs the counting pass
|
|
608
|
+
# without reading the document into Ruby. +byte_size+ is the file
|
|
609
|
+
# size.
|
|
610
|
+
#
|
|
611
|
+
# @example
|
|
612
|
+
# NOSJ.stats_file("huge.json") => {byte_size: 41_943_040, ...}
|
|
613
|
+
#
|
|
614
|
+
# @param path [String] the file to inspect (UTF-8)
|
|
615
|
+
# @param opts [Hash, nil] same options as {.stats}
|
|
616
|
+
# @return [Hash] the statistics described on {.stats}
|
|
617
|
+
# @raise [SystemCallError] +Errno::ENOENT+ and friends
|
|
618
|
+
# @raise [ParserError] when the file is malformed or not UTF-8
|
|
619
|
+
def self.stats_file(path, opts = nil)
|
|
620
|
+
stats_file_native(path, opts)
|
|
621
|
+
end
|
|
276
622
|
end
|
data/sig/nosj.rbs
CHANGED
|
@@ -25,12 +25,32 @@ module NOSJ
|
|
|
25
25
|
class Error < StandardError
|
|
26
26
|
end
|
|
27
27
|
|
|
28
|
+
# Parse failures. Position accessors are nil for failures without
|
|
29
|
+
# one (encoding refusals); positions are byte-absolute within the
|
|
30
|
+
# document, lines and (character) columns 1-based.
|
|
31
|
+
class ParserError < Error
|
|
32
|
+
def byte_offset: () -> Integer?
|
|
33
|
+
|
|
34
|
+
def line: () -> Integer?
|
|
35
|
+
|
|
36
|
+
def column: () -> Integer?
|
|
37
|
+
|
|
38
|
+
def snippet: () -> String?
|
|
39
|
+
|
|
40
|
+
def detailed_message: (?highlight: bool, **untyped) -> String
|
|
41
|
+
end
|
|
42
|
+
|
|
28
43
|
class GeneratorError < Error
|
|
29
44
|
end
|
|
30
45
|
|
|
31
46
|
class NestingError < Error
|
|
32
47
|
end
|
|
33
48
|
|
|
49
|
+
# RFC 6902 application failures (failed test, missing target, index
|
|
50
|
+
# out of range, move into own child).
|
|
51
|
+
class PatchError < Error
|
|
52
|
+
end
|
|
53
|
+
|
|
34
54
|
def self.parse: (String source, ?opts opts) -> value
|
|
35
55
|
|
|
36
56
|
def self.generate: (untyped obj, ?opts opts) -> String
|
|
@@ -39,6 +59,12 @@ module NOSJ
|
|
|
39
59
|
|
|
40
60
|
def self.valid?: (String source, ?opts opts) -> bool
|
|
41
61
|
|
|
62
|
+
# Document statistics (byte_size, root, max_depth, values, keys,
|
|
63
|
+
# key_histogram, containers, strings) from one counting-sink pass.
|
|
64
|
+
def self.stats: (String source, ?opts opts) -> Hash[Symbol, untyped]
|
|
65
|
+
|
|
66
|
+
def self.stats_file: (String path, ?opts opts) -> Hash[Symbol, untyped]
|
|
67
|
+
|
|
42
68
|
def self.dig: (String source, *path_element path) -> value
|
|
43
69
|
|
|
44
70
|
def self.dig_many: (String source, Array[Array[path_element]] paths, ?opts opts) -> Array[value]
|
|
@@ -49,6 +75,36 @@ module NOSJ
|
|
|
49
75
|
|
|
50
76
|
def self.lazy: (String source, ?opts opts) -> (Lazy | value)
|
|
51
77
|
|
|
78
|
+
# Reformat without building values: parser events pipe straight into
|
|
79
|
+
# the emission kernels. minify is compact; reformat takes pretty: and
|
|
80
|
+
# the generate formatting/escape options.
|
|
81
|
+
def self.minify: (String json, ?opts opts) -> String
|
|
82
|
+
|
|
83
|
+
def self.reformat: (String json, ?opts opts) -> String
|
|
84
|
+
|
|
85
|
+
def self.reformat_file: (String path, ?opts opts) -> String
|
|
86
|
+
|
|
87
|
+
# Byte-splicing edits: JSON Pointer => replacement value, resolved
|
|
88
|
+
# in one pass; bytes outside the target spans are copied untouched.
|
|
89
|
+
def self.splice: (String json, Hash[String, untyped] edits, ?opts opts) -> String
|
|
90
|
+
|
|
91
|
+
# RFC 6902 JSON Patch over the raw document.
|
|
92
|
+
def self.patch: (String json, Array[Hash[untyped, untyped]] ops, ?opts opts) -> String
|
|
93
|
+
|
|
94
|
+
# RFC 7386 JSON Merge Patch (semantic: parse, merge, generate).
|
|
95
|
+
def self.merge_patch: (String json, untyped patch, ?opts opts) -> String
|
|
96
|
+
|
|
97
|
+
# NDJSON / JSON Lines: one parsed value per non-blank line.
|
|
98
|
+
def self.each_line: (String source, ?opts opts) { (value) -> void } -> nil
|
|
99
|
+
| (String source, ?opts opts) -> Enumerator[value, nil]
|
|
100
|
+
|
|
101
|
+
def self.each_line_file: (String path, ?opts opts) { (value) -> void } -> nil
|
|
102
|
+
| (String path, ?opts opts) -> Enumerator[value, nil]
|
|
103
|
+
|
|
104
|
+
def self.generate_lines: (_Each[untyped] values, ?opts opts) -> String
|
|
105
|
+
|
|
106
|
+
def self.write_lines: (String path, _Each[untyped] values, ?opts opts) -> Integer
|
|
107
|
+
|
|
52
108
|
def self.load_file: (String path, ?opts opts) -> value
|
|
53
109
|
|
|
54
110
|
def self.write_file: (String path, untyped obj, ?opts opts) -> Integer
|
|
@@ -101,6 +157,16 @@ module NOSJ
|
|
|
101
157
|
alias to_s inspect
|
|
102
158
|
end
|
|
103
159
|
|
|
160
|
+
# Defined by `require "nosj/rails"`, which also installs it as
|
|
161
|
+
# ActiveSupport::JSON::Encoding.json_encoder.
|
|
162
|
+
class RailsEncoder
|
|
163
|
+
attr_reader options: Hash[Symbol, untyped]
|
|
164
|
+
|
|
165
|
+
def initialize: (?Hash[Symbol, untyped]? options) -> void
|
|
166
|
+
|
|
167
|
+
def encode: (untyped value) -> String
|
|
168
|
+
end
|
|
169
|
+
|
|
104
170
|
# Defined by `require "nosj/multi_json"`. The runtime superclass is
|
|
105
171
|
# multi_json's Adapter (whose namespace differs across multi_json
|
|
106
172
|
# versions), so it is not declared here.
|