nosj 0.1.0-x86_64-linux → 0.3.0-x86_64-linux
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +110 -1
- data/README.md +272 -24
- data/lib/nosj/3.3/nosj.so +0 -0
- data/lib/nosj/3.4/nosj.so +0 -0
- data/lib/nosj/4.0/nosj.so +0 -0
- data/lib/nosj/json.rb +30 -6
- data/lib/nosj/lazy.rb +165 -0
- data/lib/nosj/multi_json.rb +1 -1
- data/lib/nosj/rails.rb +76 -0
- data/lib/nosj/version.rb +1 -1
- data/lib/nosj.rb +434 -5
- data/sig/nosj.rbs +120 -0
- metadata +9 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 734f363fe006f81b64302674599f01f78239515aba0111125b2db71a8ea70022
|
|
4
|
+
data.tar.gz: 2f13bcd853bd227de0420cc3c86c8452a2201705ea51192ac083c25fe0bf925b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 35091b15c193843620049fbc191beaad540c0774b3e6da311bf51dfdbf72f462e517ff3b527a474b0b62686d3ed4d828a4e610c9f5e6707055d19d61e7532327
|
|
7
|
+
data.tar.gz: 909edb0644da114fb00eba22c4c3d9a4d4e09a853b6f4652b28f62bb986f9342f5c7d91803126f213c3af8c4cc3070230f3bba222de1a7087cd47ccafa48d21b
|
data/CHANGELOG.md
CHANGED
|
@@ -1,4 +1,113 @@
|
|
|
1
|
-
## [0.
|
|
1
|
+
## [0.3.0] - 2026-07-17
|
|
2
|
+
|
|
3
|
+
- Reformat without parsing. `NOSJ.minify(json, opts)` and
|
|
4
|
+
`NOSJ.reformat(json, opts)` (plus `NOSJ.reformat_file`) pipe the
|
|
5
|
+
parser's events straight into the emission kernels: zero Ruby
|
|
6
|
+
objects are allocated for the document, and output is exactly
|
|
7
|
+
`generate(parse(json))`—canonical numbers, normalized escapes, the
|
|
8
|
+
full set of `generate` formatting and escape options, with
|
|
9
|
+
`pretty: true` as a `pretty_generate` shorthand—except duplicate
|
|
10
|
+
object keys pass through and lone-surrogate string values re-escape
|
|
11
|
+
as `\uXXXX` instead of raising (the output must always reparse).
|
|
12
|
+
Acceptance options apply per `parse` (`allow_trailing_comma`
|
|
13
|
+
normalizes the commas away). Measured on the 631 KB twitter.json:
|
|
14
|
+
409µs, 3.4× faster than `NOSJ.generate(NOSJ.parse(x))`, 3.9× faster
|
|
15
|
+
than gem json's cycle, 5.3× faster than Oj's, and 1.4× the cost of
|
|
16
|
+
`NOSJ.valid?`.
|
|
17
|
+
- Byte-splicing edits and JSON Patch. `NOSJ.splice(json, pointer =>
|
|
18
|
+
value, ...)` replaces values directly in the text: every target
|
|
19
|
+
resolves in one forward pass and the result is rebuilt copying all
|
|
20
|
+
bytes outside the target spans untouched (formatting, key order, and
|
|
21
|
+
number spellings elsewhere survive exactly). Measured on
|
|
22
|
+
twitter.json: 10× faster than parse-mutate-generate for a late
|
|
23
|
+
field, 51× for an early one. Missing targets raise KeyError,
|
|
24
|
+
overlapping targets ArgumentError. `NOSJ.patch(json, ops)` applies
|
|
25
|
+
RFC 6902 JSON Patch (add/remove/replace/move/copy/test, String or
|
|
26
|
+
Symbol op keys) to the raw string the same way, with structural ops
|
|
27
|
+
walking only the parent container's span; application failures raise
|
|
28
|
+
the new `NOSJ::PatchError`, malformed patch documents ArgumentError.
|
|
29
|
+
`NOSJ.merge_patch(json, patch)` applies RFC 7386 JSON Merge Patch
|
|
30
|
+
(semantic form). Inserted values are byte-identical to
|
|
31
|
+
`NOSJ.generate` and accept its options; the RFC 6902 appendix-A
|
|
32
|
+
suite and the full RFC 7386 test table are in the specs.
|
|
33
|
+
- NDJSON / JSON Lines. `NOSJ.each_line(source, opts)` yields one
|
|
34
|
+
parsed value per line (Enumerator without a block, so
|
|
35
|
+
`.first(10)`/`.lazy` walk only what they consume), skipping blank
|
|
36
|
+
lines and enforcing one value per line; a malformed line raises the
|
|
37
|
+
rich `ParserError` whose `#line` is the physical line number in the
|
|
38
|
+
stream. `NOSJ.generate_lines(values, opts)` emits one compact
|
|
39
|
+
newline-terminated document per element in a single buffer pass
|
|
40
|
+
(measured 4.1× faster than the map-generate-join idiom on twitter
|
|
41
|
+
statuses; `each_line` is 1.6× faster than line-split +
|
|
42
|
+
`JSON.parse`), rejecting formatting options that would break line
|
|
43
|
+
framing. File forms: `NOSJ.each_line_file` streams over a read-only
|
|
44
|
+
memory map, `NOSJ.write_lines` generates straight to disk and
|
|
45
|
+
returns the byte count. Parse options apply per line; pass a frozen
|
|
46
|
+
string to `each_line` for zero-copy iteration.
|
|
47
|
+
- `NOSJ.stats(source, opts)` / `NOSJ.stats_file(path, opts)`: document
|
|
48
|
+
statistics from one counting pass through the null-sink machinery,
|
|
49
|
+
answering "what is this 40 MB blob" without building any Ruby values
|
|
50
|
+
for the document (measured ~1.3× faster than a full parse). Reports
|
|
51
|
+
`byte_size`, `root` kind, `max_depth`, value counts by type, key
|
|
52
|
+
totals, a key histogram sorted by count, largest container sizes,
|
|
53
|
+
and string byte totals. Nesting is unlimited by default (pass
|
|
54
|
+
`max_nesting` to enforce a limit); `allow_nan` and
|
|
55
|
+
`allow_trailing_comma` are honored; malformed documents raise the
|
|
56
|
+
rich `ParserError`. The file form memory-maps, so the document never
|
|
57
|
+
enters Ruby at all.
|
|
58
|
+
- Rich parse errors. Parse failures now raise `NOSJ::ParserError`
|
|
59
|
+
(previously a bare `RuntimeError`) carrying the failure position,
|
|
60
|
+
computed only when a parse fails: `#byte_offset`, 1-based `#line`,
|
|
61
|
+
character-based `#column`, and a caret `#snippet` showing the
|
|
62
|
+
offending line (windowed when the line is long, as minified JSON
|
|
63
|
+
usually is). Positions are absolute within the document you passed,
|
|
64
|
+
including through partial parsing (`dig`, `at_pointer`, batches),
|
|
65
|
+
lazy documents, and the file APIs. `#detailed_message` appends the
|
|
66
|
+
snippet, so unrescued errors print it. Failures with no position
|
|
67
|
+
(encoding refusals) leave the accessors nil. Exceeding `max_nesting`
|
|
68
|
+
during parsing now raises `NOSJ::NestingError`, matching the gem's
|
|
69
|
+
class (generation already did); rescues of the old `RuntimeError`
|
|
70
|
+
need updating to `NOSJ::ParserError`/`NOSJ::Error`.
|
|
71
|
+
- Rails mode: `require "nosj/rails"` accelerates a Rails application
|
|
72
|
+
in both directions. It installs a nosj-backed ActiveSupport JSON
|
|
73
|
+
encoder, so `obj.to_json`, `render json:`, and `ActiveSupport::JSON.encode` walk the object tree natively—values recurse through `as_json` exactly
|
|
74
|
+
like ActiveSupport's own encoder. It also loads the `nosj/json` drop-in, so
|
|
75
|
+
`ActiveSupport::JSON.decode` and JSON request-body parsing take the
|
|
76
|
+
fast path (including on Rails 7.x, whose `quirks_mode` option the
|
|
77
|
+
drop-in now accepts; the drop-in also accepts valid-UTF-8 BINARY
|
|
78
|
+
strings now, which is what Rack delivers request bodies as). The
|
|
79
|
+
HTML-safety escaping is fused into the SIMD string-emission kernels,
|
|
80
|
+
so escaped output costs the same single pass as unescaped. Measured
|
|
81
|
+
against stock ActiveSupport encoding: ×1.7 on small documents up to
|
|
82
|
+
×5.2 on large trees and ×14 on HTML-heavy content
|
|
83
|
+
(`rake bench:rails`). In a Rails Gemfile:
|
|
84
|
+
`gem "nosj", require: "nosj/rails"`.
|
|
85
|
+
- `JSON::Fragment` values now splice their pre-rendered JSON
|
|
86
|
+
everywhere the `json` gem does: in default mode, under `strict:
|
|
87
|
+
true`, and through the Rails encoder.
|
|
88
|
+
|
|
89
|
+
## [0.2.0] - 2026-07-16
|
|
90
|
+
|
|
91
|
+
- File APIs. `NOSJ.load_file(path, opts)` parses a file directly
|
|
92
|
+
(~1.3× faster than `parse(File.read(path))`—no file-sized Ruby
|
|
93
|
+
String is created), and `NOSJ.write_file(path, obj, opts)` generates
|
|
94
|
+
straight to disk, returning the byte count like `File.write`.
|
|
95
|
+
`NOSJ.load_lazy_file(path, opts)` wraps a file as a lazy document
|
|
96
|
+
over a read-only memory map, and `NOSJ.at_pointer_file` /
|
|
97
|
+
`NOSJ.dig_file` pull single values out of a file without reading the
|
|
98
|
+
rest into Ruby. Missing files raise the usual `Errno` exceptions.
|
|
99
|
+
- `NOSJ.lazy`: lazy documents. Wrap a document once, then read only
|
|
100
|
+
what you need: `doc["users"][3]["name"]` parses just that path, `#dig`
|
|
101
|
+
and `#at_pointer` resolve whole paths, and `#keys`, `#size`, and
|
|
102
|
+
`#each` inspect a node without parsing its values. Containers come
|
|
103
|
+
back lazy, scalars come back as plain Ruby values, and repeated
|
|
104
|
+
reads are cached. `#value` (also `#to_h` / `#to_a`) materializes a
|
|
105
|
+
subtree under the usual parse options (`symbolize_names`, `freeze`,
|
|
106
|
+
...). Pass a frozen string and creating the view is practically
|
|
107
|
+
free, even on megabyte documents. Malformed content raises on first
|
|
108
|
+
read, not at wrap time.
|
|
109
|
+
|
|
110
|
+
## [0.1.0] - 2026-07-16
|
|
2
111
|
|
|
3
112
|
Initial release.
|
|
4
113
|
|
data/README.md
CHANGED
|
@@ -10,12 +10,17 @@
|
|
|
10
10
|
- It is **faster** than gem json and every
|
|
11
11
|
third-party parser, including Oj, RapidJSON, FastJsonparser, Yajl. 1.0–1.8× faster than the bundled json gem, 1.3–11× faster than Oj, and up to 17×
|
|
12
12
|
faster than Yajl—[see Benchmarks](#benchmarks).
|
|
13
|
+
- It has **lazy documents**: `NOSJ.lazy` wraps a document and parses a value only when you touch it—repeated access costs nanoseconds, and everything you never read is never parsed.
|
|
14
|
+
- It has a **partial parsing mode**: JSON Pointer lookups that pull single values out of big documents in microseconds, skipping everything else.
|
|
15
|
+
- It has **file APIs**: parse, generate, dig, and lazy-wrap files directly—no throwaway file-sized Ruby String, and the partial modes memory-map the file so unread pages never even leave the disk.
|
|
16
|
+
- It does **byte-splicing edits**: `NOSJ.splice` changes a field inside a passing payload 10–51× faster than parse-mutate-generate, and RFC 6902 JSON Patch / RFC 7386 merge patch apply straight to the raw string.
|
|
17
|
+
- It is **great to debug with**: parse errors carry line, column, and a caret snippet pointing at the break, and `NOSJ.stats` X-rays a mystery blob (depth, value counts, key histogram) faster than parsing it.
|
|
18
|
+
- It accelerates a **Rails** application in both encoding and decoding.
|
|
13
19
|
- It comes **precompiled** (platform gems built with per-platform optimizations,
|
|
14
20
|
nothing to compile on install).
|
|
15
|
-
-
|
|
16
|
-
- Same API and option names as gem json.
|
|
21
|
+
- Otherwise, same API and option names as gem json.
|
|
17
22
|
|
|
18
|
-
**And there's more**: validate documents without building a single Ruby object, resolve whole batches of paths in one pass, and accelerate an entire application with a one-line drop-in.
|
|
23
|
+
**And there's more**: validate documents without building a single Ruby object, minify and pretty-print without parsing, resolve whole batches of paths in one pass, stream NDJSON / JSON Lines in both directions, and accelerate an entire application with a one-line drop-in.
|
|
19
24
|
|
|
20
25
|
- [Requirements](#requirements)
|
|
21
26
|
- [Getting started](#getting-started)
|
|
@@ -24,6 +29,7 @@ nothing to compile on install).
|
|
|
24
29
|
- [Switching from the json gem](#switching-from-the-json-gem)
|
|
25
30
|
- [How it works](#how-it-works)
|
|
26
31
|
- [Development](#development)
|
|
32
|
+
- [Acknowledgements](#acknowledgements)
|
|
27
33
|
- [License](#license)
|
|
28
34
|
|
|
29
35
|
## Requirements
|
|
@@ -48,6 +54,8 @@ NOSJ.generate({"a" => [1, true]}) #=> '{"a":[1,true]}'
|
|
|
48
54
|
|
|
49
55
|
That's it—if you know the `json` gem, you already know `nosj`.
|
|
50
56
|
|
|
57
|
+
### gem json compatibility
|
|
58
|
+
|
|
51
59
|
Want the speedup without touching your code? One line reroutes
|
|
52
60
|
`JSON.parse`, `JSON.generate`, `JSON.pretty_generate`, and `JSON.dump`
|
|
53
61
|
through nosj:
|
|
@@ -63,18 +71,39 @@ Gemfile you can do this:
|
|
|
63
71
|
gem "nosj", require: "nosj/json"
|
|
64
72
|
```
|
|
65
73
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
+
A MultiJson adapter ships too:
|
|
75
|
+
|
|
76
|
+
```ruby
|
|
77
|
+
require "nosj/multi_json"
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
And then `MultiJson.use NOSJ::MultiJsonAdapter`.
|
|
81
|
+
|
|
82
|
+
### Ruby on Rails
|
|
83
|
+
|
|
84
|
+
In a Rails app, use this for "Rails mode":
|
|
85
|
+
|
|
86
|
+
```ruby
|
|
87
|
+
gem "nosj", require: "nosj/rails"
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
That installs a nosj-backed ActiveSupport JSON encoder, so `obj.to_json`, `render json:`, and `ActiveSupport::JSON.encode` walk the object tree natively:
|
|
91
|
+
values that aren't JSON-native recurse through `as_json` exactly like
|
|
92
|
+
ActiveSupport's own encoder, non-finite floats encode as `null`, and
|
|
93
|
+
HTML-safety escaping (`escape_html_entities_in_json`) behaves
|
|
94
|
+
identically—verified differentially against ActiveSupport's encoder.
|
|
95
|
+
It loads the drop-in too, so `ActiveSupport::JSON.decode` and JSON
|
|
96
|
+
request-body parsing ride the fast path.
|
|
97
|
+
|
|
98
|
+
Measured against ActiveSupport's own encoder: ×1.9 on small documents
|
|
99
|
+
up to ×5.2 on large trees and ×14 on HTML-heavy content—see
|
|
100
|
+
[Benchmarks → Rails mode](#rails-mode).
|
|
74
101
|
|
|
75
102
|
## What's in the box
|
|
76
103
|
|
|
77
|
-
|
|
104
|
+
### json API
|
|
105
|
+
|
|
106
|
+
The `json` gem API, on the `NOSJ` module:
|
|
78
107
|
|
|
79
108
|
```ruby
|
|
80
109
|
NOSJ.parse(src, symbolize_names: true) # also: freeze, max_nesting,
|
|
@@ -83,18 +112,27 @@ NOSJ.generate(obj) # indent, space, object_nl, ...,
|
|
|
83
112
|
NOSJ.pretty_generate(obj) # ascii_only, script_safe, strict
|
|
84
113
|
```
|
|
85
114
|
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
115
|
+
### Lazy documents
|
|
116
|
+
|
|
117
|
+
`NOSJ.lazy` wraps a document in a lazy view: read
|
|
118
|
+
a field and only that path is parsed—containers stay lazy, scalars
|
|
119
|
+
arrive as plain Ruby values, and repeated reads are cached:
|
|
90
120
|
|
|
91
121
|
```ruby
|
|
92
|
-
NOSJ.
|
|
93
|
-
|
|
94
|
-
|
|
122
|
+
doc = NOSJ.lazy(json)
|
|
123
|
+
doc["users"][3]["name"] # parses only this path
|
|
124
|
+
doc.dig("meta", "count") # a whole path in one fused resolution
|
|
125
|
+
doc["users"].size # counted without materializing anything
|
|
126
|
+
doc["users"][3].value # materialize one subtree (parse options apply)
|
|
95
127
|
```
|
|
96
128
|
|
|
97
|
-
|
|
129
|
+
Pass a frozen string and creating the view is practically free—
|
|
130
|
+
nanoseconds, even on a megabyte document. Malformed content raises
|
|
131
|
+
when it is first read, not at wrap time.
|
|
132
|
+
|
|
133
|
+
### Partial parsing
|
|
134
|
+
|
|
135
|
+
Pull values out of a document without
|
|
98
136
|
materializing the rest—skipped content is stepped over at SIMD block
|
|
99
137
|
speed, so a lookup costs what it skips, not what the document weighs:
|
|
100
138
|
|
|
@@ -114,11 +152,159 @@ at the far end of a 570 KB document costs ~71µs, still 13× faster
|
|
|
114
152
|
than parse-then-dig. Misses return nil; matched subtrees materialize
|
|
115
153
|
with the same options as `parse` (`symbolize_names:`, `freeze:`).
|
|
116
154
|
|
|
155
|
+
### Files API
|
|
156
|
+
|
|
157
|
+
Every mode has a file-native form, so a document never
|
|
158
|
+
round-trips through a throwaway Ruby String:
|
|
159
|
+
|
|
160
|
+
```ruby
|
|
161
|
+
NOSJ.load_file("config.json") # 1.3× File.read + parse
|
|
162
|
+
NOSJ.write_file("out.json", obj) # generate straight to disk
|
|
163
|
+
NOSJ.dig_file("huge.json", "users", 3, "name") # never reads the rest
|
|
164
|
+
NOSJ.at_pointer_file("huge.json", "/meta/count")
|
|
165
|
+
doc = NOSJ.load_lazy_file("huge.json") # lazy view over a memory map
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
The partial and lazy forms memory-map the file, so pages you never
|
|
169
|
+
read are never loaded from disk. Missing files raise the usual
|
|
170
|
+
`Errno` exceptions. Measured numbers live in
|
|
171
|
+
[Benchmarks → File APIs](#file-apis).
|
|
172
|
+
|
|
173
|
+
### Reformat without parsing
|
|
174
|
+
|
|
175
|
+
`NOSJ.minify` and `NOSJ.reformat` pipe the parser's events straight
|
|
176
|
+
into the emission kernels: **zero Ruby objects** are built for the
|
|
177
|
+
document (the specs assert it), SIMD in and SIMD out. Minifying the
|
|
178
|
+
631 KB twitter.json takes 409µs—3.4× faster than a
|
|
179
|
+
`generate(parse(x))` round-trip, 3.9× faster than gem json's cycle,
|
|
180
|
+
and only 1.4× the cost of `valid?`:
|
|
181
|
+
|
|
182
|
+
```ruby
|
|
183
|
+
NOSJ.minify(json) # compact
|
|
184
|
+
NOSJ.reformat(json, pretty: true) # pretty_generate layout
|
|
185
|
+
NOSJ.reformat(json, ascii_only: true) # escape-transcode, no parse
|
|
186
|
+
NOSJ.reformat_file("big.json") # straight off a memory map
|
|
187
|
+
```
|
|
188
|
+
|
|
189
|
+
Output is exactly `generate(parse(json))`—canonical number spellings,
|
|
190
|
+
normalized escapes, same formatting options—except duplicate keys pass
|
|
191
|
+
through (a reformatter must not silently drop data) and lone-surrogate
|
|
192
|
+
strings re-escape instead of raising. Acceptance options apply too:
|
|
193
|
+
`minify(src, allow_trailing_comma: true)` normalizes the commas away.
|
|
194
|
+
|
|
195
|
+
### Byte-splicing edits and JSON Patch
|
|
196
|
+
|
|
197
|
+
Change a field in a passing payload without the parse → mutate →
|
|
198
|
+
generate cycle: `NOSJ.splice` skips to the target spans (one pass for
|
|
199
|
+
the whole batch) and rebuilds the string around them—every byte
|
|
200
|
+
outside the targets is copied untouched, so formatting, key order,
|
|
201
|
+
and number spellings survive exactly. On twitter.json this is
|
|
202
|
+
**10×–51× faster** than parse-mutate-generate (129µs for a late
|
|
203
|
+
field, 26µs for an early one, vs ~1.3ms):
|
|
204
|
+
|
|
205
|
+
```ruby
|
|
206
|
+
NOSJ.splice(json, "/config/timeout" => 30)
|
|
207
|
+
NOSJ.splice(json, "/a" => 1, "/b/c" => [true]) # batch, one pass
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
On top of the same machinery: **RFC 6902 JSON Patch** applied to the
|
|
211
|
+
raw string (`add`, `remove`, `replace`, `move`, `copy`, `test`—the
|
|
212
|
+
whole appendix-A suite is in the specs), and **RFC 7386 JSON Merge
|
|
213
|
+
Patch**:
|
|
214
|
+
|
|
215
|
+
```ruby
|
|
216
|
+
NOSJ.patch(json, [
|
|
217
|
+
{"op" => "test", "path" => "/a", "value" => 1},
|
|
218
|
+
{"op" => "replace", "path" => "/a", "value" => 2},
|
|
219
|
+
{"op" => "add", "path" => "/list/-", "value" => "x"}
|
|
220
|
+
])
|
|
221
|
+
NOSJ.merge_patch(json, {"config" => {"legacy" => nil, "timeout" => 30}})
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
Missing splice targets raise `KeyError` (use `patch` `add` to
|
|
225
|
+
insert); failed patches raise `NOSJ::PatchError`; inserted values are
|
|
226
|
+
byte-identical to `NOSJ.generate`'s output.
|
|
227
|
+
|
|
228
|
+
### NDJSON / JSON Lines
|
|
229
|
+
|
|
230
|
+
Logs, data pipelines, LLM batch files—one JSON document per line.
|
|
231
|
+
`NOSJ.each_line` streams values out (1.6× faster than the
|
|
232
|
+
line-split-and-`JSON.parse` idiom on twitter statuses), and
|
|
233
|
+
`NOSJ.generate_lines` builds the whole stream in one buffer pass
|
|
234
|
+
(4.1× faster than map-generate-join):
|
|
235
|
+
|
|
236
|
+
```ruby
|
|
237
|
+
NOSJ.each_line(log) { |event| ingest(event) } # or an Enumerator:
|
|
238
|
+
NOSJ.each_line(log).first(10) # walks only 10 lines
|
|
239
|
+
|
|
240
|
+
NOSJ.generate_lines(events) #=> %({"a":1}\n{"b":2}\n...)
|
|
241
|
+
NOSJ.each_line_file("events.ndjson") { |e| } # over a memory map
|
|
242
|
+
NOSJ.write_lines("out.ndjson", events) # straight to disk
|
|
243
|
+
```
|
|
244
|
+
|
|
245
|
+
Blank lines are skipped, one value per line is enforced, and a
|
|
246
|
+
malformed line raises a [rich `ParserError`](#rich-parse-errors) whose
|
|
247
|
+
`#line` is the physical line in the stream. Parse options apply per
|
|
248
|
+
line—`NOSJ.each_line(log, freeze: true)` yields Ractor-shareable
|
|
249
|
+
values.
|
|
250
|
+
|
|
251
|
+
### Validation without parsing
|
|
252
|
+
|
|
253
|
+
`NOSJ.valid?` runs the full parser—tokenizers, string decode,
|
|
254
|
+
number validation—into a null sink and allocates no Ruby objects
|
|
255
|
+
at all. It is 2-4× faster than `NOSJ.parse`:
|
|
256
|
+
|
|
257
|
+
```ruby
|
|
258
|
+
NOSJ.valid?('{"a":1}') #=> true
|
|
259
|
+
NOSJ.valid?('{"a":}') #=> false
|
|
260
|
+
NOSJ.valid?(src, max_nesting: false) # same options as parse
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
### Document statistics
|
|
264
|
+
|
|
265
|
+
`NOSJ.stats` answers "what is this 40 MB blob": one counting pass
|
|
266
|
+
through the same null-sink machinery—no Ruby value is built for
|
|
267
|
+
the document, and it costs _less_ than a parse (~1.3× faster on `twitter.json`):
|
|
268
|
+
|
|
269
|
+
```ruby
|
|
270
|
+
s = NOSJ.stats(blob) # or NOSJ.stats_file("huge.json")
|
|
271
|
+
s[:byte_size] #=> 631514
|
|
272
|
+
s[:root] #=> :object
|
|
273
|
+
s[:max_depth] #=> 10
|
|
274
|
+
s[:values] #=> {total: 13914, objects: 1264, arrays: 1050,
|
|
275
|
+
# strings: 4754, integers: 2108, ...}
|
|
276
|
+
s[:keys] #=> {total: 13345, unique: 94}
|
|
277
|
+
s[:key_histogram].first(3) #=> [["id", 447], ["id_str", 447], ["urls", 364]]
|
|
278
|
+
s[:containers] #=> {max_object_entries: 40, max_array_length: 100}
|
|
279
|
+
s[:strings] #=> {bytes: 200716, max_bytes: 463}
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
Nesting is unlimited by default (a deep blob is exactly what a
|
|
283
|
+
diagnostic should describe); malformed documents raise the usual rich
|
|
284
|
+
`ParserError`.
|
|
285
|
+
|
|
286
|
+
### Rich parse errors
|
|
287
|
+
|
|
288
|
+
A failed parse raises `NOSJ::ParserError`
|
|
289
|
+
carrying where the document broke—`#byte_offset`, `#line`, `#column`,
|
|
290
|
+
and a caret `#snippet`—computed only when a parse fails, so success
|
|
291
|
+
pays nothing. Positions stay absolute through partial parsing, lazy
|
|
292
|
+
documents, and the file APIs; unrescued errors print the snippet:
|
|
293
|
+
|
|
294
|
+
```ruby
|
|
295
|
+
NOSJ.parse(%({\n "a": 1,\n "b": }))
|
|
296
|
+
# NOSJ::ParserError: unexpected character at byte 19
|
|
297
|
+
# e.line #=> 3
|
|
298
|
+
# e.column #=> 8
|
|
299
|
+
# e.snippet #=> "b": }
|
|
300
|
+
# ^
|
|
301
|
+
```
|
|
302
|
+
|
|
117
303
|
## Benchmarks
|
|
118
304
|
|
|
119
305
|
Every installed JSON gem, benchmark-ips: AWS EC2 c7a.2xlarge (AMD EPYC 9R14, Zen 4), Ruby 4.0.6 + YJIT, json 2.21.1, Oj 3.17.4, RapidJSON 0.4.0, FastJsonparser 0.6.0, Yajl 1.4.3, PGO build, 2026-07-16. `×N` = times slower than nosj.
|
|
120
306
|
|
|
121
|
-
Parse
|
|
307
|
+
### Parse
|
|
122
308
|
|
|
123
309
|
| file | nosj (i/s) | json | Oj | FastJsonparser | RapidJSON | Yajl |
|
|
124
310
|
|---|---:|---:|---:|---:|---:|---:|
|
|
@@ -136,7 +322,7 @@ Parse:
|
|
|
136
322
|
| tolstoy | **8.9k** | ×1.79 | ×1.96 | ×2.29 | ×2.10 | ×17.31 |
|
|
137
323
|
| twitter | **1.1k** | ×1.09 | ×1.83 | ×2.25 | ×2.61 | ×5.40 |
|
|
138
324
|
|
|
139
|
-
Generate
|
|
325
|
+
### Generate
|
|
140
326
|
|
|
141
327
|
| file | nosj (i/s) | json | Oj | RapidJSON | Yajl |
|
|
142
328
|
|---|---:|---:|---:|---:|---:|
|
|
@@ -157,6 +343,44 @@ Generate:
|
|
|
157
343
|
\* canada-generate is a statistical tie with the json gem (within
|
|
158
344
|
measurement error).
|
|
159
345
|
|
|
346
|
+
### File APIs
|
|
347
|
+
|
|
348
|
+
twitter.json (570 KB) from a warm page cache, medians of 7 alternating
|
|
349
|
+
rounds against the plain-Ruby composition on the same parser (Apple
|
|
350
|
+
Silicon dev box, Ruby 4.0.6 + YJIT, PGO build, 2026-07-16):
|
|
351
|
+
|
|
352
|
+
| operation | µs/op | vs the Ruby way |
|
|
353
|
+
|---|---:|---|
|
|
354
|
+
| `NOSJ.load_file` (parse the whole file) | 948 | ×1.33 vs `NOSJ.parse(File.read(path))` |
|
|
355
|
+
| `NOSJ.dig_file` (one deep field) | 246 | ×5.2 vs read + parse + dig |
|
|
356
|
+
| `NOSJ.load_lazy_file` + one field | 257 | ×5.0 vs read + parse + dig |
|
|
357
|
+
|
|
358
|
+
`NOSJ.write_file` measures at parity with
|
|
359
|
+
`File.write(NOSJ.generate(obj))` on this box—file-write timings swing
|
|
360
|
+
too much for an honest multiplier; what it saves is the intermediate
|
|
361
|
+
file-sized Ruby String.
|
|
362
|
+
|
|
363
|
+
### Rails mode
|
|
364
|
+
|
|
365
|
+
`ActiveSupport::JSON.encode` with the nosj encoder installed, against
|
|
366
|
+
stock ActiveSupport (Apple Silicon dev box, Ruby 4.0.6 + YJIT,
|
|
367
|
+
activesupport 8.1, medians of 5 interleaved per-process rounds,
|
|
368
|
+
outputs verified byte-identical first, 2026-07-17;
|
|
369
|
+
`rake bench:rails`):
|
|
370
|
+
|
|
371
|
+
| workload | nosj (i/s) | vs ActiveSupport |
|
|
372
|
+
|---|---:|---|
|
|
373
|
+
| twitter tree (570 KB) | 4.5k | ×5.2 |
|
|
374
|
+
| 100-record index (with timestamps) | 33.7k | ×1.7 |
|
|
375
|
+
| HTML-heavy user content | 177.7k | ×14.2 |
|
|
376
|
+
| Time/Date/BigDecimal hash | 1.1M | ×3.0 |
|
|
377
|
+
| small API hash | 4.2M | ×1.9 |
|
|
378
|
+
| small hash `to_json` | 4.1M | ×1.9 |
|
|
379
|
+
|
|
380
|
+
The HTML-safety escaping that dominates stock encodes of
|
|
381
|
+
user-generated content is fused into the SIMD string-emission kernels
|
|
382
|
+
here: escaped output costs the same single pass as unescaped.
|
|
383
|
+
|
|
160
384
|
Reproduce with `rake bench` (the parity-gated comparison, after a PGO retrain—the shipping configuration) or `rake bench:ips` (the multi-gem shoot-out).
|
|
161
385
|
|
|
162
386
|
## Switching from the json gem
|
|
@@ -170,8 +394,10 @@ You mostly don't have to do anything. Some differences:
|
|
|
170
394
|
UTF-8) follow the strict semantics instead.
|
|
171
395
|
- Unlike `Array#dig`, negative indices in `NOSJ.dig` return nil (JSON
|
|
172
396
|
Pointer has no equivalent).
|
|
173
|
-
- Parse
|
|
174
|
-
|
|
397
|
+
- Parse errors raise `NOSJ::ParserError` (`NOSJ::NestingError` past
|
|
398
|
+
`max_nesting`, like the gem); messages use byte offsets rather than
|
|
399
|
+
the gem's phrasing, and the exception carries `#line`, `#column`,
|
|
400
|
+
and a caret `#snippet`.
|
|
175
401
|
|
|
176
402
|
Everything else—including the gem's exact float formatting, which is
|
|
177
403
|
not the shortest-round-trip form most libraries emit—matches
|
|
@@ -208,6 +434,28 @@ bundle exec rake bench:fast # the sweep without retraining
|
|
|
208
434
|
bundle exec rake "bench:ips[twitter]" # multi-gem shoot-out (benchmark-ips); no args = full corpus
|
|
209
435
|
```
|
|
210
436
|
|
|
437
|
+
## Acknowledgements
|
|
438
|
+
|
|
439
|
+
Thanks to [Jean Boussier](https://github.com/byroot), [Florian Frank](https://github.com/flori), [Hiroshi Shibata](https://github.com/hsbt), [Nobuyoshi Nakada](https://github.com/nobu), [Étienne Barrié](https://github.com/etiennebarrie), and the other authors and maintainers of the [json gem](https://github.com/ruby/json)—for their work on the gem itself, for optimization ideas, and for some of the JSON documents in the benchmark corpus.
|
|
440
|
+
|
|
441
|
+
Thanks to [Mat Sadler](https://github.com/matsadler) for [magnus](https://github.com/matsadler/magnus).
|
|
442
|
+
|
|
443
|
+
Thanks to the authors of the projects credited in [NOTICE](NOTICE) and the
|
|
444
|
+
crate's NOTICE: [fpconv](https://github.com/night-shift/fpconv) and
|
|
445
|
+
Florian Loitsch's Grisu2,
|
|
446
|
+
[fast_float](https://github.com/fastfloat/fast_float) and
|
|
447
|
+
[fast_double_parser](https://github.com/lemire/fast_double_parser)
|
|
448
|
+
(Daniel Lemire et al.),
|
|
449
|
+
[simdjson](https://github.com/simdjson/simdjson) (Daniel Lemire, Geoff
|
|
450
|
+
Langdale, and contributors), and the architectural lineage of
|
|
451
|
+
[yyjson](https://github.com/ibireme/yyjson),
|
|
452
|
+
[simd-json](https://github.com/simd-lite/simd-json), and
|
|
453
|
+
[sonic-rs](https://github.com/cloudwego/sonic-rs).
|
|
454
|
+
|
|
455
|
+
## Assisted by
|
|
456
|
+
|
|
457
|
+
Claude Fable 5, Claude Opus 4.8.
|
|
458
|
+
|
|
211
459
|
## License
|
|
212
460
|
|
|
213
461
|
MIT. The underlying Rust crate is `MIT AND BSL-1.0 AND Apache-2.0`; its
|
data/lib/nosj/3.3/nosj.so
CHANGED
|
Binary file
|
data/lib/nosj/3.4/nosj.so
CHANGED
|
Binary file
|
data/lib/nosj/4.0/nosj.so
CHANGED
|
Binary file
|
data/lib/nosj/json.rb
CHANGED
|
@@ -6,7 +6,7 @@
|
|
|
6
6
|
#
|
|
7
7
|
# reroutes JSON.parse, JSON.generate, JSON.pretty_generate and JSON.dump
|
|
8
8
|
# through NOSJ whenever the requested options fall within NOSJ's
|
|
9
|
-
# supported set, and falls back to
|
|
9
|
+
# supported set, and falls back to gem json's own implementation for
|
|
10
10
|
# everything else (create_additions, object_class/array_class,
|
|
11
11
|
# decimal_class, on_load procs, JSON::State instances, IO arguments).
|
|
12
12
|
# Entry points built on JSON.parse (JSON.load, JSON.parse!,
|
|
@@ -30,8 +30,11 @@ module NOSJ
|
|
|
30
30
|
# Implementation detail of `require "nosj/json"`.
|
|
31
31
|
# @private
|
|
32
32
|
module JSONDropIn
|
|
33
|
+
# quirks_mode rides the fast path because NOSJ.parse is always
|
|
34
|
+
# quirks-mode (top-level scalars parse) and ignores the key; Rails
|
|
35
|
+
# 7.x passes it from ActiveSupport::JSON.decode.
|
|
33
36
|
PARSE_OPTS = %i[symbolize_names freeze max_nesting allow_nan
|
|
34
|
-
allow_trailing_comma].freeze
|
|
37
|
+
allow_trailing_comma quirks_mode].freeze
|
|
35
38
|
GENERATE_OPTS = %i[indent space space_before object_nl array_nl
|
|
36
39
|
max_nesting allow_nan ascii_only script_safe
|
|
37
40
|
escape_slash strict depth
|
|
@@ -41,7 +44,7 @@ module NOSJ
|
|
|
41
44
|
|
|
42
45
|
# The fast path handles nil or a plain Hash whose every key NOSJ
|
|
43
46
|
# implements; anything else (JSON::State, exotic options, string
|
|
44
|
-
# keys) belongs to
|
|
47
|
+
# keys) belongs to gem json.
|
|
45
48
|
def supported?(opts, allowed)
|
|
46
49
|
return true if opts.nil?
|
|
47
50
|
return false unless opts.instance_of?(Hash)
|
|
@@ -50,8 +53,29 @@ module NOSJ
|
|
|
50
53
|
end
|
|
51
54
|
|
|
52
55
|
def parse(source, opts)
|
|
56
|
+
# NOSJ.parse is deliberately strict about encodings (json-3.0
|
|
57
|
+
# semantics), but the drop-in must match the installed gem, which
|
|
58
|
+
# accepts more. BINARY strings holding valid UTF-8 are the big
|
|
59
|
+
# real-world case: Rack delivers request bodies as BINARY, so
|
|
60
|
+
# Rails JSON params come through here. Retagging a dup is cheap
|
|
61
|
+
# (copy-on-write bytes), and the validity scan is memoized
|
|
62
|
+
# coderange the parse would compute anyway. Anything else
|
|
63
|
+
# non-UTF-8 (UTF-16, ...) belongs to gem json, which transcodes.
|
|
64
|
+
if source.is_a?(String)
|
|
65
|
+
case source.encoding
|
|
66
|
+
when Encoding::UTF_8, Encoding::US_ASCII
|
|
67
|
+
# the fast path as-is
|
|
68
|
+
when Encoding::BINARY
|
|
69
|
+
utf8 = source.dup.force_encoding(Encoding::UTF_8)
|
|
70
|
+
source = utf8 if utf8.valid_encoding?
|
|
71
|
+
else
|
|
72
|
+
return ::JSON.nosj_original_parse(source, **(opts || {}))
|
|
73
|
+
end
|
|
74
|
+
end
|
|
53
75
|
NOSJ.parse(source, opts)
|
|
54
|
-
rescue
|
|
76
|
+
rescue NOSJ::NestingError => e
|
|
77
|
+
raise ::JSON::NestingError, e.message
|
|
78
|
+
rescue NOSJ::ParserError => e
|
|
55
79
|
raise ::JSON::ParserError, e.message
|
|
56
80
|
end
|
|
57
81
|
|
|
@@ -103,9 +127,9 @@ module JSON
|
|
|
103
127
|
|
|
104
128
|
def dump(obj, an_io = nil, limit = nil, kwargs = nil)
|
|
105
129
|
# Fast path for the common shapes, dump(obj) and dump(obj, opts
|
|
106
|
-
# hash), mirroring
|
|
130
|
+
# hash), mirroring gem json: dump defaults merged under the
|
|
107
131
|
# user's options, NestingError surfaced as ArgumentError. IO and
|
|
108
|
-
# limit arguments take
|
|
132
|
+
# limit arguments take gem json's own dump.
|
|
109
133
|
if limit.nil? && kwargs.nil? && (an_io.nil? || an_io.instance_of?(Hash))
|
|
110
134
|
opts = _dump_default_options
|
|
111
135
|
opts = opts.merge(an_io) if an_io
|