fast_jsonparser 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ruby.yml +35 -0
- data/CHANGELOG.md +3 -0
- data/Gemfile.lock +1 -1
- data/README.md +9 -0
- data/ext/fast_jsonparser/fast_jsonparser.cpp +44 -11
- data/ext/fast_jsonparser/simdjson.h +4 -4
- data/lib/fast_jsonparser.rb +15 -5
- data/lib/fast_jsonparser/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a696c8dcde3ac2ec599ab0d45bc3ef0f13da2823cbddacd13842337e50cd6200
|
4
|
+
data.tar.gz: 8be657884315230f3010b4713c9646150f95db5468f3fbc12124406df51b69a4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 744771f5df6f8259ad6e75d631f8af6ba59ab6d00d94df78abbc0341d995045ba6bf5a32cffc1f476a9b48a5bce45d803c8f91c13c1fe840a2c6f931c8e549be
|
7
|
+
data.tar.gz: d14e73c088cd61d073a03d787075ea461750d7df3e1628c28398bc4f51ada3da4e6c4d036af2b9f81f16ede1ca28948fcd0fe49966b4756ee9c9f72ab63af81c
|
@@ -0,0 +1,35 @@
|
|
1
|
+
# This workflow uses actions that are not certified by GitHub.
|
2
|
+
# They are provided by a third-party and are governed by
|
3
|
+
# separate terms of service, privacy policy, and support
|
4
|
+
# documentation.
|
5
|
+
# This workflow will download a prebuilt Ruby version, install dependencies and run tests with Rake
|
6
|
+
# For more information see: https://github.com/marketplace/actions/setup-ruby-jruby-and-truffleruby
|
7
|
+
|
8
|
+
name: Ruby
|
9
|
+
|
10
|
+
on:
|
11
|
+
push:
|
12
|
+
branches: [ master ]
|
13
|
+
pull_request:
|
14
|
+
branches: [ master ]
|
15
|
+
|
16
|
+
jobs:
|
17
|
+
test:
|
18
|
+
|
19
|
+
runs-on: ubuntu-latest
|
20
|
+
|
21
|
+
steps:
|
22
|
+
- uses: actions/checkout@v2
|
23
|
+
- name: Set up Ruby
|
24
|
+
# To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
|
25
|
+
# change this to (see https://github.com/ruby/setup-ruby#versioning):
|
26
|
+
# uses: ruby/setup-ruby@v1
|
27
|
+
uses: ruby/setup-ruby@ec106b438a1ff6ff109590de34ddc62c540232e0
|
28
|
+
with:
|
29
|
+
ruby-version: 2.6
|
30
|
+
- name: Install dependencies
|
31
|
+
run: bundle install
|
32
|
+
- name: Compile
|
33
|
+
run: bundle exec rake compile
|
34
|
+
- name: Run tests
|
35
|
+
run: bundle exec rake
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
# 0.5.0
|
2
|
+
* Handle concurrent use of the parser in [Issue #15](https://github.com/anilmaurya/fast_jsonparser/pull/15), thanks to [casperisfine](https://github.com/casperisfine)
|
3
|
+
|
1
4
|
# 0.4.0
|
2
5
|
* load_many accept batch_size parameter to parse documents larger than 1 MB in [PR #5](https://github.com/anilmaurya/fast_jsonparser/pull/5), thanks to [casperisfine](https://github.com/casperisfine)
|
3
6
|
* Add option for symbolize_keys, default to true in [PR #9](https://github.com/anilmaurya/fast_jsonparser/pull/9), thanks to [casperisfine](https://github.com/casperisfine)
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -128,6 +128,15 @@ FastJsonparser.parse('{"one": 1, "two": 2}', symbolize_keys: false)
|
|
128
128
|
FastJsonparser.parse("123: 1") # FastJsonparser::ParseError (parse error)
|
129
129
|
```
|
130
130
|
|
131
|
+
### Known Incompatibilities with stdlib JSON
|
132
|
+
|
133
|
+
`FastJsonparser` behaves mostly like stdlib's `JSON`, but there are a few corner cases:
|
134
|
+
|
135
|
+
- `FastJsonparser` will use symbols for hash keys by default. You can pass `symbolize_names: false` to have strings instead like `JSON`.
|
136
|
+
- `FastJsonparser` will raise on integers outside of the 64bits range (`-9223372036854775808..18446744073709551615`), `JSON` will parse them fine.
|
137
|
+
- `FastJsonparser` will raise on invalid string escapings (`"\x15"`), `JSON` will often handle some of them.
|
138
|
+
- `FastJsonparser` will raise on `/**/` comments. `JSON` will sometimes ignore them, sometimes raise.
|
139
|
+
|
131
140
|
### Example
|
132
141
|
|
133
142
|
```
|
@@ -6,6 +6,38 @@ VALUE rb_eFastJsonparserUnknownError, rb_eFastJsonparserParseError;
|
|
6
6
|
|
7
7
|
using namespace simdjson;
|
8
8
|
|
9
|
+
typedef struct {
|
10
|
+
dom::parser *parser;
|
11
|
+
} parser_t;
|
12
|
+
|
13
|
+
static void Parser_delete(void *ptr) {
|
14
|
+
parser_t *data = (parser_t*) ptr;
|
15
|
+
delete data->parser;
|
16
|
+
}
|
17
|
+
|
18
|
+
static size_t Parser_memsize(const void *parser) {
|
19
|
+
return sizeof(dom::parser); // TODO: low priority, figure the real size, e.g. internal buffers etc.
|
20
|
+
}
|
21
|
+
|
22
|
+
static const rb_data_type_t parser_data_type = {
|
23
|
+
"Parser",
|
24
|
+
{ 0, Parser_delete, Parser_memsize, },
|
25
|
+
0, 0, RUBY_TYPED_FREE_IMMEDIATELY
|
26
|
+
};
|
27
|
+
|
28
|
+
static VALUE parser_allocate(VALUE klass) {
|
29
|
+
parser_t *data;
|
30
|
+
VALUE obj = TypedData_Make_Struct(klass, parser_t, &parser_data_type, data);
|
31
|
+
data->parser = new dom::parser;
|
32
|
+
return obj;
|
33
|
+
}
|
34
|
+
|
35
|
+
static inline dom::parser * get_parser(VALUE self) {
|
36
|
+
parser_t *data;
|
37
|
+
TypedData_Get_Struct(self, parser_t, &parser_data_type, data);
|
38
|
+
return data->parser;
|
39
|
+
}
|
40
|
+
|
9
41
|
// Convert tape to Ruby's Object
|
10
42
|
static VALUE make_ruby_object(dom::element element, bool symbolize_keys)
|
11
43
|
{
|
@@ -70,9 +102,9 @@ static VALUE make_ruby_object(dom::element element, bool symbolize_keys)
|
|
70
102
|
static VALUE rb_fast_jsonparser_parse(VALUE self, VALUE arg, VALUE symbolize_keys)
|
71
103
|
{
|
72
104
|
Check_Type(arg, T_STRING);
|
105
|
+
dom::parser *parser = get_parser(self);
|
73
106
|
|
74
|
-
|
75
|
-
auto [doc, error] = parser.parse(RSTRING_PTR(arg), RSTRING_LEN(arg));
|
107
|
+
auto [doc, error] = parser->parse(RSTRING_PTR(arg), RSTRING_LEN(arg));
|
76
108
|
if (error != SUCCESS)
|
77
109
|
{
|
78
110
|
rb_raise(rb_eFastJsonparserParseError, "%s", error_message(error));
|
@@ -83,9 +115,9 @@ static VALUE rb_fast_jsonparser_parse(VALUE self, VALUE arg, VALUE symbolize_key
|
|
83
115
|
static VALUE rb_fast_jsonparser_load(VALUE self, VALUE arg, VALUE symbolize_keys)
|
84
116
|
{
|
85
117
|
Check_Type(arg, T_STRING);
|
118
|
+
dom::parser *parser = get_parser(self);
|
86
119
|
|
87
|
-
|
88
|
-
auto [doc, error] = parser.load(RSTRING_PTR(arg));
|
120
|
+
auto [doc, error] = parser->load(RSTRING_PTR(arg));
|
89
121
|
if (error != SUCCESS)
|
90
122
|
{
|
91
123
|
rb_raise(rb_eFastJsonparserParseError, "%s", error_message(error));
|
@@ -97,11 +129,10 @@ static VALUE rb_fast_jsonparser_load_many(VALUE self, VALUE arg, VALUE symbolize
|
|
97
129
|
{
|
98
130
|
Check_Type(arg, T_STRING);
|
99
131
|
Check_Type(batch_size, T_FIXNUM);
|
132
|
+
dom::parser *parser = get_parser(self);
|
100
133
|
|
101
|
-
try
|
102
|
-
|
103
|
-
dom::parser parser;
|
104
|
-
auto [docs, error] = parser.load_many(RSTRING_PTR(arg), FIX2INT(batch_size));
|
134
|
+
try {
|
135
|
+
auto [docs, error] = parser->load_many(RSTRING_PTR(arg), FIX2INT(batch_size));
|
105
136
|
if (error != SUCCESS)
|
106
137
|
{
|
107
138
|
rb_raise(rb_eFastJsonparserParseError, "%s", error_message(error));
|
@@ -126,10 +157,12 @@ extern "C"
|
|
126
157
|
void Init_fast_jsonparser(void)
|
127
158
|
{
|
128
159
|
VALUE rb_mFastJsonparser = rb_const_get(rb_cObject, rb_intern("FastJsonparser"));
|
160
|
+
VALUE rb_cFastJsonparserNative = rb_const_get(rb_mFastJsonparser, rb_intern("Native"));
|
129
161
|
|
130
|
-
|
131
|
-
|
132
|
-
|
162
|
+
rb_define_alloc_func(rb_cFastJsonparserNative, parser_allocate);
|
163
|
+
rb_define_method(rb_cFastJsonparserNative, "_parse", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_parse), 2);
|
164
|
+
rb_define_method(rb_cFastJsonparserNative, "_load", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_load), 2);
|
165
|
+
rb_define_method(rb_cFastJsonparserNative, "_load_many", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_load_many), 3);
|
133
166
|
|
134
167
|
rb_eFastJsonparserParseError = rb_const_get(rb_mFastJsonparser, rb_intern("ParseError"));
|
135
168
|
rb_global_variable(&rb_eFastJsonparserParseError);
|
@@ -2308,7 +2308,7 @@ using ErrorValues [[deprecated("This is an alias and will be removed, use error_
|
|
2308
2308
|
* @deprecated Error codes should be stored and returned as `error_code`, use `error_message()` instead.
|
2309
2309
|
*/
|
2310
2310
|
[[deprecated("Error codes should be stored and returned as `error_code`, use `error_message()` instead.")]]
|
2311
|
-
inline const std::string
|
2311
|
+
inline const std::string error_message(int error) noexcept;
|
2312
2312
|
|
2313
2313
|
} // namespace simdjson
|
2314
2314
|
|
@@ -6367,7 +6367,7 @@ namespace internal {
|
|
6367
6367
|
// We store the error code so we can validate the error message is associated with the right code
|
6368
6368
|
struct error_code_info {
|
6369
6369
|
error_code code;
|
6370
|
-
|
6370
|
+
const char* message;
|
6371
6371
|
};
|
6372
6372
|
// These MUST match the codes in error_code. We check this constraint in basictests.
|
6373
6373
|
extern SIMDJSON_DLLIMPORTEXPORT const error_code_info error_codes[];
|
@@ -6376,10 +6376,10 @@ namespace internal {
|
|
6376
6376
|
|
6377
6377
|
inline const char *error_message(error_code error) noexcept {
|
6378
6378
|
// If you're using error_code, we're trusting you got it from the enum.
|
6379
|
-
return internal::error_codes[int(error)].message
|
6379
|
+
return internal::error_codes[int(error)].message;
|
6380
6380
|
}
|
6381
6381
|
|
6382
|
-
inline const std::string
|
6382
|
+
inline const std::string error_message(int error) noexcept {
|
6383
6383
|
if (error < 0 || error >= error_code::NUM_ERROR_CODES) {
|
6384
6384
|
return internal::error_codes[UNEXPECTED_ERROR].message;
|
6385
6385
|
}
|
data/lib/fast_jsonparser.rb
CHANGED
@@ -12,15 +12,15 @@ module FastJsonparser
|
|
12
12
|
|
13
13
|
class << self
|
14
14
|
def parse(source, symbolize_keys: true)
|
15
|
-
_parse(source, symbolize_keys)
|
15
|
+
parser._parse(source, symbolize_keys)
|
16
16
|
end
|
17
17
|
|
18
18
|
def load(source, symbolize_keys: true)
|
19
|
-
_load(source, symbolize_keys)
|
19
|
+
parser._load(source, symbolize_keys)
|
20
20
|
end
|
21
21
|
|
22
22
|
def load_many(source, symbolize_keys: true, batch_size: DEFAULT_BATCH_SIZE, &block)
|
23
|
-
_load_many(source, symbolize_keys, batch_size, &block)
|
23
|
+
Native.new._load_many(source, symbolize_keys, batch_size, &block)
|
24
24
|
rescue UnknownError => error
|
25
25
|
case error.message
|
26
26
|
when "This parser can't support a document that big"
|
@@ -30,7 +30,17 @@ module FastJsonparser
|
|
30
30
|
end
|
31
31
|
end
|
32
32
|
|
33
|
-
|
34
|
-
|
33
|
+
private
|
34
|
+
|
35
|
+
def parser
|
36
|
+
@parser ||= Native.new
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
class Native
|
35
41
|
end
|
42
|
+
|
43
|
+
require "fast_jsonparser/fast_jsonparser" # loads cpp extension
|
44
|
+
|
45
|
+
private_constant :Native
|
36
46
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fast_jsonparser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Anil Maurya
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-08-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -102,6 +102,7 @@ extensions:
|
|
102
102
|
- ext/fast_jsonparser/extconf.rb
|
103
103
|
extra_rdoc_files: []
|
104
104
|
files:
|
105
|
+
- ".github/workflows/ruby.yml"
|
105
106
|
- ".gitignore"
|
106
107
|
- ".travis.yml"
|
107
108
|
- CHANGELOG.md
|