oj 3.11.5 → 3.16.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1421 -0
- data/README.md +19 -5
- data/RELEASE_NOTES.md +61 -0
- data/ext/oj/buf.h +20 -6
- data/ext/oj/cache.c +329 -0
- data/ext/oj/cache.h +22 -0
- data/ext/oj/cache8.c +10 -9
- data/ext/oj/circarray.c +8 -6
- data/ext/oj/circarray.h +2 -2
- data/ext/oj/code.c +19 -33
- data/ext/oj/code.h +2 -2
- data/ext/oj/compat.c +27 -77
- data/ext/oj/custom.c +86 -179
- data/ext/oj/debug.c +126 -0
- data/ext/oj/dump.c +256 -249
- data/ext/oj/dump.h +26 -12
- data/ext/oj/dump_compat.c +565 -642
- data/ext/oj/dump_leaf.c +17 -63
- data/ext/oj/dump_object.c +65 -187
- data/ext/oj/dump_strict.c +27 -51
- data/ext/oj/encoder.c +43 -0
- data/ext/oj/err.c +2 -13
- data/ext/oj/err.h +24 -8
- data/ext/oj/extconf.rb +21 -6
- data/ext/oj/fast.c +149 -149
- data/ext/oj/intern.c +313 -0
- data/ext/oj/intern.h +22 -0
- data/ext/oj/mem.c +318 -0
- data/ext/oj/mem.h +53 -0
- data/ext/oj/mimic_json.c +121 -106
- data/ext/oj/object.c +85 -162
- data/ext/oj/odd.c +89 -67
- data/ext/oj/odd.h +15 -15
- data/ext/oj/oj.c +542 -411
- data/ext/oj/oj.h +99 -73
- data/ext/oj/parse.c +175 -187
- data/ext/oj/parse.h +26 -24
- data/ext/oj/parser.c +1600 -0
- data/ext/oj/parser.h +101 -0
- data/ext/oj/rails.c +112 -159
- data/ext/oj/rails.h +1 -1
- data/ext/oj/reader.c +11 -14
- data/ext/oj/reader.h +4 -2
- data/ext/oj/resolve.c +5 -24
- data/ext/oj/rxclass.c +7 -6
- data/ext/oj/rxclass.h +1 -1
- data/ext/oj/saj.c +22 -33
- data/ext/oj/saj2.c +584 -0
- data/ext/oj/saj2.h +23 -0
- data/ext/oj/scp.c +5 -28
- data/ext/oj/sparse.c +28 -72
- data/ext/oj/stream_writer.c +50 -40
- data/ext/oj/strict.c +56 -61
- data/ext/oj/string_writer.c +72 -39
- data/ext/oj/trace.h +31 -4
- data/ext/oj/usual.c +1218 -0
- data/ext/oj/usual.h +69 -0
- data/ext/oj/util.h +1 -1
- data/ext/oj/val_stack.c +14 -3
- data/ext/oj/val_stack.h +8 -7
- data/ext/oj/validate.c +46 -0
- data/ext/oj/wab.c +63 -88
- data/lib/oj/active_support_helper.rb +1 -3
- data/lib/oj/bag.rb +7 -1
- data/lib/oj/easy_hash.rb +4 -5
- data/lib/oj/error.rb +1 -2
- data/lib/oj/json.rb +162 -150
- data/lib/oj/mimic.rb +9 -7
- data/lib/oj/saj.rb +20 -6
- data/lib/oj/schandler.rb +5 -4
- data/lib/oj/state.rb +12 -8
- data/lib/oj/version.rb +1 -2
- data/lib/oj.rb +2 -0
- data/pages/Compatibility.md +1 -1
- data/pages/InstallOptions.md +20 -0
- data/pages/JsonGem.md +15 -0
- data/pages/Modes.md +8 -3
- data/pages/Options.md +43 -5
- data/pages/Parser.md +309 -0
- data/pages/Rails.md +14 -2
- data/test/_test_active.rb +8 -9
- data/test/_test_active_mimic.rb +7 -8
- data/test/_test_mimic_rails.rb +17 -20
- data/test/activerecord/result_test.rb +5 -6
- data/test/activesupport6/encoding_test.rb +63 -28
- data/test/{activesupport5 → activesupport7}/abstract_unit.rb +16 -12
- data/test/{activesupport5 → activesupport7}/decoding_test.rb +2 -10
- data/test/{activesupport5 → activesupport7}/encoding_test.rb +86 -50
- data/test/{activesupport5 → activesupport7}/encoding_test_cases.rb +6 -0
- data/test/{activesupport5 → activesupport7}/time_zone_test_helpers.rb +8 -0
- data/test/files.rb +15 -15
- data/test/foo.rb +16 -45
- data/test/helper.rb +11 -8
- data/test/isolated/shared.rb +3 -2
- data/test/json_gem/json_addition_test.rb +2 -2
- data/test/json_gem/json_common_interface_test.rb +8 -6
- data/test/json_gem/json_encoding_test.rb +0 -0
- data/test/json_gem/json_ext_parser_test.rb +1 -0
- data/test/json_gem/json_fixtures_test.rb +3 -2
- data/test/json_gem/json_generator_test.rb +56 -38
- data/test/json_gem/json_generic_object_test.rb +11 -11
- data/test/json_gem/json_parser_test.rb +54 -47
- data/test/json_gem/json_string_matching_test.rb +9 -9
- data/test/json_gem/test_helper.rb +7 -3
- data/test/mem.rb +34 -0
- data/test/perf.rb +22 -27
- data/test/perf_compat.rb +31 -33
- data/test/perf_dump.rb +50 -0
- data/test/perf_fast.rb +80 -82
- data/test/perf_file.rb +27 -29
- data/test/perf_object.rb +65 -69
- data/test/perf_once.rb +59 -0
- data/test/perf_parser.rb +183 -0
- data/test/perf_saj.rb +46 -54
- data/test/perf_scp.rb +58 -69
- data/test/perf_simple.rb +41 -39
- data/test/perf_strict.rb +74 -82
- data/test/perf_wab.rb +67 -69
- data/test/prec.rb +5 -5
- data/test/sample/change.rb +0 -1
- data/test/sample/dir.rb +0 -1
- data/test/sample/doc.rb +0 -1
- data/test/sample/file.rb +0 -1
- data/test/sample/group.rb +0 -1
- data/test/sample/hasprops.rb +0 -1
- data/test/sample/layer.rb +0 -1
- data/test/sample/rect.rb +0 -1
- data/test/sample/shape.rb +0 -1
- data/test/sample/text.rb +0 -1
- data/test/sample.rb +16 -16
- data/test/sample_json.rb +8 -8
- data/test/test_compat.rb +95 -43
- data/test/test_custom.rb +73 -51
- data/test/test_debian.rb +7 -10
- data/test/test_fast.rb +135 -79
- data/test/test_file.rb +41 -30
- data/test/test_gc.rb +16 -5
- data/test/test_generate.rb +5 -5
- data/test/test_hash.rb +5 -5
- data/test/test_integer_range.rb +9 -9
- data/test/test_null.rb +20 -20
- data/test/test_object.rb +99 -96
- data/test/test_parser.rb +11 -0
- data/test/test_parser_debug.rb +27 -0
- data/test/test_parser_saj.rb +337 -0
- data/test/test_parser_usual.rb +251 -0
- data/test/test_rails.rb +2 -2
- data/test/test_saj.rb +10 -8
- data/test/test_scp.rb +37 -39
- data/test/test_strict.rb +40 -32
- data/test/test_various.rb +165 -84
- data/test/test_wab.rb +48 -44
- data/test/test_writer.rb +47 -47
- data/test/tests.rb +13 -5
- data/test/tests_mimic.rb +12 -3
- data/test/tests_mimic_addition.rb +12 -3
- metadata +74 -128
- data/ext/oj/hash.c +0 -131
- data/ext/oj/hash.h +0 -19
- data/ext/oj/hash_test.c +0 -491
- data/test/activesupport4/decoding_test.rb +0 -108
- data/test/activesupport4/encoding_test.rb +0 -531
- data/test/activesupport4/test_helper.rb +0 -41
- data/test/activesupport5/test_helper.rb +0 -72
- data/test/bar.rb +0 -35
- data/test/baz.rb +0 -16
- data/test/zoo.rb +0 -13
data/README.md
CHANGED
@@ -1,11 +1,13 @@
|
|
1
1
|
# [![{}j](http://www.ohler.com/dev/images/oj_comet_64.svg)](http://www.ohler.com/oj) gem
|
2
2
|
|
3
|
-
[![
|
3
|
+
[![CI](https://github.com/ohler55/oj/actions/workflows/CI.yml/badge.svg)](https://github.com/ohler55/oj/actions/workflows/CI.yml)
|
4
|
+
![Gem](https://img.shields.io/gem/v/oj.svg)
|
5
|
+
![Gem](https://img.shields.io/gem/dt/oj.svg)
|
6
|
+
[![TideLift](https://tidelift.com/badges/github/ohler55/oj)](https://tidelift.com/subscription/pkg/rubygems-oj?utm_source=rubygems-oj&utm_medium=referral&utm_campaign=readme)
|
4
7
|
|
5
8
|
A *fast* JSON parser and Object marshaller as a Ruby gem.
|
6
9
|
|
7
|
-
Version 3.
|
8
|
-
also provides additional optimization options.
|
10
|
+
Version 3.13 is out with a much faster parser (`Oj::Parser`) and option isolation.
|
9
11
|
|
10
12
|
## Using
|
11
13
|
|
@@ -40,6 +42,15 @@ or in Bundler:
|
|
40
42
|
gem 'oj'
|
41
43
|
```
|
42
44
|
|
45
|
+
## Rails and json quickstart
|
46
|
+
|
47
|
+
See the Quickstart sections of the [Rails](pages/Rails.md) and [json](pages/JsonGem.md) docs.
|
48
|
+
|
49
|
+
## multi_json
|
50
|
+
|
51
|
+
Code which uses [multi_json](https://github.com/intridea/multi_json)
|
52
|
+
will automatically prefer Oj if it is installed.
|
53
|
+
|
43
54
|
## Support
|
44
55
|
|
45
56
|
[Get supported Oj with a Tidelift Subscription.](https://tidelift.com/subscription/pkg/rubygems-oj?utm_source=rubygems-oj&utm_medium=referral&utm_campaign=readme) Security updates are [supported](https://tidelift.com/security).
|
@@ -50,7 +61,7 @@ For more details on options, modes, advanced features, and more follow these
|
|
50
61
|
links.
|
51
62
|
|
52
63
|
- [{file:Options.md}](pages/Options.md) for parse and dump options.
|
53
|
-
- [{file:Modes.md}](pages/Modes.md) for details on modes for strict JSON compliance,
|
64
|
+
- [{file:Modes.md}](pages/Modes.md) for details on modes for strict JSON compliance, mimicking the JSON gem, and mimicking Rails and ActiveSupport behavior.
|
54
65
|
- [{file:JsonGem.md}](pages/JsonGem.md) includes more details on json gem compatibility and use.
|
55
66
|
- [{file:Rails.md}](pages/Rails.md) includes more details on Rails and ActiveSupport compatibility and use.
|
56
67
|
- [{file:Custom.md}](pages/Custom.md) includes more details on Custom mode.
|
@@ -58,10 +69,11 @@ links.
|
|
58
69
|
- [{file:Compatibility.md}](pages/Compatibility.md) lists current compatibility with Rubys and Rails.
|
59
70
|
- [{file:Advanced.md}](pages/Advanced.md) for fast parser and marshalling features.
|
60
71
|
- [{file:Security.md}](pages/Security.md) for security considerations.
|
72
|
+
- [{file:InstallOptions.md}](pages/InstallOptions.md) for install option.
|
61
73
|
|
62
74
|
## Releases
|
63
75
|
|
64
|
-
See [{file:CHANGELOG.md}](CHANGELOG.md)
|
76
|
+
See [{file:CHANGELOG.md}](CHANGELOG.md) and [{file:RELEASE_NOTES.md}](RELEASE_NOTES.md)
|
65
77
|
|
66
78
|
## Links
|
67
79
|
|
@@ -97,6 +109,8 @@ Follow [@peterohler on Twitter](http://twitter.com/peterohler) for announcements
|
|
97
109
|
|
98
110
|
- *Agoo-C, a high performance C web server supporting GraphQL on GitHub*: https://github.com/ohler55/agoo-c
|
99
111
|
|
112
|
+
- *oj-introspect, an example of creating an Oj parser extension in C*: https://github.com/meinac/oj-introspect
|
113
|
+
|
100
114
|
#### Contributing
|
101
115
|
|
102
116
|
+ Provide a Pull Request off the `develop` branch.
|
data/RELEASE_NOTES.md
ADDED
@@ -0,0 +1,61 @@
|
|
1
|
+
# RELEASE NOTES
|
2
|
+
|
3
|
+
The release notes here are organized by release. For a list of changes
|
4
|
+
see the See [{file:CHANGELOG.md}](CHANGELOG.md) file. In this file are
|
5
|
+
the steps to take to aid in keeping things rolling after updating to
|
6
|
+
the latest version.
|
7
|
+
|
8
|
+
## 3.13.7
|
9
|
+
|
10
|
+
The default for JSON when mimicked by Oj is now to set
|
11
|
+
`:allow_invalid_unicode`. To change that behavior JSON.load, set that
|
12
|
+
option to false.
|
13
|
+
|
14
|
+
## 3.13.x
|
15
|
+
|
16
|
+
This release included a new cache that performs better than the
|
17
|
+
earlier cache and a new high performance parser.
|
18
|
+
|
19
|
+
### Cache
|
20
|
+
|
21
|
+
The new cache includes a least recently used expiration to reduce
|
22
|
+
memory use. The cache is also self adjusting and will expand as needed
|
23
|
+
for better performance. It also handles Hash keys and string values
|
24
|
+
with two options, `:cache_keys`, a boolean and `:cache_str` an
|
25
|
+
integer. The `:cache_str` if set to more than zero is the limit for
|
26
|
+
the length of string values to cache. The maximum value is 35 which
|
27
|
+
allows strings up to 34 bytes to be cached.
|
28
|
+
|
29
|
+
One interesting aspect of the cache is not so much the string caching
|
30
|
+
which performs similar to the Ruby intern functions but the caching of
|
31
|
+
symbols and object attribute names. There is a significant gain for
|
32
|
+
symbols and object attributes.
|
33
|
+
|
34
|
+
If the cache is not desired then setting the default options to turn
|
35
|
+
it off can be done with this line:
|
36
|
+
|
37
|
+
``` ruby
|
38
|
+
Oj.default_options = { cache_keys: false, cache_str: 0 }
|
39
|
+
```
|
40
|
+
|
41
|
+
### Oj::Parser
|
42
|
+
|
43
|
+
The new parser uses a different core that follows the approach taken
|
44
|
+
by [OjC](https://github.com/ohler55/ojc) and
|
45
|
+
[OjG](https://github.com/ohler55/ojg). It also takes advantage of the
|
46
|
+
bulk Array and Hash functions. Another issue the new parser addresses
|
47
|
+
is option management. Instead of a single global default_options each
|
48
|
+
parser instance maintains it's own options.
|
49
|
+
|
50
|
+
There is a price to be paid when using the Oj::Parser. The API is not
|
51
|
+
the same the older parser. A single parser can only be used in a
|
52
|
+
single thread. This allows reuse of internal buffers for additional
|
53
|
+
improvements in performance.
|
54
|
+
|
55
|
+
The performane advantage of the Oj::Parse is that it is more than 3
|
56
|
+
times faster than the Oj::compat_load call and 6 times faster than the
|
57
|
+
JSON gem.
|
58
|
+
|
59
|
+
### Dump Performance
|
60
|
+
|
61
|
+
Thanks to Watson1978 Oj.dump also received a speed boost.
|
data/ext/oj/buf.h
CHANGED
@@ -4,6 +4,7 @@
|
|
4
4
|
#ifndef OJ_BUF_H
|
5
5
|
#define OJ_BUF_H
|
6
6
|
|
7
|
+
#include "mem.h"
|
7
8
|
#include "ruby.h"
|
8
9
|
|
9
10
|
typedef struct _buf {
|
@@ -11,7 +12,7 @@ typedef struct _buf {
|
|
11
12
|
char *end;
|
12
13
|
char *tail;
|
13
14
|
char base[1024];
|
14
|
-
} *
|
15
|
+
} *Buf;
|
15
16
|
|
16
17
|
inline static void buf_init(Buf buf) {
|
17
18
|
buf->head = buf->base;
|
@@ -19,9 +20,13 @@ inline static void buf_init(Buf buf) {
|
|
19
20
|
buf->tail = buf->head;
|
20
21
|
}
|
21
22
|
|
23
|
+
inline static void buf_reset(Buf buf) {
|
24
|
+
buf->tail = buf->head;
|
25
|
+
}
|
26
|
+
|
22
27
|
inline static void buf_cleanup(Buf buf) {
|
23
28
|
if (buf->base != buf->head) {
|
24
|
-
|
29
|
+
OJ_R_FREE(buf->head);
|
25
30
|
}
|
26
31
|
}
|
27
32
|
|
@@ -29,17 +34,26 @@ inline static size_t buf_len(Buf buf) {
|
|
29
34
|
return buf->tail - buf->head;
|
30
35
|
}
|
31
36
|
|
37
|
+
inline static const char *buf_str(Buf buf) {
|
38
|
+
*buf->tail = '\0';
|
39
|
+
return buf->head;
|
40
|
+
}
|
41
|
+
|
32
42
|
inline static void buf_append_string(Buf buf, const char *s, size_t slen) {
|
43
|
+
if (0 == slen) {
|
44
|
+
return;
|
45
|
+
}
|
46
|
+
|
33
47
|
if (buf->end <= buf->tail + slen) {
|
34
48
|
size_t len = buf->end - buf->head;
|
35
49
|
size_t toff = buf->tail - buf->head;
|
36
50
|
size_t new_len = len + slen + len / 2;
|
37
51
|
|
38
52
|
if (buf->base == buf->head) {
|
39
|
-
buf->head =
|
53
|
+
buf->head = OJ_R_ALLOC_N(char, new_len);
|
40
54
|
memcpy(buf->head, buf->base, len);
|
41
55
|
} else {
|
42
|
-
|
56
|
+
OJ_R_REALLOC_N(buf->head, char, new_len);
|
43
57
|
}
|
44
58
|
buf->tail = buf->head + toff;
|
45
59
|
buf->end = buf->head + new_len - 1;
|
@@ -55,10 +69,10 @@ inline static void buf_append(Buf buf, char c) {
|
|
55
69
|
size_t new_len = len + len / 2;
|
56
70
|
|
57
71
|
if (buf->base == buf->head) {
|
58
|
-
buf->head =
|
72
|
+
buf->head = OJ_R_ALLOC_N(char, new_len);
|
59
73
|
memcpy(buf->head, buf->base, len);
|
60
74
|
} else {
|
61
|
-
|
75
|
+
OJ_R_REALLOC_N(buf->head, char, new_len);
|
62
76
|
}
|
63
77
|
buf->tail = buf->head + toff;
|
64
78
|
buf->end = buf->head + new_len - 1;
|
data/ext/oj/cache.c
ADDED
@@ -0,0 +1,329 @@
|
|
1
|
+
// Copyright (c) 2011, 2021 Peter Ohler. All rights reserved.
|
2
|
+
// Licensed under the MIT License. See LICENSE file in the project root for license details.
|
3
|
+
|
4
|
+
#if HAVE_PTHREAD_MUTEX_INIT
|
5
|
+
#include <pthread.h>
|
6
|
+
#endif
|
7
|
+
#include <stdlib.h>
|
8
|
+
|
9
|
+
#include "cache.h"
|
10
|
+
#include "mem.h"
|
11
|
+
|
12
|
+
// The stdlib calloc, realloc, and free are used instead of the Ruby ALLOC,
|
13
|
+
// ALLOC_N, REALLOC, and xfree since the later could trigger a GC which will
|
14
|
+
// either corrupt memory or if the mark function locks will deadlock.
|
15
|
+
|
16
|
+
#define REHASH_LIMIT 4
|
17
|
+
#define MIN_SHIFT 8
|
18
|
+
#define REUSE_MAX 8192
|
19
|
+
|
20
|
+
#if HAVE_PTHREAD_MUTEX_INIT
|
21
|
+
#define CACHE_LOCK(c) pthread_mutex_lock(&((c)->mutex))
|
22
|
+
#define CACHE_UNLOCK(c) pthread_mutex_unlock(&((c)->mutex))
|
23
|
+
#else
|
24
|
+
#define CACHE_LOCK(c) rb_mutex_lock((c)->mutex)
|
25
|
+
#define CACHE_UNLOCK(c) rb_mutex_unlock((c)->mutex)
|
26
|
+
#endif
|
27
|
+
|
28
|
+
// almost the Murmur hash algorithm
|
29
|
+
#define M 0x5bd1e995
|
30
|
+
|
31
|
+
typedef struct _slot {
|
32
|
+
struct _slot *next;
|
33
|
+
VALUE val;
|
34
|
+
uint64_t hash;
|
35
|
+
volatile uint32_t use_cnt;
|
36
|
+
uint8_t klen;
|
37
|
+
char key[CACHE_MAX_KEY];
|
38
|
+
} *Slot;
|
39
|
+
|
40
|
+
typedef struct _cache {
|
41
|
+
volatile Slot *slots;
|
42
|
+
volatile size_t cnt;
|
43
|
+
VALUE (*form)(const char *str, size_t len);
|
44
|
+
uint64_t size;
|
45
|
+
uint64_t mask;
|
46
|
+
VALUE (*intern)(struct _cache *c, const char *key, size_t len);
|
47
|
+
volatile Slot reuse;
|
48
|
+
size_t rcnt;
|
49
|
+
#if HAVE_PTHREAD_MUTEX_INIT
|
50
|
+
pthread_mutex_t mutex;
|
51
|
+
#else
|
52
|
+
VALUE mutex;
|
53
|
+
#endif
|
54
|
+
uint8_t xrate;
|
55
|
+
bool mark;
|
56
|
+
} *Cache;
|
57
|
+
|
58
|
+
void cache_set_form(Cache c, VALUE (*form)(const char *str, size_t len)) {
|
59
|
+
c->form = form;
|
60
|
+
}
|
61
|
+
|
62
|
+
static uint64_t hash_calc(const uint8_t *key, size_t len) {
|
63
|
+
const uint8_t *end = key + len;
|
64
|
+
const uint8_t *endless = key + (len & 0xFFFFFFFC);
|
65
|
+
uint64_t h = (uint64_t)len;
|
66
|
+
uint64_t k;
|
67
|
+
|
68
|
+
while (key < endless) {
|
69
|
+
k = (uint64_t)*key++;
|
70
|
+
k |= (uint64_t)*key++ << 8;
|
71
|
+
k |= (uint64_t)*key++ << 16;
|
72
|
+
k |= (uint64_t)*key++ << 24;
|
73
|
+
|
74
|
+
k *= M;
|
75
|
+
k ^= k >> 24;
|
76
|
+
h *= M;
|
77
|
+
h ^= k * M;
|
78
|
+
}
|
79
|
+
if (1 < end - key) {
|
80
|
+
uint16_t k16 = (uint16_t)*key++;
|
81
|
+
|
82
|
+
k16 |= (uint16_t)*key++ << 8;
|
83
|
+
h ^= k16 << 8;
|
84
|
+
}
|
85
|
+
if (key < end) {
|
86
|
+
h ^= *key;
|
87
|
+
}
|
88
|
+
h *= M;
|
89
|
+
h ^= h >> 13;
|
90
|
+
h *= M;
|
91
|
+
h ^= h >> 15;
|
92
|
+
|
93
|
+
return h;
|
94
|
+
}
|
95
|
+
|
96
|
+
static void rehash(Cache c) {
|
97
|
+
uint64_t osize;
|
98
|
+
Slot *end;
|
99
|
+
Slot *sp;
|
100
|
+
|
101
|
+
osize = c->size;
|
102
|
+
c->size = osize * 4;
|
103
|
+
c->mask = c->size - 1;
|
104
|
+
c->slots = OJ_REALLOC((void *)c->slots, sizeof(Slot) * c->size);
|
105
|
+
memset((Slot *)c->slots + osize, 0, sizeof(Slot) * osize * 3);
|
106
|
+
end = (Slot *)c->slots + osize;
|
107
|
+
for (sp = (Slot *)c->slots; sp < end; sp++) {
|
108
|
+
Slot s = *sp;
|
109
|
+
Slot next = NULL;
|
110
|
+
|
111
|
+
*sp = NULL;
|
112
|
+
for (; NULL != s; s = next) {
|
113
|
+
uint64_t h = s->hash & c->mask;
|
114
|
+
Slot *bucket = (Slot *)c->slots + h;
|
115
|
+
|
116
|
+
next = s->next;
|
117
|
+
s->next = *bucket;
|
118
|
+
*bucket = s;
|
119
|
+
}
|
120
|
+
}
|
121
|
+
}
|
122
|
+
|
123
|
+
static VALUE lockless_intern(Cache c, const char *key, size_t len) {
|
124
|
+
uint64_t h = hash_calc((const uint8_t *)key, len);
|
125
|
+
Slot *bucket = (Slot *)c->slots + (h & c->mask);
|
126
|
+
Slot b;
|
127
|
+
volatile VALUE rkey;
|
128
|
+
|
129
|
+
while (REUSE_MAX < c->rcnt) {
|
130
|
+
if (NULL != (b = c->reuse)) {
|
131
|
+
c->reuse = b->next;
|
132
|
+
OJ_FREE(b);
|
133
|
+
c->rcnt--;
|
134
|
+
} else {
|
135
|
+
// An accounting error occured somewhere so correct it.
|
136
|
+
c->rcnt = 0;
|
137
|
+
}
|
138
|
+
}
|
139
|
+
for (b = *bucket; NULL != b; b = b->next) {
|
140
|
+
if ((uint8_t)len == b->klen && 0 == strncmp(b->key, key, len)) {
|
141
|
+
b->use_cnt += 16;
|
142
|
+
return b->val;
|
143
|
+
}
|
144
|
+
}
|
145
|
+
rkey = c->form(key, len);
|
146
|
+
if (NULL == (b = c->reuse)) {
|
147
|
+
b = OJ_CALLOC(1, sizeof(struct _slot));
|
148
|
+
} else {
|
149
|
+
c->reuse = b->next;
|
150
|
+
c->rcnt--;
|
151
|
+
}
|
152
|
+
b->hash = h;
|
153
|
+
memcpy(b->key, key, len);
|
154
|
+
b->klen = (uint8_t)len;
|
155
|
+
b->key[len] = '\0';
|
156
|
+
b->val = rkey;
|
157
|
+
b->use_cnt = 4;
|
158
|
+
b->next = *bucket;
|
159
|
+
*bucket = b;
|
160
|
+
c->cnt++; // Don't worry about wrapping. Worse case is the entry is removed and recreated.
|
161
|
+
if (REHASH_LIMIT < c->cnt / c->size) {
|
162
|
+
rehash(c);
|
163
|
+
}
|
164
|
+
return rkey;
|
165
|
+
}
|
166
|
+
|
167
|
+
static VALUE locking_intern(Cache c, const char *key, size_t len) {
|
168
|
+
uint64_t h;
|
169
|
+
Slot *bucket;
|
170
|
+
Slot b;
|
171
|
+
uint64_t old_size;
|
172
|
+
volatile VALUE rkey;
|
173
|
+
|
174
|
+
CACHE_LOCK(c);
|
175
|
+
while (REUSE_MAX < c->rcnt) {
|
176
|
+
if (NULL != (b = c->reuse)) {
|
177
|
+
c->reuse = b->next;
|
178
|
+
OJ_FREE(b);
|
179
|
+
c->rcnt--;
|
180
|
+
} else {
|
181
|
+
// An accounting error occured somewhere so correct it.
|
182
|
+
c->rcnt = 0;
|
183
|
+
}
|
184
|
+
}
|
185
|
+
h = hash_calc((const uint8_t *)key, len);
|
186
|
+
bucket = (Slot *)c->slots + (h & c->mask);
|
187
|
+
for (b = *bucket; NULL != b; b = b->next) {
|
188
|
+
if ((uint8_t)len == b->klen && 0 == strncmp(b->key, key, len)) {
|
189
|
+
b->use_cnt += 4;
|
190
|
+
CACHE_UNLOCK(c);
|
191
|
+
|
192
|
+
return b->val;
|
193
|
+
}
|
194
|
+
}
|
195
|
+
old_size = c->size;
|
196
|
+
// The creation of a new value may trigger a GC which be a problem if the
|
197
|
+
// cache is locked so make sure it is unlocked for the key value creation.
|
198
|
+
if (NULL != (b = c->reuse)) {
|
199
|
+
c->reuse = b->next;
|
200
|
+
c->rcnt--;
|
201
|
+
}
|
202
|
+
CACHE_UNLOCK(c);
|
203
|
+
if (NULL == b) {
|
204
|
+
b = OJ_CALLOC(1, sizeof(struct _slot));
|
205
|
+
}
|
206
|
+
rkey = c->form(key, len);
|
207
|
+
b->hash = h;
|
208
|
+
memcpy(b->key, key, len);
|
209
|
+
b->klen = (uint8_t)len;
|
210
|
+
b->key[len] = '\0';
|
211
|
+
b->val = rkey;
|
212
|
+
b->use_cnt = 16;
|
213
|
+
|
214
|
+
// Lock again to add the new entry.
|
215
|
+
CACHE_LOCK(c);
|
216
|
+
if (old_size != c->size) {
|
217
|
+
h = hash_calc((const uint8_t *)key, len);
|
218
|
+
bucket = (Slot *)c->slots + (h & c->mask);
|
219
|
+
}
|
220
|
+
b->next = *bucket;
|
221
|
+
*bucket = b;
|
222
|
+
c->cnt++; // Don't worry about wrapping. Worse case is the entry is removed and recreated.
|
223
|
+
if (REHASH_LIMIT < c->cnt / c->size) {
|
224
|
+
rehash(c);
|
225
|
+
}
|
226
|
+
CACHE_UNLOCK(c);
|
227
|
+
|
228
|
+
return rkey;
|
229
|
+
}
|
230
|
+
|
231
|
+
Cache cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark, bool locking) {
|
232
|
+
Cache c = OJ_CALLOC(1, sizeof(struct _cache));
|
233
|
+
int shift = 0;
|
234
|
+
|
235
|
+
for (; REHASH_LIMIT < size; size /= 2, shift++) {
|
236
|
+
}
|
237
|
+
if (shift < MIN_SHIFT) {
|
238
|
+
shift = MIN_SHIFT;
|
239
|
+
}
|
240
|
+
#if HAVE_PTHREAD_MUTEX_INIT
|
241
|
+
pthread_mutex_init(&c->mutex, NULL);
|
242
|
+
#else
|
243
|
+
c->mutex = rb_mutex_new();
|
244
|
+
#endif
|
245
|
+
c->size = 1 << shift;
|
246
|
+
c->mask = c->size - 1;
|
247
|
+
c->slots = OJ_CALLOC(c->size, sizeof(Slot));
|
248
|
+
c->form = form;
|
249
|
+
c->xrate = 1; // low
|
250
|
+
c->mark = mark;
|
251
|
+
if (locking) {
|
252
|
+
c->intern = locking_intern;
|
253
|
+
} else {
|
254
|
+
c->intern = lockless_intern;
|
255
|
+
}
|
256
|
+
return c;
|
257
|
+
}
|
258
|
+
|
259
|
+
void cache_set_expunge_rate(Cache c, int rate) {
|
260
|
+
c->xrate = (uint8_t)rate;
|
261
|
+
}
|
262
|
+
|
263
|
+
void cache_free(void *data) {
|
264
|
+
Cache c = (Cache)data;
|
265
|
+
uint64_t i;
|
266
|
+
|
267
|
+
for (i = 0; i < c->size; i++) {
|
268
|
+
Slot next;
|
269
|
+
Slot s;
|
270
|
+
|
271
|
+
for (s = c->slots[i]; NULL != s; s = next) {
|
272
|
+
next = s->next;
|
273
|
+
OJ_FREE(s);
|
274
|
+
}
|
275
|
+
}
|
276
|
+
OJ_FREE((void *)c->slots);
|
277
|
+
OJ_FREE(c);
|
278
|
+
}
|
279
|
+
|
280
|
+
void cache_mark(void *data) {
|
281
|
+
Cache c = (Cache)data;
|
282
|
+
uint64_t i;
|
283
|
+
|
284
|
+
#if !HAVE_PTHREAD_MUTEX_INIT
|
285
|
+
rb_gc_mark(c->mutex);
|
286
|
+
#endif
|
287
|
+
if (0 == c->cnt) {
|
288
|
+
return;
|
289
|
+
}
|
290
|
+
for (i = 0; i < c->size; i++) {
|
291
|
+
Slot s;
|
292
|
+
Slot prev = NULL;
|
293
|
+
Slot next;
|
294
|
+
|
295
|
+
for (s = c->slots[i]; NULL != s; s = next) {
|
296
|
+
next = s->next;
|
297
|
+
if (0 == s->use_cnt) {
|
298
|
+
if (NULL == prev) {
|
299
|
+
c->slots[i] = next;
|
300
|
+
} else {
|
301
|
+
prev->next = next;
|
302
|
+
}
|
303
|
+
c->cnt--;
|
304
|
+
s->next = c->reuse;
|
305
|
+
c->reuse = s;
|
306
|
+
c->rcnt++;
|
307
|
+
continue;
|
308
|
+
}
|
309
|
+
switch (c->xrate) {
|
310
|
+
case 0: break;
|
311
|
+
case 2: s->use_cnt -= 2; break;
|
312
|
+
case 3: s->use_cnt /= 2; break;
|
313
|
+
default: s->use_cnt--; break;
|
314
|
+
}
|
315
|
+
if (c->mark) {
|
316
|
+
rb_gc_mark(s->val);
|
317
|
+
}
|
318
|
+
prev = s;
|
319
|
+
}
|
320
|
+
}
|
321
|
+
}
|
322
|
+
|
323
|
+
VALUE
|
324
|
+
cache_intern(Cache c, const char *key, size_t len) {
|
325
|
+
if (CACHE_MAX_KEY <= len) {
|
326
|
+
return c->form(key, len);
|
327
|
+
}
|
328
|
+
return c->intern(c, key, len);
|
329
|
+
}
|
data/ext/oj/cache.h
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
// Copyright (c) 2021 Peter Ohler. All rights reserved.
|
2
|
+
// Licensed under the MIT License. See LICENSE file in the project root for license details.
|
3
|
+
|
4
|
+
#ifndef CACHE_H
|
5
|
+
#define CACHE_H
|
6
|
+
|
7
|
+
#include <ruby.h>
|
8
|
+
#include <stdbool.h>
|
9
|
+
|
10
|
+
#define CACHE_MAX_KEY 35
|
11
|
+
|
12
|
+
struct _cache;
|
13
|
+
typedef struct _cache *Cache;
|
14
|
+
|
15
|
+
extern struct _cache *cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark, bool locking);
|
16
|
+
extern void cache_free(void *data);
|
17
|
+
extern void cache_mark(void *data);
|
18
|
+
extern void cache_set_form(struct _cache *c, VALUE (*form)(const char *str, size_t len));
|
19
|
+
extern VALUE cache_intern(struct _cache *c, const char *key, size_t len);
|
20
|
+
extern void cache_set_expunge_rate(struct _cache *c, int rate);
|
21
|
+
|
22
|
+
#endif /* CACHE_H */
|
data/ext/oj/cache8.c
CHANGED
@@ -9,6 +9,7 @@
|
|
9
9
|
#include <stdlib.h>
|
10
10
|
#include <string.h>
|
11
11
|
|
12
|
+
#include "mem.h"
|
12
13
|
#include "ruby.h"
|
13
14
|
|
14
15
|
#define BITS 4
|
@@ -32,16 +33,18 @@ void oj_cache8_new(Cache8 *cache) {
|
|
32
33
|
Bucket *b;
|
33
34
|
int i;
|
34
35
|
|
35
|
-
*cache =
|
36
|
+
*cache = OJ_R_ALLOC(struct _cache8);
|
36
37
|
for (i = SLOT_CNT, b = (*cache)->buckets; 0 < i; i--, b++) {
|
37
38
|
b->value = 0;
|
38
39
|
}
|
39
40
|
}
|
40
41
|
|
41
|
-
void oj_cache8_delete(Cache8 cache) {
|
42
|
+
void oj_cache8_delete(Cache8 cache) {
|
43
|
+
cache8_delete(cache, 0);
|
44
|
+
}
|
42
45
|
|
43
46
|
static void cache8_delete(Cache8 cache, int depth) {
|
44
|
-
Bucket
|
47
|
+
Bucket *b;
|
45
48
|
unsigned int i;
|
46
49
|
|
47
50
|
for (i = 0, b = cache->buckets; i < SLOT_CNT; i++, b++) {
|
@@ -51,7 +54,7 @@ static void cache8_delete(Cache8 cache, int depth) {
|
|
51
54
|
}
|
52
55
|
}
|
53
56
|
}
|
54
|
-
|
57
|
+
OJ_R_FREE(cache);
|
55
58
|
}
|
56
59
|
|
57
60
|
slot_t oj_cache8_get(Cache8 cache, sid_t key, slot_t **slot) {
|
@@ -79,7 +82,7 @@ void oj_cache8_print(Cache8 cache) {
|
|
79
82
|
}
|
80
83
|
|
81
84
|
static void slot_print(Cache8 c, sid_t key, unsigned int depth) {
|
82
|
-
Bucket
|
85
|
+
Bucket *b;
|
83
86
|
unsigned int i;
|
84
87
|
sid_t k8 = (sid_t)key;
|
85
88
|
sid_t k;
|
@@ -90,11 +93,9 @@ static void slot_print(Cache8 c, sid_t key, unsigned int depth) {
|
|
90
93
|
/*printf("*** key: 0x%016llx depth: %u i: %u\n", k, depth, i); */
|
91
94
|
if (DEPTH - 1 == depth) {
|
92
95
|
#if IS_WINDOWS
|
93
|
-
printf("0x%016lx: %4lu\n", (long unsigned int)k,
|
94
|
-
(long unsigned int)b->value);
|
96
|
+
printf("0x%016lx: %4lu\n", (long unsigned int)k, (long unsigned int)b->value);
|
95
97
|
#else
|
96
|
-
printf("0x%016llx: %4llu\n", (long long unsigned int)k,
|
97
|
-
(long long unsigned int)b->value);
|
98
|
+
printf("0x%016llx: %4llu\n", (long long unsigned int)k, (long long unsigned int)b->value);
|
98
99
|
#endif
|
99
100
|
} else {
|
100
101
|
slot_print(b->child, k, depth + 1);
|
data/ext/oj/circarray.c
CHANGED
@@ -3,10 +3,12 @@
|
|
3
3
|
|
4
4
|
#include "circarray.h"
|
5
5
|
|
6
|
-
|
6
|
+
#include "mem.h"
|
7
|
+
|
8
|
+
CircArray oj_circ_array_new(void) {
|
7
9
|
CircArray ca;
|
8
10
|
|
9
|
-
if (0 == (ca =
|
11
|
+
if (0 == (ca = OJ_R_ALLOC(struct _circArray))) {
|
10
12
|
rb_raise(rb_eNoMemError, "not enough memory\n");
|
11
13
|
}
|
12
14
|
ca->objs = ca->obj_array;
|
@@ -18,9 +20,9 @@ CircArray oj_circ_array_new() {
|
|
18
20
|
|
19
21
|
void oj_circ_array_free(CircArray ca) {
|
20
22
|
if (ca->objs != ca->obj_array) {
|
21
|
-
|
23
|
+
OJ_R_FREE(ca->objs);
|
22
24
|
}
|
23
|
-
|
25
|
+
OJ_R_FREE(ca);
|
24
26
|
}
|
25
27
|
|
26
28
|
void oj_circ_array_set(CircArray ca, VALUE obj, unsigned long id) {
|
@@ -31,12 +33,12 @@ void oj_circ_array_set(CircArray ca, VALUE obj, unsigned long id) {
|
|
31
33
|
unsigned long cnt = id + 512;
|
32
34
|
|
33
35
|
if (ca->objs == ca->obj_array) {
|
34
|
-
if (0 == (ca->objs =
|
36
|
+
if (0 == (ca->objs = OJ_R_ALLOC_N(VALUE, cnt))) {
|
35
37
|
rb_raise(rb_eNoMemError, "not enough memory\n");
|
36
38
|
}
|
37
39
|
memcpy(ca->objs, ca->obj_array, sizeof(VALUE) * ca->cnt);
|
38
40
|
} else {
|
39
|
-
|
41
|
+
OJ_R_REALLOC_N(ca->objs, VALUE, cnt);
|
40
42
|
}
|
41
43
|
ca->size = cnt;
|
42
44
|
}
|
data/ext/oj/circarray.h
CHANGED
@@ -9,10 +9,10 @@
|
|
9
9
|
|
10
10
|
typedef struct _circArray {
|
11
11
|
VALUE obj_array[1024];
|
12
|
-
VALUE
|
12
|
+
VALUE* objs;
|
13
13
|
unsigned long size; // allocated size or initial array size
|
14
14
|
unsigned long cnt;
|
15
|
-
}
|
15
|
+
}* CircArray;
|
16
16
|
|
17
17
|
extern CircArray oj_circ_array_new(void);
|
18
18
|
extern void oj_circ_array_free(CircArray ca);
|