oj 3.11.5 → 3.16.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1421 -0
- data/README.md +19 -5
- data/RELEASE_NOTES.md +61 -0
- data/ext/oj/buf.h +20 -6
- data/ext/oj/cache.c +329 -0
- data/ext/oj/cache.h +22 -0
- data/ext/oj/cache8.c +10 -9
- data/ext/oj/circarray.c +8 -6
- data/ext/oj/circarray.h +2 -2
- data/ext/oj/code.c +19 -33
- data/ext/oj/code.h +2 -2
- data/ext/oj/compat.c +27 -77
- data/ext/oj/custom.c +86 -179
- data/ext/oj/debug.c +126 -0
- data/ext/oj/dump.c +256 -249
- data/ext/oj/dump.h +26 -12
- data/ext/oj/dump_compat.c +565 -642
- data/ext/oj/dump_leaf.c +17 -63
- data/ext/oj/dump_object.c +65 -187
- data/ext/oj/dump_strict.c +27 -51
- data/ext/oj/encoder.c +43 -0
- data/ext/oj/err.c +2 -13
- data/ext/oj/err.h +24 -8
- data/ext/oj/extconf.rb +21 -6
- data/ext/oj/fast.c +149 -149
- data/ext/oj/intern.c +313 -0
- data/ext/oj/intern.h +22 -0
- data/ext/oj/mem.c +318 -0
- data/ext/oj/mem.h +53 -0
- data/ext/oj/mimic_json.c +121 -106
- data/ext/oj/object.c +85 -162
- data/ext/oj/odd.c +89 -67
- data/ext/oj/odd.h +15 -15
- data/ext/oj/oj.c +542 -411
- data/ext/oj/oj.h +99 -73
- data/ext/oj/parse.c +175 -187
- data/ext/oj/parse.h +26 -24
- data/ext/oj/parser.c +1600 -0
- data/ext/oj/parser.h +101 -0
- data/ext/oj/rails.c +112 -159
- data/ext/oj/rails.h +1 -1
- data/ext/oj/reader.c +11 -14
- data/ext/oj/reader.h +4 -2
- data/ext/oj/resolve.c +5 -24
- data/ext/oj/rxclass.c +7 -6
- data/ext/oj/rxclass.h +1 -1
- data/ext/oj/saj.c +22 -33
- data/ext/oj/saj2.c +584 -0
- data/ext/oj/saj2.h +23 -0
- data/ext/oj/scp.c +5 -28
- data/ext/oj/sparse.c +28 -72
- data/ext/oj/stream_writer.c +50 -40
- data/ext/oj/strict.c +56 -61
- data/ext/oj/string_writer.c +72 -39
- data/ext/oj/trace.h +31 -4
- data/ext/oj/usual.c +1218 -0
- data/ext/oj/usual.h +69 -0
- data/ext/oj/util.h +1 -1
- data/ext/oj/val_stack.c +14 -3
- data/ext/oj/val_stack.h +8 -7
- data/ext/oj/validate.c +46 -0
- data/ext/oj/wab.c +63 -88
- data/lib/oj/active_support_helper.rb +1 -3
- data/lib/oj/bag.rb +7 -1
- data/lib/oj/easy_hash.rb +4 -5
- data/lib/oj/error.rb +1 -2
- data/lib/oj/json.rb +162 -150
- data/lib/oj/mimic.rb +9 -7
- data/lib/oj/saj.rb +20 -6
- data/lib/oj/schandler.rb +5 -4
- data/lib/oj/state.rb +12 -8
- data/lib/oj/version.rb +1 -2
- data/lib/oj.rb +2 -0
- data/pages/Compatibility.md +1 -1
- data/pages/InstallOptions.md +20 -0
- data/pages/JsonGem.md +15 -0
- data/pages/Modes.md +8 -3
- data/pages/Options.md +43 -5
- data/pages/Parser.md +309 -0
- data/pages/Rails.md +14 -2
- data/test/_test_active.rb +8 -9
- data/test/_test_active_mimic.rb +7 -8
- data/test/_test_mimic_rails.rb +17 -20
- data/test/activerecord/result_test.rb +5 -6
- data/test/activesupport6/encoding_test.rb +63 -28
- data/test/{activesupport5 → activesupport7}/abstract_unit.rb +16 -12
- data/test/{activesupport5 → activesupport7}/decoding_test.rb +2 -10
- data/test/{activesupport5 → activesupport7}/encoding_test.rb +86 -50
- data/test/{activesupport5 → activesupport7}/encoding_test_cases.rb +6 -0
- data/test/{activesupport5 → activesupport7}/time_zone_test_helpers.rb +8 -0
- data/test/files.rb +15 -15
- data/test/foo.rb +16 -45
- data/test/helper.rb +11 -8
- data/test/isolated/shared.rb +3 -2
- data/test/json_gem/json_addition_test.rb +2 -2
- data/test/json_gem/json_common_interface_test.rb +8 -6
- data/test/json_gem/json_encoding_test.rb +0 -0
- data/test/json_gem/json_ext_parser_test.rb +1 -0
- data/test/json_gem/json_fixtures_test.rb +3 -2
- data/test/json_gem/json_generator_test.rb +56 -38
- data/test/json_gem/json_generic_object_test.rb +11 -11
- data/test/json_gem/json_parser_test.rb +54 -47
- data/test/json_gem/json_string_matching_test.rb +9 -9
- data/test/json_gem/test_helper.rb +7 -3
- data/test/mem.rb +34 -0
- data/test/perf.rb +22 -27
- data/test/perf_compat.rb +31 -33
- data/test/perf_dump.rb +50 -0
- data/test/perf_fast.rb +80 -82
- data/test/perf_file.rb +27 -29
- data/test/perf_object.rb +65 -69
- data/test/perf_once.rb +59 -0
- data/test/perf_parser.rb +183 -0
- data/test/perf_saj.rb +46 -54
- data/test/perf_scp.rb +58 -69
- data/test/perf_simple.rb +41 -39
- data/test/perf_strict.rb +74 -82
- data/test/perf_wab.rb +67 -69
- data/test/prec.rb +5 -5
- data/test/sample/change.rb +0 -1
- data/test/sample/dir.rb +0 -1
- data/test/sample/doc.rb +0 -1
- data/test/sample/file.rb +0 -1
- data/test/sample/group.rb +0 -1
- data/test/sample/hasprops.rb +0 -1
- data/test/sample/layer.rb +0 -1
- data/test/sample/rect.rb +0 -1
- data/test/sample/shape.rb +0 -1
- data/test/sample/text.rb +0 -1
- data/test/sample.rb +16 -16
- data/test/sample_json.rb +8 -8
- data/test/test_compat.rb +95 -43
- data/test/test_custom.rb +73 -51
- data/test/test_debian.rb +7 -10
- data/test/test_fast.rb +135 -79
- data/test/test_file.rb +41 -30
- data/test/test_gc.rb +16 -5
- data/test/test_generate.rb +5 -5
- data/test/test_hash.rb +5 -5
- data/test/test_integer_range.rb +9 -9
- data/test/test_null.rb +20 -20
- data/test/test_object.rb +99 -96
- data/test/test_parser.rb +11 -0
- data/test/test_parser_debug.rb +27 -0
- data/test/test_parser_saj.rb +337 -0
- data/test/test_parser_usual.rb +251 -0
- data/test/test_rails.rb +2 -2
- data/test/test_saj.rb +10 -8
- data/test/test_scp.rb +37 -39
- data/test/test_strict.rb +40 -32
- data/test/test_various.rb +165 -84
- data/test/test_wab.rb +48 -44
- data/test/test_writer.rb +47 -47
- data/test/tests.rb +13 -5
- data/test/tests_mimic.rb +12 -3
- data/test/tests_mimic_addition.rb +12 -3
- metadata +74 -128
- data/ext/oj/hash.c +0 -131
- data/ext/oj/hash.h +0 -19
- data/ext/oj/hash_test.c +0 -491
- data/test/activesupport4/decoding_test.rb +0 -108
- data/test/activesupport4/encoding_test.rb +0 -531
- data/test/activesupport4/test_helper.rb +0 -41
- data/test/activesupport5/test_helper.rb +0 -72
- data/test/bar.rb +0 -35
- data/test/baz.rb +0 -16
- data/test/zoo.rb +0 -13
data/README.md
CHANGED
@@ -1,11 +1,13 @@
|
|
1
1
|
# [](http://www.ohler.com/oj) gem
|
2
2
|
|
3
|
-
[](https://github.com/ohler55/oj/actions/workflows/CI.yml)
|
4
|
+

|
5
|
+

|
6
|
+
[](https://tidelift.com/subscription/pkg/rubygems-oj?utm_source=rubygems-oj&utm_medium=referral&utm_campaign=readme)
|
4
7
|
|
5
8
|
A *fast* JSON parser and Object marshaller as a Ruby gem.
|
6
9
|
|
7
|
-
Version 3.
|
8
|
-
also provides additional optimization options.
|
10
|
+
Version 3.13 is out with a much faster parser (`Oj::Parser`) and option isolation.
|
9
11
|
|
10
12
|
## Using
|
11
13
|
|
@@ -40,6 +42,15 @@ or in Bundler:
|
|
40
42
|
gem 'oj'
|
41
43
|
```
|
42
44
|
|
45
|
+
## Rails and json quickstart
|
46
|
+
|
47
|
+
See the Quickstart sections of the [Rails](pages/Rails.md) and [json](pages/JsonGem.md) docs.
|
48
|
+
|
49
|
+
## multi_json
|
50
|
+
|
51
|
+
Code which uses [multi_json](https://github.com/intridea/multi_json)
|
52
|
+
will automatically prefer Oj if it is installed.
|
53
|
+
|
43
54
|
## Support
|
44
55
|
|
45
56
|
[Get supported Oj with a Tidelift Subscription.](https://tidelift.com/subscription/pkg/rubygems-oj?utm_source=rubygems-oj&utm_medium=referral&utm_campaign=readme) Security updates are [supported](https://tidelift.com/security).
|
@@ -50,7 +61,7 @@ For more details on options, modes, advanced features, and more follow these
|
|
50
61
|
links.
|
51
62
|
|
52
63
|
- [{file:Options.md}](pages/Options.md) for parse and dump options.
|
53
|
-
- [{file:Modes.md}](pages/Modes.md) for details on modes for strict JSON compliance,
|
64
|
+
- [{file:Modes.md}](pages/Modes.md) for details on modes for strict JSON compliance, mimicking the JSON gem, and mimicking Rails and ActiveSupport behavior.
|
54
65
|
- [{file:JsonGem.md}](pages/JsonGem.md) includes more details on json gem compatibility and use.
|
55
66
|
- [{file:Rails.md}](pages/Rails.md) includes more details on Rails and ActiveSupport compatibility and use.
|
56
67
|
- [{file:Custom.md}](pages/Custom.md) includes more details on Custom mode.
|
@@ -58,10 +69,11 @@ links.
|
|
58
69
|
- [{file:Compatibility.md}](pages/Compatibility.md) lists current compatibility with Rubys and Rails.
|
59
70
|
- [{file:Advanced.md}](pages/Advanced.md) for fast parser and marshalling features.
|
60
71
|
- [{file:Security.md}](pages/Security.md) for security considerations.
|
72
|
+
- [{file:InstallOptions.md}](pages/InstallOptions.md) for install option.
|
61
73
|
|
62
74
|
## Releases
|
63
75
|
|
64
|
-
See [{file:CHANGELOG.md}](CHANGELOG.md)
|
76
|
+
See [{file:CHANGELOG.md}](CHANGELOG.md) and [{file:RELEASE_NOTES.md}](RELEASE_NOTES.md)
|
65
77
|
|
66
78
|
## Links
|
67
79
|
|
@@ -97,6 +109,8 @@ Follow [@peterohler on Twitter](http://twitter.com/peterohler) for announcements
|
|
97
109
|
|
98
110
|
- *Agoo-C, a high performance C web server supporting GraphQL on GitHub*: https://github.com/ohler55/agoo-c
|
99
111
|
|
112
|
+
- *oj-introspect, an example of creating an Oj parser extension in C*: https://github.com/meinac/oj-introspect
|
113
|
+
|
100
114
|
#### Contributing
|
101
115
|
|
102
116
|
+ Provide a Pull Request off the `develop` branch.
|
data/RELEASE_NOTES.md
ADDED
@@ -0,0 +1,61 @@
|
|
1
|
+
# RELEASE NOTES
|
2
|
+
|
3
|
+
The release notes here are organized by release. For a list of changes
|
4
|
+
see the See [{file:CHANGELOG.md}](CHANGELOG.md) file. In this file are
|
5
|
+
the steps to take to aid in keeping things rolling after updating to
|
6
|
+
the latest version.
|
7
|
+
|
8
|
+
## 3.13.7
|
9
|
+
|
10
|
+
The default for JSON when mimicked by Oj is now to set
|
11
|
+
`:allow_invalid_unicode`. To change that behavior JSON.load, set that
|
12
|
+
option to false.
|
13
|
+
|
14
|
+
## 3.13.x
|
15
|
+
|
16
|
+
This release included a new cache that performs better than the
|
17
|
+
earlier cache and a new high performance parser.
|
18
|
+
|
19
|
+
### Cache
|
20
|
+
|
21
|
+
The new cache includes a least recently used expiration to reduce
|
22
|
+
memory use. The cache is also self adjusting and will expand as needed
|
23
|
+
for better performance. It also handles Hash keys and string values
|
24
|
+
with two options, `:cache_keys`, a boolean and `:cache_str` an
|
25
|
+
integer. The `:cache_str` if set to more than zero is the limit for
|
26
|
+
the length of string values to cache. The maximum value is 35 which
|
27
|
+
allows strings up to 34 bytes to be cached.
|
28
|
+
|
29
|
+
One interesting aspect of the cache is not so much the string caching
|
30
|
+
which performs similar to the Ruby intern functions but the caching of
|
31
|
+
symbols and object attribute names. There is a significant gain for
|
32
|
+
symbols and object attributes.
|
33
|
+
|
34
|
+
If the cache is not desired then setting the default options to turn
|
35
|
+
it off can be done with this line:
|
36
|
+
|
37
|
+
``` ruby
|
38
|
+
Oj.default_options = { cache_keys: false, cache_str: 0 }
|
39
|
+
```
|
40
|
+
|
41
|
+
### Oj::Parser
|
42
|
+
|
43
|
+
The new parser uses a different core that follows the approach taken
|
44
|
+
by [OjC](https://github.com/ohler55/ojc) and
|
45
|
+
[OjG](https://github.com/ohler55/ojg). It also takes advantage of the
|
46
|
+
bulk Array and Hash functions. Another issue the new parser addresses
|
47
|
+
is option management. Instead of a single global default_options each
|
48
|
+
parser instance maintains it's own options.
|
49
|
+
|
50
|
+
There is a price to be paid when using the Oj::Parser. The API is not
|
51
|
+
the same the older parser. A single parser can only be used in a
|
52
|
+
single thread. This allows reuse of internal buffers for additional
|
53
|
+
improvements in performance.
|
54
|
+
|
55
|
+
The performane advantage of the Oj::Parse is that it is more than 3
|
56
|
+
times faster than the Oj::compat_load call and 6 times faster than the
|
57
|
+
JSON gem.
|
58
|
+
|
59
|
+
### Dump Performance
|
60
|
+
|
61
|
+
Thanks to Watson1978 Oj.dump also received a speed boost.
|
data/ext/oj/buf.h
CHANGED
@@ -4,6 +4,7 @@
|
|
4
4
|
#ifndef OJ_BUF_H
|
5
5
|
#define OJ_BUF_H
|
6
6
|
|
7
|
+
#include "mem.h"
|
7
8
|
#include "ruby.h"
|
8
9
|
|
9
10
|
typedef struct _buf {
|
@@ -11,7 +12,7 @@ typedef struct _buf {
|
|
11
12
|
char *end;
|
12
13
|
char *tail;
|
13
14
|
char base[1024];
|
14
|
-
} *
|
15
|
+
} *Buf;
|
15
16
|
|
16
17
|
inline static void buf_init(Buf buf) {
|
17
18
|
buf->head = buf->base;
|
@@ -19,9 +20,13 @@ inline static void buf_init(Buf buf) {
|
|
19
20
|
buf->tail = buf->head;
|
20
21
|
}
|
21
22
|
|
23
|
+
inline static void buf_reset(Buf buf) {
|
24
|
+
buf->tail = buf->head;
|
25
|
+
}
|
26
|
+
|
22
27
|
inline static void buf_cleanup(Buf buf) {
|
23
28
|
if (buf->base != buf->head) {
|
24
|
-
|
29
|
+
OJ_R_FREE(buf->head);
|
25
30
|
}
|
26
31
|
}
|
27
32
|
|
@@ -29,17 +34,26 @@ inline static size_t buf_len(Buf buf) {
|
|
29
34
|
return buf->tail - buf->head;
|
30
35
|
}
|
31
36
|
|
37
|
+
inline static const char *buf_str(Buf buf) {
|
38
|
+
*buf->tail = '\0';
|
39
|
+
return buf->head;
|
40
|
+
}
|
41
|
+
|
32
42
|
inline static void buf_append_string(Buf buf, const char *s, size_t slen) {
|
43
|
+
if (0 == slen) {
|
44
|
+
return;
|
45
|
+
}
|
46
|
+
|
33
47
|
if (buf->end <= buf->tail + slen) {
|
34
48
|
size_t len = buf->end - buf->head;
|
35
49
|
size_t toff = buf->tail - buf->head;
|
36
50
|
size_t new_len = len + slen + len / 2;
|
37
51
|
|
38
52
|
if (buf->base == buf->head) {
|
39
|
-
buf->head =
|
53
|
+
buf->head = OJ_R_ALLOC_N(char, new_len);
|
40
54
|
memcpy(buf->head, buf->base, len);
|
41
55
|
} else {
|
42
|
-
|
56
|
+
OJ_R_REALLOC_N(buf->head, char, new_len);
|
43
57
|
}
|
44
58
|
buf->tail = buf->head + toff;
|
45
59
|
buf->end = buf->head + new_len - 1;
|
@@ -55,10 +69,10 @@ inline static void buf_append(Buf buf, char c) {
|
|
55
69
|
size_t new_len = len + len / 2;
|
56
70
|
|
57
71
|
if (buf->base == buf->head) {
|
58
|
-
buf->head =
|
72
|
+
buf->head = OJ_R_ALLOC_N(char, new_len);
|
59
73
|
memcpy(buf->head, buf->base, len);
|
60
74
|
} else {
|
61
|
-
|
75
|
+
OJ_R_REALLOC_N(buf->head, char, new_len);
|
62
76
|
}
|
63
77
|
buf->tail = buf->head + toff;
|
64
78
|
buf->end = buf->head + new_len - 1;
|
data/ext/oj/cache.c
ADDED
@@ -0,0 +1,329 @@
|
|
1
|
+
// Copyright (c) 2011, 2021 Peter Ohler. All rights reserved.
|
2
|
+
// Licensed under the MIT License. See LICENSE file in the project root for license details.
|
3
|
+
|
4
|
+
#if HAVE_PTHREAD_MUTEX_INIT
|
5
|
+
#include <pthread.h>
|
6
|
+
#endif
|
7
|
+
#include <stdlib.h>
|
8
|
+
|
9
|
+
#include "cache.h"
|
10
|
+
#include "mem.h"
|
11
|
+
|
12
|
+
// The stdlib calloc, realloc, and free are used instead of the Ruby ALLOC,
|
13
|
+
// ALLOC_N, REALLOC, and xfree since the later could trigger a GC which will
|
14
|
+
// either corrupt memory or if the mark function locks will deadlock.
|
15
|
+
|
16
|
+
#define REHASH_LIMIT 4
|
17
|
+
#define MIN_SHIFT 8
|
18
|
+
#define REUSE_MAX 8192
|
19
|
+
|
20
|
+
#if HAVE_PTHREAD_MUTEX_INIT
|
21
|
+
#define CACHE_LOCK(c) pthread_mutex_lock(&((c)->mutex))
|
22
|
+
#define CACHE_UNLOCK(c) pthread_mutex_unlock(&((c)->mutex))
|
23
|
+
#else
|
24
|
+
#define CACHE_LOCK(c) rb_mutex_lock((c)->mutex)
|
25
|
+
#define CACHE_UNLOCK(c) rb_mutex_unlock((c)->mutex)
|
26
|
+
#endif
|
27
|
+
|
28
|
+
// almost the Murmur hash algorithm
|
29
|
+
#define M 0x5bd1e995
|
30
|
+
|
31
|
+
typedef struct _slot {
|
32
|
+
struct _slot *next;
|
33
|
+
VALUE val;
|
34
|
+
uint64_t hash;
|
35
|
+
volatile uint32_t use_cnt;
|
36
|
+
uint8_t klen;
|
37
|
+
char key[CACHE_MAX_KEY];
|
38
|
+
} *Slot;
|
39
|
+
|
40
|
+
typedef struct _cache {
|
41
|
+
volatile Slot *slots;
|
42
|
+
volatile size_t cnt;
|
43
|
+
VALUE (*form)(const char *str, size_t len);
|
44
|
+
uint64_t size;
|
45
|
+
uint64_t mask;
|
46
|
+
VALUE (*intern)(struct _cache *c, const char *key, size_t len);
|
47
|
+
volatile Slot reuse;
|
48
|
+
size_t rcnt;
|
49
|
+
#if HAVE_PTHREAD_MUTEX_INIT
|
50
|
+
pthread_mutex_t mutex;
|
51
|
+
#else
|
52
|
+
VALUE mutex;
|
53
|
+
#endif
|
54
|
+
uint8_t xrate;
|
55
|
+
bool mark;
|
56
|
+
} *Cache;
|
57
|
+
|
58
|
+
void cache_set_form(Cache c, VALUE (*form)(const char *str, size_t len)) {
|
59
|
+
c->form = form;
|
60
|
+
}
|
61
|
+
|
62
|
+
static uint64_t hash_calc(const uint8_t *key, size_t len) {
|
63
|
+
const uint8_t *end = key + len;
|
64
|
+
const uint8_t *endless = key + (len & 0xFFFFFFFC);
|
65
|
+
uint64_t h = (uint64_t)len;
|
66
|
+
uint64_t k;
|
67
|
+
|
68
|
+
while (key < endless) {
|
69
|
+
k = (uint64_t)*key++;
|
70
|
+
k |= (uint64_t)*key++ << 8;
|
71
|
+
k |= (uint64_t)*key++ << 16;
|
72
|
+
k |= (uint64_t)*key++ << 24;
|
73
|
+
|
74
|
+
k *= M;
|
75
|
+
k ^= k >> 24;
|
76
|
+
h *= M;
|
77
|
+
h ^= k * M;
|
78
|
+
}
|
79
|
+
if (1 < end - key) {
|
80
|
+
uint16_t k16 = (uint16_t)*key++;
|
81
|
+
|
82
|
+
k16 |= (uint16_t)*key++ << 8;
|
83
|
+
h ^= k16 << 8;
|
84
|
+
}
|
85
|
+
if (key < end) {
|
86
|
+
h ^= *key;
|
87
|
+
}
|
88
|
+
h *= M;
|
89
|
+
h ^= h >> 13;
|
90
|
+
h *= M;
|
91
|
+
h ^= h >> 15;
|
92
|
+
|
93
|
+
return h;
|
94
|
+
}
|
95
|
+
|
96
|
+
static void rehash(Cache c) {
|
97
|
+
uint64_t osize;
|
98
|
+
Slot *end;
|
99
|
+
Slot *sp;
|
100
|
+
|
101
|
+
osize = c->size;
|
102
|
+
c->size = osize * 4;
|
103
|
+
c->mask = c->size - 1;
|
104
|
+
c->slots = OJ_REALLOC((void *)c->slots, sizeof(Slot) * c->size);
|
105
|
+
memset((Slot *)c->slots + osize, 0, sizeof(Slot) * osize * 3);
|
106
|
+
end = (Slot *)c->slots + osize;
|
107
|
+
for (sp = (Slot *)c->slots; sp < end; sp++) {
|
108
|
+
Slot s = *sp;
|
109
|
+
Slot next = NULL;
|
110
|
+
|
111
|
+
*sp = NULL;
|
112
|
+
for (; NULL != s; s = next) {
|
113
|
+
uint64_t h = s->hash & c->mask;
|
114
|
+
Slot *bucket = (Slot *)c->slots + h;
|
115
|
+
|
116
|
+
next = s->next;
|
117
|
+
s->next = *bucket;
|
118
|
+
*bucket = s;
|
119
|
+
}
|
120
|
+
}
|
121
|
+
}
|
122
|
+
|
123
|
+
static VALUE lockless_intern(Cache c, const char *key, size_t len) {
|
124
|
+
uint64_t h = hash_calc((const uint8_t *)key, len);
|
125
|
+
Slot *bucket = (Slot *)c->slots + (h & c->mask);
|
126
|
+
Slot b;
|
127
|
+
volatile VALUE rkey;
|
128
|
+
|
129
|
+
while (REUSE_MAX < c->rcnt) {
|
130
|
+
if (NULL != (b = c->reuse)) {
|
131
|
+
c->reuse = b->next;
|
132
|
+
OJ_FREE(b);
|
133
|
+
c->rcnt--;
|
134
|
+
} else {
|
135
|
+
// An accounting error occured somewhere so correct it.
|
136
|
+
c->rcnt = 0;
|
137
|
+
}
|
138
|
+
}
|
139
|
+
for (b = *bucket; NULL != b; b = b->next) {
|
140
|
+
if ((uint8_t)len == b->klen && 0 == strncmp(b->key, key, len)) {
|
141
|
+
b->use_cnt += 16;
|
142
|
+
return b->val;
|
143
|
+
}
|
144
|
+
}
|
145
|
+
rkey = c->form(key, len);
|
146
|
+
if (NULL == (b = c->reuse)) {
|
147
|
+
b = OJ_CALLOC(1, sizeof(struct _slot));
|
148
|
+
} else {
|
149
|
+
c->reuse = b->next;
|
150
|
+
c->rcnt--;
|
151
|
+
}
|
152
|
+
b->hash = h;
|
153
|
+
memcpy(b->key, key, len);
|
154
|
+
b->klen = (uint8_t)len;
|
155
|
+
b->key[len] = '\0';
|
156
|
+
b->val = rkey;
|
157
|
+
b->use_cnt = 4;
|
158
|
+
b->next = *bucket;
|
159
|
+
*bucket = b;
|
160
|
+
c->cnt++; // Don't worry about wrapping. Worse case is the entry is removed and recreated.
|
161
|
+
if (REHASH_LIMIT < c->cnt / c->size) {
|
162
|
+
rehash(c);
|
163
|
+
}
|
164
|
+
return rkey;
|
165
|
+
}
|
166
|
+
|
167
|
+
static VALUE locking_intern(Cache c, const char *key, size_t len) {
|
168
|
+
uint64_t h;
|
169
|
+
Slot *bucket;
|
170
|
+
Slot b;
|
171
|
+
uint64_t old_size;
|
172
|
+
volatile VALUE rkey;
|
173
|
+
|
174
|
+
CACHE_LOCK(c);
|
175
|
+
while (REUSE_MAX < c->rcnt) {
|
176
|
+
if (NULL != (b = c->reuse)) {
|
177
|
+
c->reuse = b->next;
|
178
|
+
OJ_FREE(b);
|
179
|
+
c->rcnt--;
|
180
|
+
} else {
|
181
|
+
// An accounting error occured somewhere so correct it.
|
182
|
+
c->rcnt = 0;
|
183
|
+
}
|
184
|
+
}
|
185
|
+
h = hash_calc((const uint8_t *)key, len);
|
186
|
+
bucket = (Slot *)c->slots + (h & c->mask);
|
187
|
+
for (b = *bucket; NULL != b; b = b->next) {
|
188
|
+
if ((uint8_t)len == b->klen && 0 == strncmp(b->key, key, len)) {
|
189
|
+
b->use_cnt += 4;
|
190
|
+
CACHE_UNLOCK(c);
|
191
|
+
|
192
|
+
return b->val;
|
193
|
+
}
|
194
|
+
}
|
195
|
+
old_size = c->size;
|
196
|
+
// The creation of a new value may trigger a GC which be a problem if the
|
197
|
+
// cache is locked so make sure it is unlocked for the key value creation.
|
198
|
+
if (NULL != (b = c->reuse)) {
|
199
|
+
c->reuse = b->next;
|
200
|
+
c->rcnt--;
|
201
|
+
}
|
202
|
+
CACHE_UNLOCK(c);
|
203
|
+
if (NULL == b) {
|
204
|
+
b = OJ_CALLOC(1, sizeof(struct _slot));
|
205
|
+
}
|
206
|
+
rkey = c->form(key, len);
|
207
|
+
b->hash = h;
|
208
|
+
memcpy(b->key, key, len);
|
209
|
+
b->klen = (uint8_t)len;
|
210
|
+
b->key[len] = '\0';
|
211
|
+
b->val = rkey;
|
212
|
+
b->use_cnt = 16;
|
213
|
+
|
214
|
+
// Lock again to add the new entry.
|
215
|
+
CACHE_LOCK(c);
|
216
|
+
if (old_size != c->size) {
|
217
|
+
h = hash_calc((const uint8_t *)key, len);
|
218
|
+
bucket = (Slot *)c->slots + (h & c->mask);
|
219
|
+
}
|
220
|
+
b->next = *bucket;
|
221
|
+
*bucket = b;
|
222
|
+
c->cnt++; // Don't worry about wrapping. Worse case is the entry is removed and recreated.
|
223
|
+
if (REHASH_LIMIT < c->cnt / c->size) {
|
224
|
+
rehash(c);
|
225
|
+
}
|
226
|
+
CACHE_UNLOCK(c);
|
227
|
+
|
228
|
+
return rkey;
|
229
|
+
}
|
230
|
+
|
231
|
+
Cache cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark, bool locking) {
|
232
|
+
Cache c = OJ_CALLOC(1, sizeof(struct _cache));
|
233
|
+
int shift = 0;
|
234
|
+
|
235
|
+
for (; REHASH_LIMIT < size; size /= 2, shift++) {
|
236
|
+
}
|
237
|
+
if (shift < MIN_SHIFT) {
|
238
|
+
shift = MIN_SHIFT;
|
239
|
+
}
|
240
|
+
#if HAVE_PTHREAD_MUTEX_INIT
|
241
|
+
pthread_mutex_init(&c->mutex, NULL);
|
242
|
+
#else
|
243
|
+
c->mutex = rb_mutex_new();
|
244
|
+
#endif
|
245
|
+
c->size = 1 << shift;
|
246
|
+
c->mask = c->size - 1;
|
247
|
+
c->slots = OJ_CALLOC(c->size, sizeof(Slot));
|
248
|
+
c->form = form;
|
249
|
+
c->xrate = 1; // low
|
250
|
+
c->mark = mark;
|
251
|
+
if (locking) {
|
252
|
+
c->intern = locking_intern;
|
253
|
+
} else {
|
254
|
+
c->intern = lockless_intern;
|
255
|
+
}
|
256
|
+
return c;
|
257
|
+
}
|
258
|
+
|
259
|
+
void cache_set_expunge_rate(Cache c, int rate) {
|
260
|
+
c->xrate = (uint8_t)rate;
|
261
|
+
}
|
262
|
+
|
263
|
+
void cache_free(void *data) {
|
264
|
+
Cache c = (Cache)data;
|
265
|
+
uint64_t i;
|
266
|
+
|
267
|
+
for (i = 0; i < c->size; i++) {
|
268
|
+
Slot next;
|
269
|
+
Slot s;
|
270
|
+
|
271
|
+
for (s = c->slots[i]; NULL != s; s = next) {
|
272
|
+
next = s->next;
|
273
|
+
OJ_FREE(s);
|
274
|
+
}
|
275
|
+
}
|
276
|
+
OJ_FREE((void *)c->slots);
|
277
|
+
OJ_FREE(c);
|
278
|
+
}
|
279
|
+
|
280
|
+
void cache_mark(void *data) {
|
281
|
+
Cache c = (Cache)data;
|
282
|
+
uint64_t i;
|
283
|
+
|
284
|
+
#if !HAVE_PTHREAD_MUTEX_INIT
|
285
|
+
rb_gc_mark(c->mutex);
|
286
|
+
#endif
|
287
|
+
if (0 == c->cnt) {
|
288
|
+
return;
|
289
|
+
}
|
290
|
+
for (i = 0; i < c->size; i++) {
|
291
|
+
Slot s;
|
292
|
+
Slot prev = NULL;
|
293
|
+
Slot next;
|
294
|
+
|
295
|
+
for (s = c->slots[i]; NULL != s; s = next) {
|
296
|
+
next = s->next;
|
297
|
+
if (0 == s->use_cnt) {
|
298
|
+
if (NULL == prev) {
|
299
|
+
c->slots[i] = next;
|
300
|
+
} else {
|
301
|
+
prev->next = next;
|
302
|
+
}
|
303
|
+
c->cnt--;
|
304
|
+
s->next = c->reuse;
|
305
|
+
c->reuse = s;
|
306
|
+
c->rcnt++;
|
307
|
+
continue;
|
308
|
+
}
|
309
|
+
switch (c->xrate) {
|
310
|
+
case 0: break;
|
311
|
+
case 2: s->use_cnt -= 2; break;
|
312
|
+
case 3: s->use_cnt /= 2; break;
|
313
|
+
default: s->use_cnt--; break;
|
314
|
+
}
|
315
|
+
if (c->mark) {
|
316
|
+
rb_gc_mark(s->val);
|
317
|
+
}
|
318
|
+
prev = s;
|
319
|
+
}
|
320
|
+
}
|
321
|
+
}
|
322
|
+
|
323
|
+
VALUE
|
324
|
+
cache_intern(Cache c, const char *key, size_t len) {
|
325
|
+
if (CACHE_MAX_KEY <= len) {
|
326
|
+
return c->form(key, len);
|
327
|
+
}
|
328
|
+
return c->intern(c, key, len);
|
329
|
+
}
|
data/ext/oj/cache.h
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
// Copyright (c) 2021 Peter Ohler. All rights reserved.
|
2
|
+
// Licensed under the MIT License. See LICENSE file in the project root for license details.
|
3
|
+
|
4
|
+
#ifndef CACHE_H
|
5
|
+
#define CACHE_H
|
6
|
+
|
7
|
+
#include <ruby.h>
|
8
|
+
#include <stdbool.h>
|
9
|
+
|
10
|
+
#define CACHE_MAX_KEY 35
|
11
|
+
|
12
|
+
struct _cache;
|
13
|
+
typedef struct _cache *Cache;
|
14
|
+
|
15
|
+
extern struct _cache *cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark, bool locking);
|
16
|
+
extern void cache_free(void *data);
|
17
|
+
extern void cache_mark(void *data);
|
18
|
+
extern void cache_set_form(struct _cache *c, VALUE (*form)(const char *str, size_t len));
|
19
|
+
extern VALUE cache_intern(struct _cache *c, const char *key, size_t len);
|
20
|
+
extern void cache_set_expunge_rate(struct _cache *c, int rate);
|
21
|
+
|
22
|
+
#endif /* CACHE_H */
|
data/ext/oj/cache8.c
CHANGED
@@ -9,6 +9,7 @@
|
|
9
9
|
#include <stdlib.h>
|
10
10
|
#include <string.h>
|
11
11
|
|
12
|
+
#include "mem.h"
|
12
13
|
#include "ruby.h"
|
13
14
|
|
14
15
|
#define BITS 4
|
@@ -32,16 +33,18 @@ void oj_cache8_new(Cache8 *cache) {
|
|
32
33
|
Bucket *b;
|
33
34
|
int i;
|
34
35
|
|
35
|
-
*cache =
|
36
|
+
*cache = OJ_R_ALLOC(struct _cache8);
|
36
37
|
for (i = SLOT_CNT, b = (*cache)->buckets; 0 < i; i--, b++) {
|
37
38
|
b->value = 0;
|
38
39
|
}
|
39
40
|
}
|
40
41
|
|
41
|
-
void oj_cache8_delete(Cache8 cache) {
|
42
|
+
void oj_cache8_delete(Cache8 cache) {
|
43
|
+
cache8_delete(cache, 0);
|
44
|
+
}
|
42
45
|
|
43
46
|
static void cache8_delete(Cache8 cache, int depth) {
|
44
|
-
Bucket
|
47
|
+
Bucket *b;
|
45
48
|
unsigned int i;
|
46
49
|
|
47
50
|
for (i = 0, b = cache->buckets; i < SLOT_CNT; i++, b++) {
|
@@ -51,7 +54,7 @@ static void cache8_delete(Cache8 cache, int depth) {
|
|
51
54
|
}
|
52
55
|
}
|
53
56
|
}
|
54
|
-
|
57
|
+
OJ_R_FREE(cache);
|
55
58
|
}
|
56
59
|
|
57
60
|
slot_t oj_cache8_get(Cache8 cache, sid_t key, slot_t **slot) {
|
@@ -79,7 +82,7 @@ void oj_cache8_print(Cache8 cache) {
|
|
79
82
|
}
|
80
83
|
|
81
84
|
static void slot_print(Cache8 c, sid_t key, unsigned int depth) {
|
82
|
-
Bucket
|
85
|
+
Bucket *b;
|
83
86
|
unsigned int i;
|
84
87
|
sid_t k8 = (sid_t)key;
|
85
88
|
sid_t k;
|
@@ -90,11 +93,9 @@ static void slot_print(Cache8 c, sid_t key, unsigned int depth) {
|
|
90
93
|
/*printf("*** key: 0x%016llx depth: %u i: %u\n", k, depth, i); */
|
91
94
|
if (DEPTH - 1 == depth) {
|
92
95
|
#if IS_WINDOWS
|
93
|
-
printf("0x%016lx: %4lu\n", (long unsigned int)k,
|
94
|
-
(long unsigned int)b->value);
|
96
|
+
printf("0x%016lx: %4lu\n", (long unsigned int)k, (long unsigned int)b->value);
|
95
97
|
#else
|
96
|
-
printf("0x%016llx: %4llu\n", (long long unsigned int)k,
|
97
|
-
(long long unsigned int)b->value);
|
98
|
+
printf("0x%016llx: %4llu\n", (long long unsigned int)k, (long long unsigned int)b->value);
|
98
99
|
#endif
|
99
100
|
} else {
|
100
101
|
slot_print(b->child, k, depth + 1);
|
data/ext/oj/circarray.c
CHANGED
@@ -3,10 +3,12 @@
|
|
3
3
|
|
4
4
|
#include "circarray.h"
|
5
5
|
|
6
|
-
|
6
|
+
#include "mem.h"
|
7
|
+
|
8
|
+
CircArray oj_circ_array_new(void) {
|
7
9
|
CircArray ca;
|
8
10
|
|
9
|
-
if (0 == (ca =
|
11
|
+
if (0 == (ca = OJ_R_ALLOC(struct _circArray))) {
|
10
12
|
rb_raise(rb_eNoMemError, "not enough memory\n");
|
11
13
|
}
|
12
14
|
ca->objs = ca->obj_array;
|
@@ -18,9 +20,9 @@ CircArray oj_circ_array_new() {
|
|
18
20
|
|
19
21
|
void oj_circ_array_free(CircArray ca) {
|
20
22
|
if (ca->objs != ca->obj_array) {
|
21
|
-
|
23
|
+
OJ_R_FREE(ca->objs);
|
22
24
|
}
|
23
|
-
|
25
|
+
OJ_R_FREE(ca);
|
24
26
|
}
|
25
27
|
|
26
28
|
void oj_circ_array_set(CircArray ca, VALUE obj, unsigned long id) {
|
@@ -31,12 +33,12 @@ void oj_circ_array_set(CircArray ca, VALUE obj, unsigned long id) {
|
|
31
33
|
unsigned long cnt = id + 512;
|
32
34
|
|
33
35
|
if (ca->objs == ca->obj_array) {
|
34
|
-
if (0 == (ca->objs =
|
36
|
+
if (0 == (ca->objs = OJ_R_ALLOC_N(VALUE, cnt))) {
|
35
37
|
rb_raise(rb_eNoMemError, "not enough memory\n");
|
36
38
|
}
|
37
39
|
memcpy(ca->objs, ca->obj_array, sizeof(VALUE) * ca->cnt);
|
38
40
|
} else {
|
39
|
-
|
41
|
+
OJ_R_REALLOC_N(ca->objs, VALUE, cnt);
|
40
42
|
}
|
41
43
|
ca->size = cnt;
|
42
44
|
}
|
data/ext/oj/circarray.h
CHANGED
@@ -9,10 +9,10 @@
|
|
9
9
|
|
10
10
|
typedef struct _circArray {
|
11
11
|
VALUE obj_array[1024];
|
12
|
-
VALUE
|
12
|
+
VALUE* objs;
|
13
13
|
unsigned long size; // allocated size or initial array size
|
14
14
|
unsigned long cnt;
|
15
|
-
}
|
15
|
+
}* CircArray;
|
16
16
|
|
17
17
|
extern CircArray oj_circ_array_new(void);
|
18
18
|
extern void oj_circ_array_free(CircArray ca);
|