oj 3.11.5 → 3.16.5

Sign up to get free protection for your applications and to get access to all the features.
Files changed (168) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1421 -0
  3. data/README.md +19 -5
  4. data/RELEASE_NOTES.md +61 -0
  5. data/ext/oj/buf.h +20 -6
  6. data/ext/oj/cache.c +329 -0
  7. data/ext/oj/cache.h +22 -0
  8. data/ext/oj/cache8.c +10 -9
  9. data/ext/oj/circarray.c +8 -6
  10. data/ext/oj/circarray.h +2 -2
  11. data/ext/oj/code.c +19 -33
  12. data/ext/oj/code.h +2 -2
  13. data/ext/oj/compat.c +27 -77
  14. data/ext/oj/custom.c +86 -179
  15. data/ext/oj/debug.c +126 -0
  16. data/ext/oj/dump.c +256 -249
  17. data/ext/oj/dump.h +26 -12
  18. data/ext/oj/dump_compat.c +565 -642
  19. data/ext/oj/dump_leaf.c +17 -63
  20. data/ext/oj/dump_object.c +65 -187
  21. data/ext/oj/dump_strict.c +27 -51
  22. data/ext/oj/encoder.c +43 -0
  23. data/ext/oj/err.c +2 -13
  24. data/ext/oj/err.h +24 -8
  25. data/ext/oj/extconf.rb +21 -6
  26. data/ext/oj/fast.c +149 -149
  27. data/ext/oj/intern.c +313 -0
  28. data/ext/oj/intern.h +22 -0
  29. data/ext/oj/mem.c +318 -0
  30. data/ext/oj/mem.h +53 -0
  31. data/ext/oj/mimic_json.c +121 -106
  32. data/ext/oj/object.c +85 -162
  33. data/ext/oj/odd.c +89 -67
  34. data/ext/oj/odd.h +15 -15
  35. data/ext/oj/oj.c +542 -411
  36. data/ext/oj/oj.h +99 -73
  37. data/ext/oj/parse.c +175 -187
  38. data/ext/oj/parse.h +26 -24
  39. data/ext/oj/parser.c +1600 -0
  40. data/ext/oj/parser.h +101 -0
  41. data/ext/oj/rails.c +112 -159
  42. data/ext/oj/rails.h +1 -1
  43. data/ext/oj/reader.c +11 -14
  44. data/ext/oj/reader.h +4 -2
  45. data/ext/oj/resolve.c +5 -24
  46. data/ext/oj/rxclass.c +7 -6
  47. data/ext/oj/rxclass.h +1 -1
  48. data/ext/oj/saj.c +22 -33
  49. data/ext/oj/saj2.c +584 -0
  50. data/ext/oj/saj2.h +23 -0
  51. data/ext/oj/scp.c +5 -28
  52. data/ext/oj/sparse.c +28 -72
  53. data/ext/oj/stream_writer.c +50 -40
  54. data/ext/oj/strict.c +56 -61
  55. data/ext/oj/string_writer.c +72 -39
  56. data/ext/oj/trace.h +31 -4
  57. data/ext/oj/usual.c +1218 -0
  58. data/ext/oj/usual.h +69 -0
  59. data/ext/oj/util.h +1 -1
  60. data/ext/oj/val_stack.c +14 -3
  61. data/ext/oj/val_stack.h +8 -7
  62. data/ext/oj/validate.c +46 -0
  63. data/ext/oj/wab.c +63 -88
  64. data/lib/oj/active_support_helper.rb +1 -3
  65. data/lib/oj/bag.rb +7 -1
  66. data/lib/oj/easy_hash.rb +4 -5
  67. data/lib/oj/error.rb +1 -2
  68. data/lib/oj/json.rb +162 -150
  69. data/lib/oj/mimic.rb +9 -7
  70. data/lib/oj/saj.rb +20 -6
  71. data/lib/oj/schandler.rb +5 -4
  72. data/lib/oj/state.rb +12 -8
  73. data/lib/oj/version.rb +1 -2
  74. data/lib/oj.rb +2 -0
  75. data/pages/Compatibility.md +1 -1
  76. data/pages/InstallOptions.md +20 -0
  77. data/pages/JsonGem.md +15 -0
  78. data/pages/Modes.md +8 -3
  79. data/pages/Options.md +43 -5
  80. data/pages/Parser.md +309 -0
  81. data/pages/Rails.md +14 -2
  82. data/test/_test_active.rb +8 -9
  83. data/test/_test_active_mimic.rb +7 -8
  84. data/test/_test_mimic_rails.rb +17 -20
  85. data/test/activerecord/result_test.rb +5 -6
  86. data/test/activesupport6/encoding_test.rb +63 -28
  87. data/test/{activesupport5 → activesupport7}/abstract_unit.rb +16 -12
  88. data/test/{activesupport5 → activesupport7}/decoding_test.rb +2 -10
  89. data/test/{activesupport5 → activesupport7}/encoding_test.rb +86 -50
  90. data/test/{activesupport5 → activesupport7}/encoding_test_cases.rb +6 -0
  91. data/test/{activesupport5 → activesupport7}/time_zone_test_helpers.rb +8 -0
  92. data/test/files.rb +15 -15
  93. data/test/foo.rb +16 -45
  94. data/test/helper.rb +11 -8
  95. data/test/isolated/shared.rb +3 -2
  96. data/test/json_gem/json_addition_test.rb +2 -2
  97. data/test/json_gem/json_common_interface_test.rb +8 -6
  98. data/test/json_gem/json_encoding_test.rb +0 -0
  99. data/test/json_gem/json_ext_parser_test.rb +1 -0
  100. data/test/json_gem/json_fixtures_test.rb +3 -2
  101. data/test/json_gem/json_generator_test.rb +56 -38
  102. data/test/json_gem/json_generic_object_test.rb +11 -11
  103. data/test/json_gem/json_parser_test.rb +54 -47
  104. data/test/json_gem/json_string_matching_test.rb +9 -9
  105. data/test/json_gem/test_helper.rb +7 -3
  106. data/test/mem.rb +34 -0
  107. data/test/perf.rb +22 -27
  108. data/test/perf_compat.rb +31 -33
  109. data/test/perf_dump.rb +50 -0
  110. data/test/perf_fast.rb +80 -82
  111. data/test/perf_file.rb +27 -29
  112. data/test/perf_object.rb +65 -69
  113. data/test/perf_once.rb +59 -0
  114. data/test/perf_parser.rb +183 -0
  115. data/test/perf_saj.rb +46 -54
  116. data/test/perf_scp.rb +58 -69
  117. data/test/perf_simple.rb +41 -39
  118. data/test/perf_strict.rb +74 -82
  119. data/test/perf_wab.rb +67 -69
  120. data/test/prec.rb +5 -5
  121. data/test/sample/change.rb +0 -1
  122. data/test/sample/dir.rb +0 -1
  123. data/test/sample/doc.rb +0 -1
  124. data/test/sample/file.rb +0 -1
  125. data/test/sample/group.rb +0 -1
  126. data/test/sample/hasprops.rb +0 -1
  127. data/test/sample/layer.rb +0 -1
  128. data/test/sample/rect.rb +0 -1
  129. data/test/sample/shape.rb +0 -1
  130. data/test/sample/text.rb +0 -1
  131. data/test/sample.rb +16 -16
  132. data/test/sample_json.rb +8 -8
  133. data/test/test_compat.rb +95 -43
  134. data/test/test_custom.rb +73 -51
  135. data/test/test_debian.rb +7 -10
  136. data/test/test_fast.rb +135 -79
  137. data/test/test_file.rb +41 -30
  138. data/test/test_gc.rb +16 -5
  139. data/test/test_generate.rb +5 -5
  140. data/test/test_hash.rb +5 -5
  141. data/test/test_integer_range.rb +9 -9
  142. data/test/test_null.rb +20 -20
  143. data/test/test_object.rb +99 -96
  144. data/test/test_parser.rb +11 -0
  145. data/test/test_parser_debug.rb +27 -0
  146. data/test/test_parser_saj.rb +337 -0
  147. data/test/test_parser_usual.rb +251 -0
  148. data/test/test_rails.rb +2 -2
  149. data/test/test_saj.rb +10 -8
  150. data/test/test_scp.rb +37 -39
  151. data/test/test_strict.rb +40 -32
  152. data/test/test_various.rb +165 -84
  153. data/test/test_wab.rb +48 -44
  154. data/test/test_writer.rb +47 -47
  155. data/test/tests.rb +13 -5
  156. data/test/tests_mimic.rb +12 -3
  157. data/test/tests_mimic_addition.rb +12 -3
  158. metadata +74 -128
  159. data/ext/oj/hash.c +0 -131
  160. data/ext/oj/hash.h +0 -19
  161. data/ext/oj/hash_test.c +0 -491
  162. data/test/activesupport4/decoding_test.rb +0 -108
  163. data/test/activesupport4/encoding_test.rb +0 -531
  164. data/test/activesupport4/test_helper.rb +0 -41
  165. data/test/activesupport5/test_helper.rb +0 -72
  166. data/test/bar.rb +0 -35
  167. data/test/baz.rb +0 -16
  168. data/test/zoo.rb +0 -13
data/README.md CHANGED
@@ -1,11 +1,13 @@
1
1
  # [![{}j](http://www.ohler.com/dev/images/oj_comet_64.svg)](http://www.ohler.com/oj) gem
2
2
 
3
- [![Build Status](https://img.shields.io/travis/ohler55/oj/master.svg?logo=travis)](http://travis-ci.org/ohler55/oj?branch=master) ![Gem](https://img.shields.io/gem/v/oj.svg) ![Gem](https://img.shields.io/gem/dt/oj.svg) [![SemVer compatibility](https://api.dependabot.com/badges/compatibility_score?dependency-name=oj&package-manager=bundler&version-scheme=semver)](https://dependabot.com/compatibility-score.html?dependency-name=oj&package-manager=bundler&version-scheme=semver) [![TideLift](https://tidelift.com/badges/github/ohler55/oj)](https://tidelift.com/subscription/pkg/rubygems-oj?utm_source=rubygems-oj&utm_medium=referral&utm_campaign=readme)
3
+ [![CI](https://github.com/ohler55/oj/actions/workflows/CI.yml/badge.svg)](https://github.com/ohler55/oj/actions/workflows/CI.yml)
4
+ ![Gem](https://img.shields.io/gem/v/oj.svg)
5
+ ![Gem](https://img.shields.io/gem/dt/oj.svg)
6
+ [![TideLift](https://tidelift.com/badges/github/ohler55/oj)](https://tidelift.com/subscription/pkg/rubygems-oj?utm_source=rubygems-oj&utm_medium=referral&utm_campaign=readme)
4
7
 
5
8
  A *fast* JSON parser and Object marshaller as a Ruby gem.
6
9
 
7
- Version 3.0 is out! 3.0 provides better json gem and Rails compatibility. It
8
- also provides additional optimization options.
10
+ Version 3.13 is out with a much faster parser (`Oj::Parser`) and option isolation.
9
11
 
10
12
  ## Using
11
13
 
@@ -40,6 +42,15 @@ or in Bundler:
40
42
  gem 'oj'
41
43
  ```
42
44
 
45
+ ## Rails and json quickstart
46
+
47
+ See the Quickstart sections of the [Rails](pages/Rails.md) and [json](pages/JsonGem.md) docs.
48
+
49
+ ## multi_json
50
+
51
+ Code which uses [multi_json](https://github.com/intridea/multi_json)
52
+ will automatically prefer Oj if it is installed.
53
+
43
54
  ## Support
44
55
 
45
56
  [Get supported Oj with a Tidelift Subscription.](https://tidelift.com/subscription/pkg/rubygems-oj?utm_source=rubygems-oj&utm_medium=referral&utm_campaign=readme) Security updates are [supported](https://tidelift.com/security).
@@ -50,7 +61,7 @@ For more details on options, modes, advanced features, and more follow these
50
61
  links.
51
62
 
52
63
  - [{file:Options.md}](pages/Options.md) for parse and dump options.
53
- - [{file:Modes.md}](pages/Modes.md) for details on modes for strict JSON compliance, mimicing the JSON gem, and mimicing Rails and ActiveSupport behavior.
64
+ - [{file:Modes.md}](pages/Modes.md) for details on modes for strict JSON compliance, mimicking the JSON gem, and mimicking Rails and ActiveSupport behavior.
54
65
  - [{file:JsonGem.md}](pages/JsonGem.md) includes more details on json gem compatibility and use.
55
66
  - [{file:Rails.md}](pages/Rails.md) includes more details on Rails and ActiveSupport compatibility and use.
56
67
  - [{file:Custom.md}](pages/Custom.md) includes more details on Custom mode.
@@ -58,10 +69,11 @@ links.
58
69
  - [{file:Compatibility.md}](pages/Compatibility.md) lists current compatibility with Rubys and Rails.
59
70
  - [{file:Advanced.md}](pages/Advanced.md) for fast parser and marshalling features.
60
71
  - [{file:Security.md}](pages/Security.md) for security considerations.
72
+ - [{file:InstallOptions.md}](pages/InstallOptions.md) for install option.
61
73
 
62
74
  ## Releases
63
75
 
64
- See [{file:CHANGELOG.md}](CHANGELOG.md)
76
+ See [{file:CHANGELOG.md}](CHANGELOG.md) and [{file:RELEASE_NOTES.md}](RELEASE_NOTES.md)
65
77
 
66
78
  ## Links
67
79
 
@@ -97,6 +109,8 @@ Follow [@peterohler on Twitter](http://twitter.com/peterohler) for announcements
97
109
 
98
110
  - *Agoo-C, a high performance C web server supporting GraphQL on GitHub*: https://github.com/ohler55/agoo-c
99
111
 
112
+ - *oj-introspect, an example of creating an Oj parser extension in C*: https://github.com/meinac/oj-introspect
113
+
100
114
  #### Contributing
101
115
 
102
116
  + Provide a Pull Request off the `develop` branch.
data/RELEASE_NOTES.md ADDED
@@ -0,0 +1,61 @@
1
+ # RELEASE NOTES
2
+
3
+ The release notes here are organized by release. For a list of changes
4
+ see the See [{file:CHANGELOG.md}](CHANGELOG.md) file. In this file are
5
+ the steps to take to aid in keeping things rolling after updating to
6
+ the latest version.
7
+
8
+ ## 3.13.7
9
+
10
+ The default for JSON when mimicked by Oj is now to set
11
+ `:allow_invalid_unicode`. To change that behavior JSON.load, set that
12
+ option to false.
13
+
14
+ ## 3.13.x
15
+
16
+ This release included a new cache that performs better than the
17
+ earlier cache and a new high performance parser.
18
+
19
+ ### Cache
20
+
21
+ The new cache includes a least recently used expiration to reduce
22
+ memory use. The cache is also self adjusting and will expand as needed
23
+ for better performance. It also handles Hash keys and string values
24
+ with two options, `:cache_keys`, a boolean and `:cache_str` an
25
+ integer. The `:cache_str` if set to more than zero is the limit for
26
+ the length of string values to cache. The maximum value is 35 which
27
+ allows strings up to 34 bytes to be cached.
28
+
29
+ One interesting aspect of the cache is not so much the string caching
30
+ which performs similar to the Ruby intern functions but the caching of
31
+ symbols and object attribute names. There is a significant gain for
32
+ symbols and object attributes.
33
+
34
+ If the cache is not desired then setting the default options to turn
35
+ it off can be done with this line:
36
+
37
+ ``` ruby
38
+ Oj.default_options = { cache_keys: false, cache_str: 0 }
39
+ ```
40
+
41
+ ### Oj::Parser
42
+
43
+ The new parser uses a different core that follows the approach taken
44
+ by [OjC](https://github.com/ohler55/ojc) and
45
+ [OjG](https://github.com/ohler55/ojg). It also takes advantage of the
46
+ bulk Array and Hash functions. Another issue the new parser addresses
47
+ is option management. Instead of a single global default_options each
48
+ parser instance maintains it's own options.
49
+
50
+ There is a price to be paid when using the Oj::Parser. The API is not
51
+ the same the older parser. A single parser can only be used in a
52
+ single thread. This allows reuse of internal buffers for additional
53
+ improvements in performance.
54
+
55
+ The performane advantage of the Oj::Parse is that it is more than 3
56
+ times faster than the Oj::compat_load call and 6 times faster than the
57
+ JSON gem.
58
+
59
+ ### Dump Performance
60
+
61
+ Thanks to Watson1978 Oj.dump also received a speed boost.
data/ext/oj/buf.h CHANGED
@@ -4,6 +4,7 @@
4
4
  #ifndef OJ_BUF_H
5
5
  #define OJ_BUF_H
6
6
 
7
+ #include "mem.h"
7
8
  #include "ruby.h"
8
9
 
9
10
  typedef struct _buf {
@@ -11,7 +12,7 @@ typedef struct _buf {
11
12
  char *end;
12
13
  char *tail;
13
14
  char base[1024];
14
- } * Buf;
15
+ } *Buf;
15
16
 
16
17
  inline static void buf_init(Buf buf) {
17
18
  buf->head = buf->base;
@@ -19,9 +20,13 @@ inline static void buf_init(Buf buf) {
19
20
  buf->tail = buf->head;
20
21
  }
21
22
 
23
+ inline static void buf_reset(Buf buf) {
24
+ buf->tail = buf->head;
25
+ }
26
+
22
27
  inline static void buf_cleanup(Buf buf) {
23
28
  if (buf->base != buf->head) {
24
- xfree(buf->head);
29
+ OJ_R_FREE(buf->head);
25
30
  }
26
31
  }
27
32
 
@@ -29,17 +34,26 @@ inline static size_t buf_len(Buf buf) {
29
34
  return buf->tail - buf->head;
30
35
  }
31
36
 
37
+ inline static const char *buf_str(Buf buf) {
38
+ *buf->tail = '\0';
39
+ return buf->head;
40
+ }
41
+
32
42
  inline static void buf_append_string(Buf buf, const char *s, size_t slen) {
43
+ if (0 == slen) {
44
+ return;
45
+ }
46
+
33
47
  if (buf->end <= buf->tail + slen) {
34
48
  size_t len = buf->end - buf->head;
35
49
  size_t toff = buf->tail - buf->head;
36
50
  size_t new_len = len + slen + len / 2;
37
51
 
38
52
  if (buf->base == buf->head) {
39
- buf->head = ALLOC_N(char, new_len);
53
+ buf->head = OJ_R_ALLOC_N(char, new_len);
40
54
  memcpy(buf->head, buf->base, len);
41
55
  } else {
42
- REALLOC_N(buf->head, char, new_len);
56
+ OJ_R_REALLOC_N(buf->head, char, new_len);
43
57
  }
44
58
  buf->tail = buf->head + toff;
45
59
  buf->end = buf->head + new_len - 1;
@@ -55,10 +69,10 @@ inline static void buf_append(Buf buf, char c) {
55
69
  size_t new_len = len + len / 2;
56
70
 
57
71
  if (buf->base == buf->head) {
58
- buf->head = ALLOC_N(char, new_len);
72
+ buf->head = OJ_R_ALLOC_N(char, new_len);
59
73
  memcpy(buf->head, buf->base, len);
60
74
  } else {
61
- REALLOC_N(buf->head, char, new_len);
75
+ OJ_R_REALLOC_N(buf->head, char, new_len);
62
76
  }
63
77
  buf->tail = buf->head + toff;
64
78
  buf->end = buf->head + new_len - 1;
data/ext/oj/cache.c ADDED
@@ -0,0 +1,329 @@
1
+ // Copyright (c) 2011, 2021 Peter Ohler. All rights reserved.
2
+ // Licensed under the MIT License. See LICENSE file in the project root for license details.
3
+
4
+ #if HAVE_PTHREAD_MUTEX_INIT
5
+ #include <pthread.h>
6
+ #endif
7
+ #include <stdlib.h>
8
+
9
+ #include "cache.h"
10
+ #include "mem.h"
11
+
12
+ // The stdlib calloc, realloc, and free are used instead of the Ruby ALLOC,
13
+ // ALLOC_N, REALLOC, and xfree since the later could trigger a GC which will
14
+ // either corrupt memory or if the mark function locks will deadlock.
15
+
16
+ #define REHASH_LIMIT 4
17
+ #define MIN_SHIFT 8
18
+ #define REUSE_MAX 8192
19
+
20
+ #if HAVE_PTHREAD_MUTEX_INIT
21
+ #define CACHE_LOCK(c) pthread_mutex_lock(&((c)->mutex))
22
+ #define CACHE_UNLOCK(c) pthread_mutex_unlock(&((c)->mutex))
23
+ #else
24
+ #define CACHE_LOCK(c) rb_mutex_lock((c)->mutex)
25
+ #define CACHE_UNLOCK(c) rb_mutex_unlock((c)->mutex)
26
+ #endif
27
+
28
+ // almost the Murmur hash algorithm
29
+ #define M 0x5bd1e995
30
+
31
+ typedef struct _slot {
32
+ struct _slot *next;
33
+ VALUE val;
34
+ uint64_t hash;
35
+ volatile uint32_t use_cnt;
36
+ uint8_t klen;
37
+ char key[CACHE_MAX_KEY];
38
+ } *Slot;
39
+
40
+ typedef struct _cache {
41
+ volatile Slot *slots;
42
+ volatile size_t cnt;
43
+ VALUE (*form)(const char *str, size_t len);
44
+ uint64_t size;
45
+ uint64_t mask;
46
+ VALUE (*intern)(struct _cache *c, const char *key, size_t len);
47
+ volatile Slot reuse;
48
+ size_t rcnt;
49
+ #if HAVE_PTHREAD_MUTEX_INIT
50
+ pthread_mutex_t mutex;
51
+ #else
52
+ VALUE mutex;
53
+ #endif
54
+ uint8_t xrate;
55
+ bool mark;
56
+ } *Cache;
57
+
58
+ void cache_set_form(Cache c, VALUE (*form)(const char *str, size_t len)) {
59
+ c->form = form;
60
+ }
61
+
62
+ static uint64_t hash_calc(const uint8_t *key, size_t len) {
63
+ const uint8_t *end = key + len;
64
+ const uint8_t *endless = key + (len & 0xFFFFFFFC);
65
+ uint64_t h = (uint64_t)len;
66
+ uint64_t k;
67
+
68
+ while (key < endless) {
69
+ k = (uint64_t)*key++;
70
+ k |= (uint64_t)*key++ << 8;
71
+ k |= (uint64_t)*key++ << 16;
72
+ k |= (uint64_t)*key++ << 24;
73
+
74
+ k *= M;
75
+ k ^= k >> 24;
76
+ h *= M;
77
+ h ^= k * M;
78
+ }
79
+ if (1 < end - key) {
80
+ uint16_t k16 = (uint16_t)*key++;
81
+
82
+ k16 |= (uint16_t)*key++ << 8;
83
+ h ^= k16 << 8;
84
+ }
85
+ if (key < end) {
86
+ h ^= *key;
87
+ }
88
+ h *= M;
89
+ h ^= h >> 13;
90
+ h *= M;
91
+ h ^= h >> 15;
92
+
93
+ return h;
94
+ }
95
+
96
+ static void rehash(Cache c) {
97
+ uint64_t osize;
98
+ Slot *end;
99
+ Slot *sp;
100
+
101
+ osize = c->size;
102
+ c->size = osize * 4;
103
+ c->mask = c->size - 1;
104
+ c->slots = OJ_REALLOC((void *)c->slots, sizeof(Slot) * c->size);
105
+ memset((Slot *)c->slots + osize, 0, sizeof(Slot) * osize * 3);
106
+ end = (Slot *)c->slots + osize;
107
+ for (sp = (Slot *)c->slots; sp < end; sp++) {
108
+ Slot s = *sp;
109
+ Slot next = NULL;
110
+
111
+ *sp = NULL;
112
+ for (; NULL != s; s = next) {
113
+ uint64_t h = s->hash & c->mask;
114
+ Slot *bucket = (Slot *)c->slots + h;
115
+
116
+ next = s->next;
117
+ s->next = *bucket;
118
+ *bucket = s;
119
+ }
120
+ }
121
+ }
122
+
123
+ static VALUE lockless_intern(Cache c, const char *key, size_t len) {
124
+ uint64_t h = hash_calc((const uint8_t *)key, len);
125
+ Slot *bucket = (Slot *)c->slots + (h & c->mask);
126
+ Slot b;
127
+ volatile VALUE rkey;
128
+
129
+ while (REUSE_MAX < c->rcnt) {
130
+ if (NULL != (b = c->reuse)) {
131
+ c->reuse = b->next;
132
+ OJ_FREE(b);
133
+ c->rcnt--;
134
+ } else {
135
+ // An accounting error occured somewhere so correct it.
136
+ c->rcnt = 0;
137
+ }
138
+ }
139
+ for (b = *bucket; NULL != b; b = b->next) {
140
+ if ((uint8_t)len == b->klen && 0 == strncmp(b->key, key, len)) {
141
+ b->use_cnt += 16;
142
+ return b->val;
143
+ }
144
+ }
145
+ rkey = c->form(key, len);
146
+ if (NULL == (b = c->reuse)) {
147
+ b = OJ_CALLOC(1, sizeof(struct _slot));
148
+ } else {
149
+ c->reuse = b->next;
150
+ c->rcnt--;
151
+ }
152
+ b->hash = h;
153
+ memcpy(b->key, key, len);
154
+ b->klen = (uint8_t)len;
155
+ b->key[len] = '\0';
156
+ b->val = rkey;
157
+ b->use_cnt = 4;
158
+ b->next = *bucket;
159
+ *bucket = b;
160
+ c->cnt++; // Don't worry about wrapping. Worse case is the entry is removed and recreated.
161
+ if (REHASH_LIMIT < c->cnt / c->size) {
162
+ rehash(c);
163
+ }
164
+ return rkey;
165
+ }
166
+
167
+ static VALUE locking_intern(Cache c, const char *key, size_t len) {
168
+ uint64_t h;
169
+ Slot *bucket;
170
+ Slot b;
171
+ uint64_t old_size;
172
+ volatile VALUE rkey;
173
+
174
+ CACHE_LOCK(c);
175
+ while (REUSE_MAX < c->rcnt) {
176
+ if (NULL != (b = c->reuse)) {
177
+ c->reuse = b->next;
178
+ OJ_FREE(b);
179
+ c->rcnt--;
180
+ } else {
181
+ // An accounting error occured somewhere so correct it.
182
+ c->rcnt = 0;
183
+ }
184
+ }
185
+ h = hash_calc((const uint8_t *)key, len);
186
+ bucket = (Slot *)c->slots + (h & c->mask);
187
+ for (b = *bucket; NULL != b; b = b->next) {
188
+ if ((uint8_t)len == b->klen && 0 == strncmp(b->key, key, len)) {
189
+ b->use_cnt += 4;
190
+ CACHE_UNLOCK(c);
191
+
192
+ return b->val;
193
+ }
194
+ }
195
+ old_size = c->size;
196
+ // The creation of a new value may trigger a GC which be a problem if the
197
+ // cache is locked so make sure it is unlocked for the key value creation.
198
+ if (NULL != (b = c->reuse)) {
199
+ c->reuse = b->next;
200
+ c->rcnt--;
201
+ }
202
+ CACHE_UNLOCK(c);
203
+ if (NULL == b) {
204
+ b = OJ_CALLOC(1, sizeof(struct _slot));
205
+ }
206
+ rkey = c->form(key, len);
207
+ b->hash = h;
208
+ memcpy(b->key, key, len);
209
+ b->klen = (uint8_t)len;
210
+ b->key[len] = '\0';
211
+ b->val = rkey;
212
+ b->use_cnt = 16;
213
+
214
+ // Lock again to add the new entry.
215
+ CACHE_LOCK(c);
216
+ if (old_size != c->size) {
217
+ h = hash_calc((const uint8_t *)key, len);
218
+ bucket = (Slot *)c->slots + (h & c->mask);
219
+ }
220
+ b->next = *bucket;
221
+ *bucket = b;
222
+ c->cnt++; // Don't worry about wrapping. Worse case is the entry is removed and recreated.
223
+ if (REHASH_LIMIT < c->cnt / c->size) {
224
+ rehash(c);
225
+ }
226
+ CACHE_UNLOCK(c);
227
+
228
+ return rkey;
229
+ }
230
+
231
+ Cache cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark, bool locking) {
232
+ Cache c = OJ_CALLOC(1, sizeof(struct _cache));
233
+ int shift = 0;
234
+
235
+ for (; REHASH_LIMIT < size; size /= 2, shift++) {
236
+ }
237
+ if (shift < MIN_SHIFT) {
238
+ shift = MIN_SHIFT;
239
+ }
240
+ #if HAVE_PTHREAD_MUTEX_INIT
241
+ pthread_mutex_init(&c->mutex, NULL);
242
+ #else
243
+ c->mutex = rb_mutex_new();
244
+ #endif
245
+ c->size = 1 << shift;
246
+ c->mask = c->size - 1;
247
+ c->slots = OJ_CALLOC(c->size, sizeof(Slot));
248
+ c->form = form;
249
+ c->xrate = 1; // low
250
+ c->mark = mark;
251
+ if (locking) {
252
+ c->intern = locking_intern;
253
+ } else {
254
+ c->intern = lockless_intern;
255
+ }
256
+ return c;
257
+ }
258
+
259
+ void cache_set_expunge_rate(Cache c, int rate) {
260
+ c->xrate = (uint8_t)rate;
261
+ }
262
+
263
+ void cache_free(void *data) {
264
+ Cache c = (Cache)data;
265
+ uint64_t i;
266
+
267
+ for (i = 0; i < c->size; i++) {
268
+ Slot next;
269
+ Slot s;
270
+
271
+ for (s = c->slots[i]; NULL != s; s = next) {
272
+ next = s->next;
273
+ OJ_FREE(s);
274
+ }
275
+ }
276
+ OJ_FREE((void *)c->slots);
277
+ OJ_FREE(c);
278
+ }
279
+
280
+ void cache_mark(void *data) {
281
+ Cache c = (Cache)data;
282
+ uint64_t i;
283
+
284
+ #if !HAVE_PTHREAD_MUTEX_INIT
285
+ rb_gc_mark(c->mutex);
286
+ #endif
287
+ if (0 == c->cnt) {
288
+ return;
289
+ }
290
+ for (i = 0; i < c->size; i++) {
291
+ Slot s;
292
+ Slot prev = NULL;
293
+ Slot next;
294
+
295
+ for (s = c->slots[i]; NULL != s; s = next) {
296
+ next = s->next;
297
+ if (0 == s->use_cnt) {
298
+ if (NULL == prev) {
299
+ c->slots[i] = next;
300
+ } else {
301
+ prev->next = next;
302
+ }
303
+ c->cnt--;
304
+ s->next = c->reuse;
305
+ c->reuse = s;
306
+ c->rcnt++;
307
+ continue;
308
+ }
309
+ switch (c->xrate) {
310
+ case 0: break;
311
+ case 2: s->use_cnt -= 2; break;
312
+ case 3: s->use_cnt /= 2; break;
313
+ default: s->use_cnt--; break;
314
+ }
315
+ if (c->mark) {
316
+ rb_gc_mark(s->val);
317
+ }
318
+ prev = s;
319
+ }
320
+ }
321
+ }
322
+
323
+ VALUE
324
+ cache_intern(Cache c, const char *key, size_t len) {
325
+ if (CACHE_MAX_KEY <= len) {
326
+ return c->form(key, len);
327
+ }
328
+ return c->intern(c, key, len);
329
+ }
data/ext/oj/cache.h ADDED
@@ -0,0 +1,22 @@
1
+ // Copyright (c) 2021 Peter Ohler. All rights reserved.
2
+ // Licensed under the MIT License. See LICENSE file in the project root for license details.
3
+
4
+ #ifndef CACHE_H
5
+ #define CACHE_H
6
+
7
+ #include <ruby.h>
8
+ #include <stdbool.h>
9
+
10
+ #define CACHE_MAX_KEY 35
11
+
12
+ struct _cache;
13
+ typedef struct _cache *Cache;
14
+
15
+ extern struct _cache *cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark, bool locking);
16
+ extern void cache_free(void *data);
17
+ extern void cache_mark(void *data);
18
+ extern void cache_set_form(struct _cache *c, VALUE (*form)(const char *str, size_t len));
19
+ extern VALUE cache_intern(struct _cache *c, const char *key, size_t len);
20
+ extern void cache_set_expunge_rate(struct _cache *c, int rate);
21
+
22
+ #endif /* CACHE_H */
data/ext/oj/cache8.c CHANGED
@@ -9,6 +9,7 @@
9
9
  #include <stdlib.h>
10
10
  #include <string.h>
11
11
 
12
+ #include "mem.h"
12
13
  #include "ruby.h"
13
14
 
14
15
  #define BITS 4
@@ -32,16 +33,18 @@ void oj_cache8_new(Cache8 *cache) {
32
33
  Bucket *b;
33
34
  int i;
34
35
 
35
- *cache = ALLOC(struct _cache8);
36
+ *cache = OJ_R_ALLOC(struct _cache8);
36
37
  for (i = SLOT_CNT, b = (*cache)->buckets; 0 < i; i--, b++) {
37
38
  b->value = 0;
38
39
  }
39
40
  }
40
41
 
41
- void oj_cache8_delete(Cache8 cache) { cache8_delete(cache, 0); }
42
+ void oj_cache8_delete(Cache8 cache) {
43
+ cache8_delete(cache, 0);
44
+ }
42
45
 
43
46
  static void cache8_delete(Cache8 cache, int depth) {
44
- Bucket * b;
47
+ Bucket *b;
45
48
  unsigned int i;
46
49
 
47
50
  for (i = 0, b = cache->buckets; i < SLOT_CNT; i++, b++) {
@@ -51,7 +54,7 @@ static void cache8_delete(Cache8 cache, int depth) {
51
54
  }
52
55
  }
53
56
  }
54
- xfree(cache);
57
+ OJ_R_FREE(cache);
55
58
  }
56
59
 
57
60
  slot_t oj_cache8_get(Cache8 cache, sid_t key, slot_t **slot) {
@@ -79,7 +82,7 @@ void oj_cache8_print(Cache8 cache) {
79
82
  }
80
83
 
81
84
  static void slot_print(Cache8 c, sid_t key, unsigned int depth) {
82
- Bucket * b;
85
+ Bucket *b;
83
86
  unsigned int i;
84
87
  sid_t k8 = (sid_t)key;
85
88
  sid_t k;
@@ -90,11 +93,9 @@ static void slot_print(Cache8 c, sid_t key, unsigned int depth) {
90
93
  /*printf("*** key: 0x%016llx depth: %u i: %u\n", k, depth, i); */
91
94
  if (DEPTH - 1 == depth) {
92
95
  #if IS_WINDOWS
93
- printf("0x%016lx: %4lu\n", (long unsigned int)k,
94
- (long unsigned int)b->value);
96
+ printf("0x%016lx: %4lu\n", (long unsigned int)k, (long unsigned int)b->value);
95
97
  #else
96
- printf("0x%016llx: %4llu\n", (long long unsigned int)k,
97
- (long long unsigned int)b->value);
98
+ printf("0x%016llx: %4llu\n", (long long unsigned int)k, (long long unsigned int)b->value);
98
99
  #endif
99
100
  } else {
100
101
  slot_print(b->child, k, depth + 1);
data/ext/oj/circarray.c CHANGED
@@ -3,10 +3,12 @@
3
3
 
4
4
  #include "circarray.h"
5
5
 
6
- CircArray oj_circ_array_new() {
6
+ #include "mem.h"
7
+
8
+ CircArray oj_circ_array_new(void) {
7
9
  CircArray ca;
8
10
 
9
- if (0 == (ca = ALLOC(struct _circArray))) {
11
+ if (0 == (ca = OJ_R_ALLOC(struct _circArray))) {
10
12
  rb_raise(rb_eNoMemError, "not enough memory\n");
11
13
  }
12
14
  ca->objs = ca->obj_array;
@@ -18,9 +20,9 @@ CircArray oj_circ_array_new() {
18
20
 
19
21
  void oj_circ_array_free(CircArray ca) {
20
22
  if (ca->objs != ca->obj_array) {
21
- xfree(ca->objs);
23
+ OJ_R_FREE(ca->objs);
22
24
  }
23
- xfree(ca);
25
+ OJ_R_FREE(ca);
24
26
  }
25
27
 
26
28
  void oj_circ_array_set(CircArray ca, VALUE obj, unsigned long id) {
@@ -31,12 +33,12 @@ void oj_circ_array_set(CircArray ca, VALUE obj, unsigned long id) {
31
33
  unsigned long cnt = id + 512;
32
34
 
33
35
  if (ca->objs == ca->obj_array) {
34
- if (0 == (ca->objs = ALLOC_N(VALUE, cnt))) {
36
+ if (0 == (ca->objs = OJ_R_ALLOC_N(VALUE, cnt))) {
35
37
  rb_raise(rb_eNoMemError, "not enough memory\n");
36
38
  }
37
39
  memcpy(ca->objs, ca->obj_array, sizeof(VALUE) * ca->cnt);
38
40
  } else {
39
- REALLOC_N(ca->objs, VALUE, cnt);
41
+ OJ_R_REALLOC_N(ca->objs, VALUE, cnt);
40
42
  }
41
43
  ca->size = cnt;
42
44
  }
data/ext/oj/circarray.h CHANGED
@@ -9,10 +9,10 @@
9
9
 
10
10
  typedef struct _circArray {
11
11
  VALUE obj_array[1024];
12
- VALUE * objs;
12
+ VALUE* objs;
13
13
  unsigned long size; // allocated size or initial array size
14
14
  unsigned long cnt;
15
- } * CircArray;
15
+ }* CircArray;
16
16
 
17
17
  extern CircArray oj_circ_array_new(void);
18
18
  extern void oj_circ_array_free(CircArray ca);