oj 3.11.5 → 3.16.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1421 -0
  3. data/README.md +19 -5
  4. data/RELEASE_NOTES.md +61 -0
  5. data/ext/oj/buf.h +20 -6
  6. data/ext/oj/cache.c +329 -0
  7. data/ext/oj/cache.h +22 -0
  8. data/ext/oj/cache8.c +10 -9
  9. data/ext/oj/circarray.c +8 -6
  10. data/ext/oj/circarray.h +2 -2
  11. data/ext/oj/code.c +19 -33
  12. data/ext/oj/code.h +2 -2
  13. data/ext/oj/compat.c +27 -77
  14. data/ext/oj/custom.c +86 -179
  15. data/ext/oj/debug.c +126 -0
  16. data/ext/oj/dump.c +256 -249
  17. data/ext/oj/dump.h +26 -12
  18. data/ext/oj/dump_compat.c +565 -642
  19. data/ext/oj/dump_leaf.c +17 -63
  20. data/ext/oj/dump_object.c +65 -187
  21. data/ext/oj/dump_strict.c +27 -51
  22. data/ext/oj/encoder.c +43 -0
  23. data/ext/oj/err.c +2 -13
  24. data/ext/oj/err.h +24 -8
  25. data/ext/oj/extconf.rb +21 -6
  26. data/ext/oj/fast.c +149 -149
  27. data/ext/oj/intern.c +313 -0
  28. data/ext/oj/intern.h +22 -0
  29. data/ext/oj/mem.c +318 -0
  30. data/ext/oj/mem.h +53 -0
  31. data/ext/oj/mimic_json.c +121 -106
  32. data/ext/oj/object.c +85 -162
  33. data/ext/oj/odd.c +89 -67
  34. data/ext/oj/odd.h +15 -15
  35. data/ext/oj/oj.c +542 -411
  36. data/ext/oj/oj.h +99 -73
  37. data/ext/oj/parse.c +175 -187
  38. data/ext/oj/parse.h +26 -24
  39. data/ext/oj/parser.c +1600 -0
  40. data/ext/oj/parser.h +101 -0
  41. data/ext/oj/rails.c +112 -159
  42. data/ext/oj/rails.h +1 -1
  43. data/ext/oj/reader.c +11 -14
  44. data/ext/oj/reader.h +4 -2
  45. data/ext/oj/resolve.c +5 -24
  46. data/ext/oj/rxclass.c +7 -6
  47. data/ext/oj/rxclass.h +1 -1
  48. data/ext/oj/saj.c +22 -33
  49. data/ext/oj/saj2.c +584 -0
  50. data/ext/oj/saj2.h +23 -0
  51. data/ext/oj/scp.c +5 -28
  52. data/ext/oj/sparse.c +28 -72
  53. data/ext/oj/stream_writer.c +50 -40
  54. data/ext/oj/strict.c +56 -61
  55. data/ext/oj/string_writer.c +72 -39
  56. data/ext/oj/trace.h +31 -4
  57. data/ext/oj/usual.c +1218 -0
  58. data/ext/oj/usual.h +69 -0
  59. data/ext/oj/util.h +1 -1
  60. data/ext/oj/val_stack.c +14 -3
  61. data/ext/oj/val_stack.h +8 -7
  62. data/ext/oj/validate.c +46 -0
  63. data/ext/oj/wab.c +63 -88
  64. data/lib/oj/active_support_helper.rb +1 -3
  65. data/lib/oj/bag.rb +7 -1
  66. data/lib/oj/easy_hash.rb +4 -5
  67. data/lib/oj/error.rb +1 -2
  68. data/lib/oj/json.rb +162 -150
  69. data/lib/oj/mimic.rb +9 -7
  70. data/lib/oj/saj.rb +20 -6
  71. data/lib/oj/schandler.rb +5 -4
  72. data/lib/oj/state.rb +12 -8
  73. data/lib/oj/version.rb +1 -2
  74. data/lib/oj.rb +2 -0
  75. data/pages/Compatibility.md +1 -1
  76. data/pages/InstallOptions.md +20 -0
  77. data/pages/JsonGem.md +15 -0
  78. data/pages/Modes.md +8 -3
  79. data/pages/Options.md +43 -5
  80. data/pages/Parser.md +309 -0
  81. data/pages/Rails.md +14 -2
  82. data/test/_test_active.rb +8 -9
  83. data/test/_test_active_mimic.rb +7 -8
  84. data/test/_test_mimic_rails.rb +17 -20
  85. data/test/activerecord/result_test.rb +5 -6
  86. data/test/activesupport6/encoding_test.rb +63 -28
  87. data/test/{activesupport5 → activesupport7}/abstract_unit.rb +16 -12
  88. data/test/{activesupport5 → activesupport7}/decoding_test.rb +2 -10
  89. data/test/{activesupport5 → activesupport7}/encoding_test.rb +86 -50
  90. data/test/{activesupport5 → activesupport7}/encoding_test_cases.rb +6 -0
  91. data/test/{activesupport5 → activesupport7}/time_zone_test_helpers.rb +8 -0
  92. data/test/files.rb +15 -15
  93. data/test/foo.rb +16 -45
  94. data/test/helper.rb +11 -8
  95. data/test/isolated/shared.rb +3 -2
  96. data/test/json_gem/json_addition_test.rb +2 -2
  97. data/test/json_gem/json_common_interface_test.rb +8 -6
  98. data/test/json_gem/json_encoding_test.rb +0 -0
  99. data/test/json_gem/json_ext_parser_test.rb +1 -0
  100. data/test/json_gem/json_fixtures_test.rb +3 -2
  101. data/test/json_gem/json_generator_test.rb +56 -38
  102. data/test/json_gem/json_generic_object_test.rb +11 -11
  103. data/test/json_gem/json_parser_test.rb +54 -47
  104. data/test/json_gem/json_string_matching_test.rb +9 -9
  105. data/test/json_gem/test_helper.rb +7 -3
  106. data/test/mem.rb +34 -0
  107. data/test/perf.rb +22 -27
  108. data/test/perf_compat.rb +31 -33
  109. data/test/perf_dump.rb +50 -0
  110. data/test/perf_fast.rb +80 -82
  111. data/test/perf_file.rb +27 -29
  112. data/test/perf_object.rb +65 -69
  113. data/test/perf_once.rb +59 -0
  114. data/test/perf_parser.rb +183 -0
  115. data/test/perf_saj.rb +46 -54
  116. data/test/perf_scp.rb +58 -69
  117. data/test/perf_simple.rb +41 -39
  118. data/test/perf_strict.rb +74 -82
  119. data/test/perf_wab.rb +67 -69
  120. data/test/prec.rb +5 -5
  121. data/test/sample/change.rb +0 -1
  122. data/test/sample/dir.rb +0 -1
  123. data/test/sample/doc.rb +0 -1
  124. data/test/sample/file.rb +0 -1
  125. data/test/sample/group.rb +0 -1
  126. data/test/sample/hasprops.rb +0 -1
  127. data/test/sample/layer.rb +0 -1
  128. data/test/sample/rect.rb +0 -1
  129. data/test/sample/shape.rb +0 -1
  130. data/test/sample/text.rb +0 -1
  131. data/test/sample.rb +16 -16
  132. data/test/sample_json.rb +8 -8
  133. data/test/test_compat.rb +95 -43
  134. data/test/test_custom.rb +73 -51
  135. data/test/test_debian.rb +7 -10
  136. data/test/test_fast.rb +135 -79
  137. data/test/test_file.rb +41 -30
  138. data/test/test_gc.rb +16 -5
  139. data/test/test_generate.rb +5 -5
  140. data/test/test_hash.rb +5 -5
  141. data/test/test_integer_range.rb +9 -9
  142. data/test/test_null.rb +20 -20
  143. data/test/test_object.rb +99 -96
  144. data/test/test_parser.rb +11 -0
  145. data/test/test_parser_debug.rb +27 -0
  146. data/test/test_parser_saj.rb +337 -0
  147. data/test/test_parser_usual.rb +251 -0
  148. data/test/test_rails.rb +2 -2
  149. data/test/test_saj.rb +10 -8
  150. data/test/test_scp.rb +37 -39
  151. data/test/test_strict.rb +40 -32
  152. data/test/test_various.rb +165 -84
  153. data/test/test_wab.rb +48 -44
  154. data/test/test_writer.rb +47 -47
  155. data/test/tests.rb +13 -5
  156. data/test/tests_mimic.rb +12 -3
  157. data/test/tests_mimic_addition.rb +12 -3
  158. metadata +74 -128
  159. data/ext/oj/hash.c +0 -131
  160. data/ext/oj/hash.h +0 -19
  161. data/ext/oj/hash_test.c +0 -491
  162. data/test/activesupport4/decoding_test.rb +0 -108
  163. data/test/activesupport4/encoding_test.rb +0 -531
  164. data/test/activesupport4/test_helper.rb +0 -41
  165. data/test/activesupport5/test_helper.rb +0 -72
  166. data/test/bar.rb +0 -35
  167. data/test/baz.rb +0 -16
  168. data/test/zoo.rb +0 -13
data/README.md CHANGED
@@ -1,11 +1,13 @@
1
1
  # [![{}j](http://www.ohler.com/dev/images/oj_comet_64.svg)](http://www.ohler.com/oj) gem
2
2
 
3
- [![Build Status](https://img.shields.io/travis/ohler55/oj/master.svg?logo=travis)](http://travis-ci.org/ohler55/oj?branch=master) ![Gem](https://img.shields.io/gem/v/oj.svg) ![Gem](https://img.shields.io/gem/dt/oj.svg) [![SemVer compatibility](https://api.dependabot.com/badges/compatibility_score?dependency-name=oj&package-manager=bundler&version-scheme=semver)](https://dependabot.com/compatibility-score.html?dependency-name=oj&package-manager=bundler&version-scheme=semver) [![TideLift](https://tidelift.com/badges/github/ohler55/oj)](https://tidelift.com/subscription/pkg/rubygems-oj?utm_source=rubygems-oj&utm_medium=referral&utm_campaign=readme)
3
+ [![CI](https://github.com/ohler55/oj/actions/workflows/CI.yml/badge.svg)](https://github.com/ohler55/oj/actions/workflows/CI.yml)
4
+ ![Gem](https://img.shields.io/gem/v/oj.svg)
5
+ ![Gem](https://img.shields.io/gem/dt/oj.svg)
6
+ [![TideLift](https://tidelift.com/badges/github/ohler55/oj)](https://tidelift.com/subscription/pkg/rubygems-oj?utm_source=rubygems-oj&utm_medium=referral&utm_campaign=readme)
4
7
 
5
8
  A *fast* JSON parser and Object marshaller as a Ruby gem.
6
9
 
7
- Version 3.0 is out! 3.0 provides better json gem and Rails compatibility. It
8
- also provides additional optimization options.
10
+ Version 3.13 is out with a much faster parser (`Oj::Parser`) and option isolation.
9
11
 
10
12
  ## Using
11
13
 
@@ -40,6 +42,15 @@ or in Bundler:
40
42
  gem 'oj'
41
43
  ```
42
44
 
45
+ ## Rails and json quickstart
46
+
47
+ See the Quickstart sections of the [Rails](pages/Rails.md) and [json](pages/JsonGem.md) docs.
48
+
49
+ ## multi_json
50
+
51
+ Code which uses [multi_json](https://github.com/intridea/multi_json)
52
+ will automatically prefer Oj if it is installed.
53
+
43
54
  ## Support
44
55
 
45
56
  [Get supported Oj with a Tidelift Subscription.](https://tidelift.com/subscription/pkg/rubygems-oj?utm_source=rubygems-oj&utm_medium=referral&utm_campaign=readme) Security updates are [supported](https://tidelift.com/security).
@@ -50,7 +61,7 @@ For more details on options, modes, advanced features, and more follow these
50
61
  links.
51
62
 
52
63
  - [{file:Options.md}](pages/Options.md) for parse and dump options.
53
- - [{file:Modes.md}](pages/Modes.md) for details on modes for strict JSON compliance, mimicing the JSON gem, and mimicing Rails and ActiveSupport behavior.
64
+ - [{file:Modes.md}](pages/Modes.md) for details on modes for strict JSON compliance, mimicking the JSON gem, and mimicking Rails and ActiveSupport behavior.
54
65
  - [{file:JsonGem.md}](pages/JsonGem.md) includes more details on json gem compatibility and use.
55
66
  - [{file:Rails.md}](pages/Rails.md) includes more details on Rails and ActiveSupport compatibility and use.
56
67
  - [{file:Custom.md}](pages/Custom.md) includes more details on Custom mode.
@@ -58,10 +69,11 @@ links.
58
69
  - [{file:Compatibility.md}](pages/Compatibility.md) lists current compatibility with Rubys and Rails.
59
70
  - [{file:Advanced.md}](pages/Advanced.md) for fast parser and marshalling features.
60
71
  - [{file:Security.md}](pages/Security.md) for security considerations.
72
+ - [{file:InstallOptions.md}](pages/InstallOptions.md) for install option.
61
73
 
62
74
  ## Releases
63
75
 
64
- See [{file:CHANGELOG.md}](CHANGELOG.md)
76
+ See [{file:CHANGELOG.md}](CHANGELOG.md) and [{file:RELEASE_NOTES.md}](RELEASE_NOTES.md)
65
77
 
66
78
  ## Links
67
79
 
@@ -97,6 +109,8 @@ Follow [@peterohler on Twitter](http://twitter.com/peterohler) for announcements
97
109
 
98
110
  - *Agoo-C, a high performance C web server supporting GraphQL on GitHub*: https://github.com/ohler55/agoo-c
99
111
 
112
+ - *oj-introspect, an example of creating an Oj parser extension in C*: https://github.com/meinac/oj-introspect
113
+
100
114
  #### Contributing
101
115
 
102
116
  + Provide a Pull Request off the `develop` branch.
data/RELEASE_NOTES.md ADDED
@@ -0,0 +1,61 @@
1
+ # RELEASE NOTES
2
+
3
+ The release notes here are organized by release. For a list of changes
4
+ see the See [{file:CHANGELOG.md}](CHANGELOG.md) file. In this file are
5
+ the steps to take to aid in keeping things rolling after updating to
6
+ the latest version.
7
+
8
+ ## 3.13.7
9
+
10
+ The default for JSON when mimicked by Oj is now to set
11
+ `:allow_invalid_unicode`. To change that behavior JSON.load, set that
12
+ option to false.
13
+
14
+ ## 3.13.x
15
+
16
+ This release included a new cache that performs better than the
17
+ earlier cache and a new high performance parser.
18
+
19
+ ### Cache
20
+
21
+ The new cache includes a least recently used expiration to reduce
22
+ memory use. The cache is also self adjusting and will expand as needed
23
+ for better performance. It also handles Hash keys and string values
24
+ with two options, `:cache_keys`, a boolean and `:cache_str` an
25
+ integer. The `:cache_str` if set to more than zero is the limit for
26
+ the length of string values to cache. The maximum value is 35 which
27
+ allows strings up to 34 bytes to be cached.
28
+
29
+ One interesting aspect of the cache is not so much the string caching
30
+ which performs similar to the Ruby intern functions but the caching of
31
+ symbols and object attribute names. There is a significant gain for
32
+ symbols and object attributes.
33
+
34
+ If the cache is not desired then setting the default options to turn
35
+ it off can be done with this line:
36
+
37
+ ``` ruby
38
+ Oj.default_options = { cache_keys: false, cache_str: 0 }
39
+ ```
40
+
41
+ ### Oj::Parser
42
+
43
+ The new parser uses a different core that follows the approach taken
44
+ by [OjC](https://github.com/ohler55/ojc) and
45
+ [OjG](https://github.com/ohler55/ojg). It also takes advantage of the
46
+ bulk Array and Hash functions. Another issue the new parser addresses
47
+ is option management. Instead of a single global default_options each
48
+ parser instance maintains it's own options.
49
+
50
+ There is a price to be paid when using the Oj::Parser. The API is not
51
+ the same the older parser. A single parser can only be used in a
52
+ single thread. This allows reuse of internal buffers for additional
53
+ improvements in performance.
54
+
55
+ The performane advantage of the Oj::Parse is that it is more than 3
56
+ times faster than the Oj::compat_load call and 6 times faster than the
57
+ JSON gem.
58
+
59
+ ### Dump Performance
60
+
61
+ Thanks to Watson1978 Oj.dump also received a speed boost.
data/ext/oj/buf.h CHANGED
@@ -4,6 +4,7 @@
4
4
  #ifndef OJ_BUF_H
5
5
  #define OJ_BUF_H
6
6
 
7
+ #include "mem.h"
7
8
  #include "ruby.h"
8
9
 
9
10
  typedef struct _buf {
@@ -11,7 +12,7 @@ typedef struct _buf {
11
12
  char *end;
12
13
  char *tail;
13
14
  char base[1024];
14
- } * Buf;
15
+ } *Buf;
15
16
 
16
17
  inline static void buf_init(Buf buf) {
17
18
  buf->head = buf->base;
@@ -19,9 +20,13 @@ inline static void buf_init(Buf buf) {
19
20
  buf->tail = buf->head;
20
21
  }
21
22
 
23
+ inline static void buf_reset(Buf buf) {
24
+ buf->tail = buf->head;
25
+ }
26
+
22
27
  inline static void buf_cleanup(Buf buf) {
23
28
  if (buf->base != buf->head) {
24
- xfree(buf->head);
29
+ OJ_R_FREE(buf->head);
25
30
  }
26
31
  }
27
32
 
@@ -29,17 +34,26 @@ inline static size_t buf_len(Buf buf) {
29
34
  return buf->tail - buf->head;
30
35
  }
31
36
 
37
+ inline static const char *buf_str(Buf buf) {
38
+ *buf->tail = '\0';
39
+ return buf->head;
40
+ }
41
+
32
42
  inline static void buf_append_string(Buf buf, const char *s, size_t slen) {
43
+ if (0 == slen) {
44
+ return;
45
+ }
46
+
33
47
  if (buf->end <= buf->tail + slen) {
34
48
  size_t len = buf->end - buf->head;
35
49
  size_t toff = buf->tail - buf->head;
36
50
  size_t new_len = len + slen + len / 2;
37
51
 
38
52
  if (buf->base == buf->head) {
39
- buf->head = ALLOC_N(char, new_len);
53
+ buf->head = OJ_R_ALLOC_N(char, new_len);
40
54
  memcpy(buf->head, buf->base, len);
41
55
  } else {
42
- REALLOC_N(buf->head, char, new_len);
56
+ OJ_R_REALLOC_N(buf->head, char, new_len);
43
57
  }
44
58
  buf->tail = buf->head + toff;
45
59
  buf->end = buf->head + new_len - 1;
@@ -55,10 +69,10 @@ inline static void buf_append(Buf buf, char c) {
55
69
  size_t new_len = len + len / 2;
56
70
 
57
71
  if (buf->base == buf->head) {
58
- buf->head = ALLOC_N(char, new_len);
72
+ buf->head = OJ_R_ALLOC_N(char, new_len);
59
73
  memcpy(buf->head, buf->base, len);
60
74
  } else {
61
- REALLOC_N(buf->head, char, new_len);
75
+ OJ_R_REALLOC_N(buf->head, char, new_len);
62
76
  }
63
77
  buf->tail = buf->head + toff;
64
78
  buf->end = buf->head + new_len - 1;
data/ext/oj/cache.c ADDED
@@ -0,0 +1,329 @@
1
+ // Copyright (c) 2011, 2021 Peter Ohler. All rights reserved.
2
+ // Licensed under the MIT License. See LICENSE file in the project root for license details.
3
+
4
+ #if HAVE_PTHREAD_MUTEX_INIT
5
+ #include <pthread.h>
6
+ #endif
7
+ #include <stdlib.h>
8
+
9
+ #include "cache.h"
10
+ #include "mem.h"
11
+
12
+ // The stdlib calloc, realloc, and free are used instead of the Ruby ALLOC,
13
+ // ALLOC_N, REALLOC, and xfree since the later could trigger a GC which will
14
+ // either corrupt memory or if the mark function locks will deadlock.
15
+
16
+ #define REHASH_LIMIT 4
17
+ #define MIN_SHIFT 8
18
+ #define REUSE_MAX 8192
19
+
20
+ #if HAVE_PTHREAD_MUTEX_INIT
21
+ #define CACHE_LOCK(c) pthread_mutex_lock(&((c)->mutex))
22
+ #define CACHE_UNLOCK(c) pthread_mutex_unlock(&((c)->mutex))
23
+ #else
24
+ #define CACHE_LOCK(c) rb_mutex_lock((c)->mutex)
25
+ #define CACHE_UNLOCK(c) rb_mutex_unlock((c)->mutex)
26
+ #endif
27
+
28
+ // almost the Murmur hash algorithm
29
+ #define M 0x5bd1e995
30
+
31
+ typedef struct _slot {
32
+ struct _slot *next;
33
+ VALUE val;
34
+ uint64_t hash;
35
+ volatile uint32_t use_cnt;
36
+ uint8_t klen;
37
+ char key[CACHE_MAX_KEY];
38
+ } *Slot;
39
+
40
+ typedef struct _cache {
41
+ volatile Slot *slots;
42
+ volatile size_t cnt;
43
+ VALUE (*form)(const char *str, size_t len);
44
+ uint64_t size;
45
+ uint64_t mask;
46
+ VALUE (*intern)(struct _cache *c, const char *key, size_t len);
47
+ volatile Slot reuse;
48
+ size_t rcnt;
49
+ #if HAVE_PTHREAD_MUTEX_INIT
50
+ pthread_mutex_t mutex;
51
+ #else
52
+ VALUE mutex;
53
+ #endif
54
+ uint8_t xrate;
55
+ bool mark;
56
+ } *Cache;
57
+
58
+ void cache_set_form(Cache c, VALUE (*form)(const char *str, size_t len)) {
59
+ c->form = form;
60
+ }
61
+
62
+ static uint64_t hash_calc(const uint8_t *key, size_t len) {
63
+ const uint8_t *end = key + len;
64
+ const uint8_t *endless = key + (len & 0xFFFFFFFC);
65
+ uint64_t h = (uint64_t)len;
66
+ uint64_t k;
67
+
68
+ while (key < endless) {
69
+ k = (uint64_t)*key++;
70
+ k |= (uint64_t)*key++ << 8;
71
+ k |= (uint64_t)*key++ << 16;
72
+ k |= (uint64_t)*key++ << 24;
73
+
74
+ k *= M;
75
+ k ^= k >> 24;
76
+ h *= M;
77
+ h ^= k * M;
78
+ }
79
+ if (1 < end - key) {
80
+ uint16_t k16 = (uint16_t)*key++;
81
+
82
+ k16 |= (uint16_t)*key++ << 8;
83
+ h ^= k16 << 8;
84
+ }
85
+ if (key < end) {
86
+ h ^= *key;
87
+ }
88
+ h *= M;
89
+ h ^= h >> 13;
90
+ h *= M;
91
+ h ^= h >> 15;
92
+
93
+ return h;
94
+ }
95
+
96
+ static void rehash(Cache c) {
97
+ uint64_t osize;
98
+ Slot *end;
99
+ Slot *sp;
100
+
101
+ osize = c->size;
102
+ c->size = osize * 4;
103
+ c->mask = c->size - 1;
104
+ c->slots = OJ_REALLOC((void *)c->slots, sizeof(Slot) * c->size);
105
+ memset((Slot *)c->slots + osize, 0, sizeof(Slot) * osize * 3);
106
+ end = (Slot *)c->slots + osize;
107
+ for (sp = (Slot *)c->slots; sp < end; sp++) {
108
+ Slot s = *sp;
109
+ Slot next = NULL;
110
+
111
+ *sp = NULL;
112
+ for (; NULL != s; s = next) {
113
+ uint64_t h = s->hash & c->mask;
114
+ Slot *bucket = (Slot *)c->slots + h;
115
+
116
+ next = s->next;
117
+ s->next = *bucket;
118
+ *bucket = s;
119
+ }
120
+ }
121
+ }
122
+
123
+ static VALUE lockless_intern(Cache c, const char *key, size_t len) {
124
+ uint64_t h = hash_calc((const uint8_t *)key, len);
125
+ Slot *bucket = (Slot *)c->slots + (h & c->mask);
126
+ Slot b;
127
+ volatile VALUE rkey;
128
+
129
+ while (REUSE_MAX < c->rcnt) {
130
+ if (NULL != (b = c->reuse)) {
131
+ c->reuse = b->next;
132
+ OJ_FREE(b);
133
+ c->rcnt--;
134
+ } else {
135
+ // An accounting error occured somewhere so correct it.
136
+ c->rcnt = 0;
137
+ }
138
+ }
139
+ for (b = *bucket; NULL != b; b = b->next) {
140
+ if ((uint8_t)len == b->klen && 0 == strncmp(b->key, key, len)) {
141
+ b->use_cnt += 16;
142
+ return b->val;
143
+ }
144
+ }
145
+ rkey = c->form(key, len);
146
+ if (NULL == (b = c->reuse)) {
147
+ b = OJ_CALLOC(1, sizeof(struct _slot));
148
+ } else {
149
+ c->reuse = b->next;
150
+ c->rcnt--;
151
+ }
152
+ b->hash = h;
153
+ memcpy(b->key, key, len);
154
+ b->klen = (uint8_t)len;
155
+ b->key[len] = '\0';
156
+ b->val = rkey;
157
+ b->use_cnt = 4;
158
+ b->next = *bucket;
159
+ *bucket = b;
160
+ c->cnt++; // Don't worry about wrapping. Worse case is the entry is removed and recreated.
161
+ if (REHASH_LIMIT < c->cnt / c->size) {
162
+ rehash(c);
163
+ }
164
+ return rkey;
165
+ }
166
+
167
+ static VALUE locking_intern(Cache c, const char *key, size_t len) {
168
+ uint64_t h;
169
+ Slot *bucket;
170
+ Slot b;
171
+ uint64_t old_size;
172
+ volatile VALUE rkey;
173
+
174
+ CACHE_LOCK(c);
175
+ while (REUSE_MAX < c->rcnt) {
176
+ if (NULL != (b = c->reuse)) {
177
+ c->reuse = b->next;
178
+ OJ_FREE(b);
179
+ c->rcnt--;
180
+ } else {
181
+ // An accounting error occured somewhere so correct it.
182
+ c->rcnt = 0;
183
+ }
184
+ }
185
+ h = hash_calc((const uint8_t *)key, len);
186
+ bucket = (Slot *)c->slots + (h & c->mask);
187
+ for (b = *bucket; NULL != b; b = b->next) {
188
+ if ((uint8_t)len == b->klen && 0 == strncmp(b->key, key, len)) {
189
+ b->use_cnt += 4;
190
+ CACHE_UNLOCK(c);
191
+
192
+ return b->val;
193
+ }
194
+ }
195
+ old_size = c->size;
196
+ // The creation of a new value may trigger a GC which be a problem if the
197
+ // cache is locked so make sure it is unlocked for the key value creation.
198
+ if (NULL != (b = c->reuse)) {
199
+ c->reuse = b->next;
200
+ c->rcnt--;
201
+ }
202
+ CACHE_UNLOCK(c);
203
+ if (NULL == b) {
204
+ b = OJ_CALLOC(1, sizeof(struct _slot));
205
+ }
206
+ rkey = c->form(key, len);
207
+ b->hash = h;
208
+ memcpy(b->key, key, len);
209
+ b->klen = (uint8_t)len;
210
+ b->key[len] = '\0';
211
+ b->val = rkey;
212
+ b->use_cnt = 16;
213
+
214
+ // Lock again to add the new entry.
215
+ CACHE_LOCK(c);
216
+ if (old_size != c->size) {
217
+ h = hash_calc((const uint8_t *)key, len);
218
+ bucket = (Slot *)c->slots + (h & c->mask);
219
+ }
220
+ b->next = *bucket;
221
+ *bucket = b;
222
+ c->cnt++; // Don't worry about wrapping. Worse case is the entry is removed and recreated.
223
+ if (REHASH_LIMIT < c->cnt / c->size) {
224
+ rehash(c);
225
+ }
226
+ CACHE_UNLOCK(c);
227
+
228
+ return rkey;
229
+ }
230
+
231
+ Cache cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark, bool locking) {
232
+ Cache c = OJ_CALLOC(1, sizeof(struct _cache));
233
+ int shift = 0;
234
+
235
+ for (; REHASH_LIMIT < size; size /= 2, shift++) {
236
+ }
237
+ if (shift < MIN_SHIFT) {
238
+ shift = MIN_SHIFT;
239
+ }
240
+ #if HAVE_PTHREAD_MUTEX_INIT
241
+ pthread_mutex_init(&c->mutex, NULL);
242
+ #else
243
+ c->mutex = rb_mutex_new();
244
+ #endif
245
+ c->size = 1 << shift;
246
+ c->mask = c->size - 1;
247
+ c->slots = OJ_CALLOC(c->size, sizeof(Slot));
248
+ c->form = form;
249
+ c->xrate = 1; // low
250
+ c->mark = mark;
251
+ if (locking) {
252
+ c->intern = locking_intern;
253
+ } else {
254
+ c->intern = lockless_intern;
255
+ }
256
+ return c;
257
+ }
258
+
259
+ void cache_set_expunge_rate(Cache c, int rate) {
260
+ c->xrate = (uint8_t)rate;
261
+ }
262
+
263
+ void cache_free(void *data) {
264
+ Cache c = (Cache)data;
265
+ uint64_t i;
266
+
267
+ for (i = 0; i < c->size; i++) {
268
+ Slot next;
269
+ Slot s;
270
+
271
+ for (s = c->slots[i]; NULL != s; s = next) {
272
+ next = s->next;
273
+ OJ_FREE(s);
274
+ }
275
+ }
276
+ OJ_FREE((void *)c->slots);
277
+ OJ_FREE(c);
278
+ }
279
+
280
+ void cache_mark(void *data) {
281
+ Cache c = (Cache)data;
282
+ uint64_t i;
283
+
284
+ #if !HAVE_PTHREAD_MUTEX_INIT
285
+ rb_gc_mark(c->mutex);
286
+ #endif
287
+ if (0 == c->cnt) {
288
+ return;
289
+ }
290
+ for (i = 0; i < c->size; i++) {
291
+ Slot s;
292
+ Slot prev = NULL;
293
+ Slot next;
294
+
295
+ for (s = c->slots[i]; NULL != s; s = next) {
296
+ next = s->next;
297
+ if (0 == s->use_cnt) {
298
+ if (NULL == prev) {
299
+ c->slots[i] = next;
300
+ } else {
301
+ prev->next = next;
302
+ }
303
+ c->cnt--;
304
+ s->next = c->reuse;
305
+ c->reuse = s;
306
+ c->rcnt++;
307
+ continue;
308
+ }
309
+ switch (c->xrate) {
310
+ case 0: break;
311
+ case 2: s->use_cnt -= 2; break;
312
+ case 3: s->use_cnt /= 2; break;
313
+ default: s->use_cnt--; break;
314
+ }
315
+ if (c->mark) {
316
+ rb_gc_mark(s->val);
317
+ }
318
+ prev = s;
319
+ }
320
+ }
321
+ }
322
+
323
+ VALUE
324
+ cache_intern(Cache c, const char *key, size_t len) {
325
+ if (CACHE_MAX_KEY <= len) {
326
+ return c->form(key, len);
327
+ }
328
+ return c->intern(c, key, len);
329
+ }
data/ext/oj/cache.h ADDED
@@ -0,0 +1,22 @@
1
+ // Copyright (c) 2021 Peter Ohler. All rights reserved.
2
+ // Licensed under the MIT License. See LICENSE file in the project root for license details.
3
+
4
+ #ifndef CACHE_H
5
+ #define CACHE_H
6
+
7
+ #include <ruby.h>
8
+ #include <stdbool.h>
9
+
10
+ #define CACHE_MAX_KEY 35
11
+
12
+ struct _cache;
13
+ typedef struct _cache *Cache;
14
+
15
+ extern struct _cache *cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark, bool locking);
16
+ extern void cache_free(void *data);
17
+ extern void cache_mark(void *data);
18
+ extern void cache_set_form(struct _cache *c, VALUE (*form)(const char *str, size_t len));
19
+ extern VALUE cache_intern(struct _cache *c, const char *key, size_t len);
20
+ extern void cache_set_expunge_rate(struct _cache *c, int rate);
21
+
22
+ #endif /* CACHE_H */
data/ext/oj/cache8.c CHANGED
@@ -9,6 +9,7 @@
9
9
  #include <stdlib.h>
10
10
  #include <string.h>
11
11
 
12
+ #include "mem.h"
12
13
  #include "ruby.h"
13
14
 
14
15
  #define BITS 4
@@ -32,16 +33,18 @@ void oj_cache8_new(Cache8 *cache) {
32
33
  Bucket *b;
33
34
  int i;
34
35
 
35
- *cache = ALLOC(struct _cache8);
36
+ *cache = OJ_R_ALLOC(struct _cache8);
36
37
  for (i = SLOT_CNT, b = (*cache)->buckets; 0 < i; i--, b++) {
37
38
  b->value = 0;
38
39
  }
39
40
  }
40
41
 
41
- void oj_cache8_delete(Cache8 cache) { cache8_delete(cache, 0); }
42
+ void oj_cache8_delete(Cache8 cache) {
43
+ cache8_delete(cache, 0);
44
+ }
42
45
 
43
46
  static void cache8_delete(Cache8 cache, int depth) {
44
- Bucket * b;
47
+ Bucket *b;
45
48
  unsigned int i;
46
49
 
47
50
  for (i = 0, b = cache->buckets; i < SLOT_CNT; i++, b++) {
@@ -51,7 +54,7 @@ static void cache8_delete(Cache8 cache, int depth) {
51
54
  }
52
55
  }
53
56
  }
54
- xfree(cache);
57
+ OJ_R_FREE(cache);
55
58
  }
56
59
 
57
60
  slot_t oj_cache8_get(Cache8 cache, sid_t key, slot_t **slot) {
@@ -79,7 +82,7 @@ void oj_cache8_print(Cache8 cache) {
79
82
  }
80
83
 
81
84
  static void slot_print(Cache8 c, sid_t key, unsigned int depth) {
82
- Bucket * b;
85
+ Bucket *b;
83
86
  unsigned int i;
84
87
  sid_t k8 = (sid_t)key;
85
88
  sid_t k;
@@ -90,11 +93,9 @@ static void slot_print(Cache8 c, sid_t key, unsigned int depth) {
90
93
  /*printf("*** key: 0x%016llx depth: %u i: %u\n", k, depth, i); */
91
94
  if (DEPTH - 1 == depth) {
92
95
  #if IS_WINDOWS
93
- printf("0x%016lx: %4lu\n", (long unsigned int)k,
94
- (long unsigned int)b->value);
96
+ printf("0x%016lx: %4lu\n", (long unsigned int)k, (long unsigned int)b->value);
95
97
  #else
96
- printf("0x%016llx: %4llu\n", (long long unsigned int)k,
97
- (long long unsigned int)b->value);
98
+ printf("0x%016llx: %4llu\n", (long long unsigned int)k, (long long unsigned int)b->value);
98
99
  #endif
99
100
  } else {
100
101
  slot_print(b->child, k, depth + 1);
data/ext/oj/circarray.c CHANGED
@@ -3,10 +3,12 @@
3
3
 
4
4
  #include "circarray.h"
5
5
 
6
- CircArray oj_circ_array_new() {
6
+ #include "mem.h"
7
+
8
+ CircArray oj_circ_array_new(void) {
7
9
  CircArray ca;
8
10
 
9
- if (0 == (ca = ALLOC(struct _circArray))) {
11
+ if (0 == (ca = OJ_R_ALLOC(struct _circArray))) {
10
12
  rb_raise(rb_eNoMemError, "not enough memory\n");
11
13
  }
12
14
  ca->objs = ca->obj_array;
@@ -18,9 +20,9 @@ CircArray oj_circ_array_new() {
18
20
 
19
21
  void oj_circ_array_free(CircArray ca) {
20
22
  if (ca->objs != ca->obj_array) {
21
- xfree(ca->objs);
23
+ OJ_R_FREE(ca->objs);
22
24
  }
23
- xfree(ca);
25
+ OJ_R_FREE(ca);
24
26
  }
25
27
 
26
28
  void oj_circ_array_set(CircArray ca, VALUE obj, unsigned long id) {
@@ -31,12 +33,12 @@ void oj_circ_array_set(CircArray ca, VALUE obj, unsigned long id) {
31
33
  unsigned long cnt = id + 512;
32
34
 
33
35
  if (ca->objs == ca->obj_array) {
34
- if (0 == (ca->objs = ALLOC_N(VALUE, cnt))) {
36
+ if (0 == (ca->objs = OJ_R_ALLOC_N(VALUE, cnt))) {
35
37
  rb_raise(rb_eNoMemError, "not enough memory\n");
36
38
  }
37
39
  memcpy(ca->objs, ca->obj_array, sizeof(VALUE) * ca->cnt);
38
40
  } else {
39
- REALLOC_N(ca->objs, VALUE, cnt);
41
+ OJ_R_REALLOC_N(ca->objs, VALUE, cnt);
40
42
  }
41
43
  ca->size = cnt;
42
44
  }
data/ext/oj/circarray.h CHANGED
@@ -9,10 +9,10 @@
9
9
 
10
10
  typedef struct _circArray {
11
11
  VALUE obj_array[1024];
12
- VALUE * objs;
12
+ VALUE* objs;
13
13
  unsigned long size; // allocated size or initial array size
14
14
  unsigned long cnt;
15
- } * CircArray;
15
+ }* CircArray;
16
16
 
17
17
  extern CircArray oj_circ_array_new(void);
18
18
  extern void oj_circ_array_free(CircArray ca);