ox 2.14.6 → 2.14.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8cef7183a7bea9cd7baba3313afd0e008861d159ae8d99ef229dca9cb34ba1fd
-  data.tar.gz: 4929590f1b84ac3e5350c2657a3374eb49a2a258668bbf13d2ee5624110402c5
+  metadata.gz: e6a58bfa91298fd0da2d02538dfe039bb5a426edfba8e4f245d8bd91b7388cbb
+  data.tar.gz: 77765350035814045329fa94935cd3de29975eaefbaef1666e4ffe29269338d8
 SHA512:
-  metadata.gz: fb7fb21b9be8f9e046b22e3dde36bb2469256daab163e17e082c9c15865926e21d2793052f544ebadaa958fb6589d30e9a5b7c6fed018f1fa6e32782271a2e8c
-  data.tar.gz: f259be386f39855faf5f871119cd60c16ee6c23872baf2239754f17512355fd19c2dbfa8c254746e1f44b0819f32b3e63700a5bfe3f6f0ff831c5f9c107f9df1
+  metadata.gz: 9ac7c0bd9c0ac54cd033f432a0e14cd29b265b078459025c889ca08a907703d1aa202aee9ccae792e87a47b783bacf3044e62bedfc0c25090457dd5035ee2ee3
+  data.tar.gz: 73fc0775f561ca3290b6f6a875d0c8076d4434a642f0c0e7321952ed8b1adc655aa0f880916e1fde5f02f0d29b5c7cc57de871e4e58c869540c219bd46d75403

data/CHANGELOG.md

@@ -2,6 +2,38 @@
 
 All changes to the Ox gem are documented here. Releases follow semantic versioning.
 
+## [2.14.10] - 2022-03-10
+
+### Fixed
+
+- Writing strings over 16K to a file with builder no longer causes a crash.
+
+## [2.14.9] - 2022-02-11
+
+### Fixed
+
+- Fixed the `\r` replacement with `\n` with the SAX parser according to https://www.w3.org/TR/2008/REC-xml-20081126/#sec-line-ends.
+
+## [2.14.8] - 2022-02-09
+
+### Fixed
+
+- Renamed internal functions to avoid linking issues where Oj and Ox function names collided.
+
+## [2.14.7] - 2022-02-03
+
+### Fixed
+
+- All classes and symbols are now registered to avoid issues with GC compaction movement.
+- Parsing of any size processing instruction is now allowed. There is no 1024 limit.
+- Fixed the `\r` replacement with `\n` according to https://www.w3.org/TR/2008/REC-xml-20081126/#sec-line-ends.
+
+### Changed
+
+- Symbol and string caching changed but should have no impact on use
+  other than being slightly faster and handles large numbers of cached
+  items more efficiently.
+
 ## [2.14.6] - 2021-11-03
 
 ### Fixed

data/ext/ox/buf.h

@@ -86,8 +86,16 @@ buf_append_string(Buf buf, const char *s, size_t slen) {
 
 	    if (len != (size_t)write(buf->fd, buf->head, len)) {
 		buf->err = true;
+		return;
 	    }
 	    buf->tail = buf->head;
+	    if (sizeof(buf->base) <= slen) {
+		if (slen != (size_t)write(buf->fd, s, slen)) {
+		    buf->err = true;
+		    return;
+		}
+		return;
+	    }
 	} else {
 	    size_t	len = buf->end - buf->head;
 	    size_t	toff = buf->tail - buf->head;

data/ext/ox/builder.c

@@ -9,9 +9,7 @@
 #include <string.h>
 
 #include "ruby.h"
-#if HAVE_RB_ENC_ASSOCIATE
 #include "ruby/encoding.h"
-#endif
 #include "ox.h"
 #include "buf.h"
 #include "err.h"
@@ -335,9 +333,7 @@ to_s(Builder b) {
     rstr = rb_str_new(b->buf.head, buf_len(&b->buf));
 
     if ('\0' != *b->encoding) {
-#if HAVE_RB_ENC_ASSOCIATE
 	rb_enc_associate(rstr, rb_enc_find(b->encoding));
-#endif
     }
     return rstr;
 }

data/ext/ox/cache.c

@@ -1,160 +1,338 @@
-/* cache.c
- * Copyright (c) 2011, Peter Ohler
- * All rights reserved.
- */
+// Copyright (c) 2011, 2021 Peter Ohler. All rights reserved.
+// Licensed under the MIT License. See LICENSE file in the project root for license details.
 
+#if HAVE_PTHREAD_MUTEX_INIT
+#include <pthread.h>
+#endif
 #include <stdlib.h>
-#include <errno.h>
-#include <stdio.h>
-#include <string.h>
-#include <strings.h>
-#include <stdarg.h>
-#include <stdint.h>
 
 #include "cache.h"
 
-struct _cache {
-    /* The key is a length byte followed by the key as a string. If the key is longer than 254 characters then the
-       length is 255. The key can be for a premature value and in that case the length byte is greater than the length
-       of the key. */
-    char                *key;
-    VALUE               value;
-    struct _cache       *slots[16];
-};
+// The stdlib calloc, realloc, and free are used instead of the Ruby ALLOC,
+// ALLOC_N, REALLOC, and xfree since the later could trigger a GC which will
+// either corrupt memory or if the mark function locks will deadlock.
 
-static void     slot_print(Cache cache, unsigned int depth);
+#define REHASH_LIMIT 4
+#define MIN_SHIFT 8
+#define REUSE_MAX 8192
 
-static char* form_key(const char *s) {
-    size_t	len = strlen(s);
-    char	*d = ALLOC_N(char, len + 2);
+#if HAVE_PTHREAD_MUTEX_INIT
+#define CACHE_LOCK(c) pthread_mutex_lock(&((c)->mutex))
+#define CACHE_UNLOCK(c) pthread_mutex_unlock(&((c)->mutex))
+#else
+#define CACHE_LOCK(c) rb_mutex_lock((c)->mutex)
+#define CACHE_UNLOCK(c) rb_mutex_unlock((c)->mutex)
+#endif
 
-    *(uint8_t*)d = (255 <= len) ? 255 : len;
-    memcpy(d + 1, s, len + 1);
+// almost the Murmur hash algorithm
+#define M 0x5bd1e995
 
-    return d;
+typedef struct _slot {
+    struct _slot     *next;
+    VALUE             val;
+    uint64_t          hash;
+    volatile uint32_t use_cnt;
+    uint8_t           klen;
+    char              key[CACHE_MAX_KEY];
+} * Slot;
+
+typedef struct _cache {
+    volatile Slot  *slots;
+    volatile size_t cnt;
+    VALUE (*form)(const char *str, size_t len);
+    uint64_t size;
+    uint64_t mask;
+    VALUE (*intern)(struct _cache *c, const char *key, size_t len, const char **keyp);
+    volatile Slot reuse;
+    size_t        rcnt;
+#if HAVE_PTHREAD_MUTEX_INIT
+    pthread_mutex_t mutex;
+#else
+    VALUE mutex;
+#endif
+    uint8_t xrate;
+    bool    mark;
+} * Cache;
+
+static uint64_t hash_calc(const uint8_t *key, size_t len) {
+    const uint8_t *end     = key + len;
+    const uint8_t *endless = key + (len & 0xFFFFFFFC);
+    uint64_t       h       = (uint64_t)len;
+    uint64_t       k;
+
+    while (key < endless) {
+        k = (uint64_t)*key++;
+        k |= (uint64_t)*key++ << 8;
+        k |= (uint64_t)*key++ << 16;
+        k |= (uint64_t)*key++ << 24;
+
+        k *= M;
+        k ^= k >> 24;
+        h *= M;
+        h ^= k * M;
+    }
+    if (1 < end - key) {
+        uint16_t k16 = (uint16_t)*key++;
+
+        k16 |= (uint16_t)*key++ << 8;
+        h ^= k16 << 8;
+    }
+    if (key < end) {
+        h ^= *key;
+    }
+    h *= M;
+    h ^= h >> 13;
+    h *= M;
+    h ^= h >> 15;
+
+    return h;
 }
 
-void
-ox_cache_new(Cache *cache) {
-    *cache = ALLOC(struct _cache);
-    (*cache)->key = 0;
-    (*cache)->value = Qundef;
-    memset((*cache)->slots, 0, sizeof((*cache)->slots));
+static void rehash(Cache c) {
+    uint64_t osize;
+    Slot    *end;
+    Slot    *sp;
+
+    osize    = c->size;
+    c->size  = osize * 4;
+    c->mask  = c->size - 1;
+    c->slots = realloc((void *)c->slots, sizeof(Slot) * c->size);
+    memset((Slot *)c->slots + osize, 0, sizeof(Slot) * osize * 3);
+    end = (Slot *)c->slots + osize;
+    for (sp = (Slot *)c->slots; sp < end; sp++) {
+        Slot s    = *sp;
+        Slot next = NULL;
+
+        *sp = NULL;
+        for (; NULL != s; s = next) {
+            uint64_t h      = s->hash & c->mask;
+            Slot    *bucket = (Slot *)c->slots + h;
+
+            next    = s->next;
+            s->next = *bucket;
+            *bucket = s;
+        }
+    }
 }
 
-VALUE
-ox_cache_get(Cache cache, const char *key, VALUE **slot, const char **keyp) {
-    unsigned char       *k = (unsigned char*)key;
-    Cache               *cp;
-
-    for (; '\0' != *k; k++) {
-        cp = cache->slots + (unsigned int)(*k >> 4); /* upper 4 bits */
-        if (0 == *cp) {
-            ox_cache_new(cp);
+static VALUE ox_lockless_intern(Cache c, const char *key, size_t len, const char **keyp) {
+    uint64_t       h      = hash_calc((const uint8_t *)key, len);
+    Slot          *bucket = (Slot *)c->slots + (h & c->mask);
+    Slot           b;
+    volatile VALUE rkey;
+
+    while (REUSE_MAX < c->rcnt) {
+        if (NULL != (b = c->reuse)) {
+            c->reuse = b->next;
+            free(b);
+            c->rcnt--;
+        } else {
+            // An accounting error occured somewhere so correct it.
+            c->rcnt = 0;
+        }
+    }
+    for (b = *bucket; NULL != b; b = b->next) {
+        if ((uint8_t)len == b->klen && 0 == strncmp(b->key, key, len)) {
+            b->use_cnt += 16;
+            if (NULL != keyp) {
+                *keyp = b->key;
+            }
+            return b->val;
         }
-        cache = *cp;
-        cp = cache->slots + (unsigned int)(*k & 0x0F); /* lower 4 bits */
-        if (0 == *cp) { /* nothing on this tree so set key and value as a premature key/value pair */
-            ox_cache_new(cp);
-            cache = *cp;
-            cache->key = form_key(key);
-            break;
-	} else {
-	    int	depth = (int)(k - (unsigned char*)key + 1);
-
-	    cache = *cp;
-
-	    if ('\0' == *(k + 1)) { /* exact match */
-		if (0 == cache->key) { /* nothing in this spot so take it */
-		    cache->key = form_key(key);
-		    break;
-		} else if ((depth == *cache->key || 255 < depth) && 0 == strcmp(key, cache->key + 1)) { /* match */
-		    break;
-		} else { /* have to move the current premature key/value deeper */
-		    unsigned char	*ck = (unsigned char*)(cache->key + depth + 1);
-                    Cache		orig = *cp;
-
-                    cp = (*cp)->slots + (*ck >> 4);
-                    ox_cache_new(cp);
-                    cp = (*cp)->slots + (*ck & 0x0F);
-                    ox_cache_new(cp);
-		    (*cp)->key = cache->key;
-		    (*cp)->value = cache->value;
-		    orig->key = form_key(key);
-		    orig->value = Qundef;
-		}
-	    } else { /* not exact match but on the path */
-		if (0 != cache->key) { /* there is a key/value here already */
-		    if (depth == *cache->key || (255 <= depth && 0 == strncmp(cache->key, key, depth) && '\0' == cache->key[depth])) { /* key belongs here */
-			continue;
-		    } else {
-			unsigned char	*ck = (unsigned char*)(cache->key + depth + 1);
-			Cache		orig = *cp;
-
-			cp = (*cp)->slots + (*ck >> 4);
-			ox_cache_new(cp);
-			cp = (*cp)->slots + (*ck & 0x0F);
-			ox_cache_new(cp);
-			(*cp)->key = cache->key;
-			(*cp)->value = cache->value;
-			orig->key = 0;
-			orig->value = Qundef;
-		    }
-		}
-	    }
+    }
+    rkey = c->form(key, len);
+    if (NULL == (b = c->reuse)) {
+        b = calloc(1, sizeof(struct _slot));
+    } else {
+        c->reuse = b->next;
+        c->rcnt--;
+    }
+    b->hash = h;
+    memcpy(b->key, key, len);
+    b->klen     = (uint8_t)len;
+    b->key[len] = '\0';
+    b->val      = rkey;
+    b->use_cnt  = 4;
+    b->next     = *bucket;
+    *bucket     = b;
+    c->cnt++;  // Don't worry about wrapping. Worse case is the entry is removed and recreated.
+    if (NULL != keyp) {
+        *keyp = b->key;
+    }
+    if (REHASH_LIMIT < c->cnt / c->size) {
+        rehash(c);
+    }
+    return rkey;
+}
+
+static VALUE ox_locking_intern(Cache c, const char *key, size_t len, const char **keyp) {
+    uint64_t       h;
+    Slot          *bucket;
+    Slot           b;
+    uint64_t       old_size;
+    volatile VALUE rkey;
+
+    CACHE_LOCK(c);
+    while (REUSE_MAX < c->rcnt) {
+        if (NULL != (b = c->reuse)) {
+            c->reuse = b->next;
+            free(b);
+            c->rcnt--;
+        } else {
+            // An accounting error occured somewhere so correct it.
+            c->rcnt = 0;
+        }
+    }
+    h      = hash_calc((const uint8_t *)key, len);
+    bucket = (Slot *)c->slots + (h & c->mask);
+    for (b = *bucket; NULL != b; b = b->next) {
+        if ((uint8_t)len == b->klen && 0 == strncmp(b->key, key, len)) {
+            b->use_cnt += 4;
+            if (NULL != keyp) {
+                *keyp = b->key;
+            }
+            CACHE_UNLOCK(c);
+
+            return b->val;
         }
     }
-    *slot = &cache->value;
-    if (0 != keyp) {
-	if (0 == cache->key) {
-	    printf("*** Error: failed to set the key for '%s'\n", key);
-	    *keyp = 0;
-	} else {
-	    *keyp = cache->key + 1;
-	}
+    old_size = c->size;
+    // The creation of a new value may trigger a GC which be a problem if the
+    // cache is locked so make sure it is unlocked for the key value creation.
+    if (NULL != (b = c->reuse)) {
+        c->reuse = b->next;
+        c->rcnt--;
+    }
+    CACHE_UNLOCK(c);
+    if (NULL == b) {
+        b = calloc(1, sizeof(struct _slot));
+    }
+    rkey    = c->form(key, len);
+    b->hash = h;
+    memcpy(b->key, key, len);
+    b->klen     = (uint8_t)len;
+    b->key[len] = '\0';
+    b->val      = rkey;
+    b->use_cnt  = 16;
+
+    // Lock again to add the new entry.
+    CACHE_LOCK(c);
+    if (old_size != c->size) {
+        h      = hash_calc((const uint8_t *)key, len);
+        bucket = (Slot *)c->slots + (h & c->mask);
+    }
+    b->next = *bucket;
+    *bucket = b;
+    c->cnt++;  // Don't worry about wrapping. Worse case is the entry is removed and recreated.
+    if (NULL != keyp) {
+        *keyp = b->key;
     }
-    return cache->value;
+    if (REHASH_LIMIT < c->cnt / c->size) {
+        rehash(c);
+    }
+    CACHE_UNLOCK(c);
+
+    return rkey;
 }
 
-void
-ox_cache_print(Cache cache) {
-    /*printf("-------------------------------------------\n");*/
-    slot_print(cache, 0);
+Cache ox_cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark, bool locking) {
+    Cache c     = calloc(1, sizeof(struct _cache));
+    int   shift = 0;
+
+    for (; REHASH_LIMIT < size; size /= 2, shift++) {
+    }
+    if (shift < MIN_SHIFT) {
+        shift = MIN_SHIFT;
+    }
+#if HAVE_PTHREAD_MUTEX_INIT
+    pthread_mutex_init(&c->mutex, NULL);
+#else
+    c->mutex = rb_mutex_new();
+#endif
+    c->size  = 1 << shift;
+    c->mask  = c->size - 1;
+    c->slots = calloc(c->size, sizeof(Slot));
+    c->form  = form;
+    c->xrate = 1;  // low
+    c->mark  = mark;
+    if (locking) {
+        c->intern = ox_locking_intern;
+    } else {
+        c->intern = ox_lockless_intern;
+    }
+    return c;
 }
 
-static void
-slot_print(Cache c, unsigned int depth) {
-    char                indent[256];
-    Cache               *cp;
-    unsigned int        i;
+void ox_cache_free(Cache c) {
+    uint64_t i;
 
-    if (sizeof(indent) - 1 < depth) {
-        depth = ((int)sizeof(indent) - 1);
+    for (i = 0; i < c->size; i++) {
+        Slot next;
+        Slot s;
+
+        for (s = c->slots[i]; NULL != s; s = next) {
+            next = s->next;
+            free(s);
+        }
     }
-    memset(indent, ' ', depth);
-    indent[depth] = '\0';
-    for (i = 0, cp = c->slots; i < 16; i++, cp++) {
-        if (0 == *cp) {
-            /*printf("%s%02u:\n", indent, i);*/
-        } else {
-            if (0 == (*cp)->key && Qundef == (*cp)->value) {
-                printf("%s%02u:\n", indent, i);
-            } else {
-                const char      *vs;
-                const char      *clas;
-
-                if (Qundef == (*cp)->value) {
-                    vs = "undefined";
-                    clas = "";
-                } else {
-                    VALUE       rs = rb_funcall2((*cp)->value, rb_intern("to_s"), 0, 0);
+    free((void *)c->slots);
+    free(c);
+}
+
+void ox_cache_mark(Cache c) {
+    uint64_t i;
+
+#if !HAVE_PTHREAD_MUTEX_INIT
+    rb_gc_mark(c->mutex);
+#endif
+    if (0 == c->cnt) {
+        return;
+    }
+    for (i = 0; i < c->size; i++) {
+        Slot s;
+        Slot prev = NULL;
+        Slot next;
 
-                    vs = StringValuePtr(rs);
-                    clas = rb_class2name(rb_obj_class((*cp)->value));
+        for (s = c->slots[i]; NULL != s; s = next) {
+            next = s->next;
+            if (0 == s->use_cnt) {
+                if (NULL == prev) {
+                    c->slots[i] = next;
+                } else {
+                    prev->next = next;
                 }
-                printf("%s%02u: %s = %s (%s)\n", indent, i, (*cp)->key, vs, clas);
+                c->cnt--;
+                s->next  = c->reuse;
+                c->reuse = s;
+                c->rcnt++;
+                continue;
+            }
+            switch (c->xrate) {
+            case 0: break;
+            case 2: s->use_cnt -= 2; break;
+            case 3: s->use_cnt /= 2; break;
+            default: s->use_cnt--; break;
+            }
+            if (c->mark) {
+                rb_gc_mark(s->val);
+            }
+            prev = s;
+        }
+    }
+}
+
+VALUE
+ox_cache_intern(Cache c, const char *key, size_t len, const char **keyp) {
+    if (CACHE_MAX_KEY <= len) {
+        if (NULL != keyp) {
+            volatile VALUE rkey = c->form(key, len);
+
+            if (SYMBOL_P(rkey)) {
+                *keyp = rb_id2name(rb_sym2id(rkey));
             }
-            slot_print(*cp, depth + 2);
+            return rkey;
         }
+        return c->form(key, len);
     }
+    return c->intern(c, key, len, keyp);
 }

data/ext/ox/cache.h

@@ -1,19 +1,19 @@
-/* cache.h
- * Copyright (c) 2011, Peter Ohler
- * All rights reserved.
- */
+// Copyright (c) 2021 Peter Ohler. All rights reserved.
+// Licensed under the MIT License. See LICENSE file in the project root for license details.
 
 #ifndef OX_CACHE_H
 #define OX_CACHE_H
 
-#include "ruby.h"
+#include <ruby.h>
+#include <stdbool.h>
 
-typedef struct _cache   *Cache;
+#define CACHE_MAX_KEY 35
 
-extern void     ox_cache_new(Cache *cache);
+struct _cache;
 
-extern VALUE    ox_cache_get(Cache cache, const char *key, VALUE **slot, const char **keyp);
-
-extern void     ox_cache_print(Cache cache);
+extern struct _cache *ox_cache_create(size_t size, VALUE (*form)(const char *str, size_t len), bool mark, bool locking);
+extern void           ox_cache_free(struct _cache *c);
+extern void           ox_cache_mark(struct _cache *c);
+extern VALUE          ox_cache_intern(struct _cache *c, const char *key, size_t len, const char **keyp);
 
 #endif /* OX_CACHE_H */

data/ext/ox/dump.c

@@ -511,7 +511,7 @@ dump_date(Out out, VALUE obj) {
 static void
 dump_time_xsd(Out out, VALUE obj) {
     struct tm		*tm;
-#if HAS_RB_TIME_TIMESPEC
+#if HAVE_RB_TIME_TIMESPEC
     struct timespec	ts = rb_time_timespec(obj);
     time_t		sec = ts.tv_sec;
     long		nsec = ts.tv_nsec;
@@ -528,7 +528,7 @@ dump_time_xsd(Out out, VALUE obj) {
     }
     /* 2010-07-09T10:47:45.895826+09:00 */
     tm = localtime(&sec);
-#if HAS_TM_GMTOFF
+#if HAVE_ST_TM_GMTOFF
     if (0 > tm->tm_gmtoff) {
 	tzsign = '-';
 	tzhour = (int)(tm->tm_gmtoff / -3600);

data/ext/ox/extconf.rb

@@ -33,12 +33,14 @@ CONFIG['warnflags'].slice!(/ -Wdeclaration-after-statement/)
 CONFIG['warnflags'].slice!(/ -Wmissing-noreturn/)
 
 have_func('rb_time_timespec')
-have_func('rb_enc_associate')
-have_func('rb_enc_find')
 have_func('rb_struct_alloc_noinit')
 have_func('rb_obj_encoding')
 have_func('rb_ivar_foreach')
 have_func('rb_ext_ractor_safe', 'ruby.h')
+have_func('pthread_mutex_init')
+have_func('rb_enc_interned_str')
+have_func('rb_time_nano_new')
+have_func('index')
 
 have_header('ruby/st.h')
 have_header('sys/uio.h')