prism 0.17.1 → 0.19.0

data/src/util/pm_buffer.c

@@ -138,7 +138,7 @@ pm_buffer_append_byte(pm_buffer_t *buffer, uint8_t value) {
  * Append a 32-bit unsigned integer to the buffer as a variable-length integer.
  */
 void
-pm_buffer_append_varint(pm_buffer_t *buffer, uint32_t value) {
+pm_buffer_append_varuint(pm_buffer_t *buffer, uint32_t value) {
     if (value < 128) {
         pm_buffer_append_byte(buffer, (uint8_t) value);
     } else {
@@ -151,6 +151,15 @@ pm_buffer_append_varint(pm_buffer_t *buffer, uint32_t value) {
     }
 }
 
+/**
+ * Append a 32-bit signed integer to the buffer as a variable-length integer.
+ */
+void
+pm_buffer_append_varsint(pm_buffer_t *buffer, int32_t value) {
+    uint32_t unsigned_int = ((uint32_t)(value) << 1) ^ ((uint32_t)(value >> 31));
+    pm_buffer_append_varuint(buffer, unsigned_int);
+}
+
 /**
  * Concatenate one buffer onto another.
  */

data/src/util/pm_memchr.c

@@ -8,7 +8,7 @@
  * of a multibyte character.
  */
 void *
-pm_memchr(const void *memory, int character, size_t number, bool encoding_changed, pm_encoding_t *encoding) {
+pm_memchr(const void *memory, int character, size_t number, bool encoding_changed, const pm_encoding_t *encoding) {
     if (encoding_changed && encoding->multibyte && character >= PRISM_MEMCHR_TRAILING_BYTE_MINIMUM) {
         const uint8_t *source = (const uint8_t *) memory;
         size_t index = 0;

data/src/util/pm_strpbrk.c

@@ -4,7 +4,7 @@
  * This is the slow path that does care about the encoding.
  */
 static inline const uint8_t *
-pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum) {
+pm_strpbrk_multi_byte(const pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum) {
     size_t index = 0;
 
     while (index < maximum) {
@@ -12,7 +12,7 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
             return source + index;
         }
 
-        size_t width = parser->encoding.char_width(source + index, (ptrdiff_t) (maximum - index));
+        size_t width = parser->encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
         if (width == 0) {
             return NULL;
         }
@@ -61,10 +61,10 @@ pm_strpbrk_single_byte(const uint8_t *source, const uint8_t *charset, size_t max
  * need to take a slower path and iterate one multi-byte character at a time.
  */
 const uint8_t *
-pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length) {
+pm_strpbrk(const pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length) {
     if (length <= 0) {
         return NULL;
-    } else if (parser->encoding_changed && parser->encoding.multibyte) {
+    } else if (parser->encoding_changed && parser->encoding->multibyte) {
         return pm_strpbrk_multi_byte(parser, source, charset, (size_t) length);
     } else {
         return pm_strpbrk_single_byte(source, charset, (size_t) length);

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prism
 version: !ruby/object:Gem::Version
-  version: 0.17.1
+  version: 0.19.0
 platform: ruby
 authors:
 - Shopify
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-11-03 00:00:00.000000000 Z
+date: 2023-12-14 00:00:00.000000000 Z
 dependencies: []
 description:
 email:
@@ -33,6 +33,7 @@ files:
 - docs/fuzzing.md
 - docs/heredocs.md
 - docs/javascript.md
+- docs/local_variable_depth.md
 - docs/mapping.md
 - docs/releasing.md
 - docs/ripper.md
@@ -48,7 +49,7 @@ files:
 - include/prism/ast.h
 - include/prism/defines.h
 - include/prism/diagnostic.h
-- include/prism/enc/pm_encoding.h
+- include/prism/encoding.h
 - include/prism/node.h
 - include/prism/options.h
 - include/prism/pack.h
@@ -72,6 +73,7 @@ files:
 - lib/prism/debug.rb
 - lib/prism/desugar_compiler.rb
 - lib/prism/dispatcher.rb
+- lib/prism/dot_visitor.rb
 - lib/prism/dsl.rb
 - lib/prism/ffi.rb
 - lib/prism/lex_compat.rb
@@ -93,13 +95,7 @@ files:
 - sig/prism.rbs
 - sig/prism_static.rbs
 - src/diagnostic.c
-- src/enc/pm_big5.c
-- src/enc/pm_euc_jp.c
-- src/enc/pm_gbk.c
-- src/enc/pm_shift_jis.c
-- src/enc/pm_tables.c
-- src/enc/pm_unicode.c
-- src/enc/pm_windows_31j.c
+- src/encoding.c
 - src/node.c
 - src/options.c
 - src/pack.c
@@ -124,6 +120,8 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
+  source_code_uri: https://github.com/ruby/prism
+  changelog_uri: https://github.com/ruby/prism/blob/main/CHANGELOG.md
 post_install_message:
 rdoc_options: []
 require_paths:

data/src/enc/pm_big5.c

@@ -1,53 +0,0 @@
-#include "prism/enc/pm_encoding.h"
-
-static size_t
-pm_encoding_big5_char_width(const uint8_t *b, ptrdiff_t n) {
-    // These are the single byte characters.
-    if (*b < 0x80) {
-        return 1;
-    }
-
-    // These are the double byte characters.
-    if ((n > 1) && (b[0] >= 0xA1 && b[0] <= 0xFE) && (b[1] >= 0x40 && b[1] <= 0xFE)) {
-        return 2;
-    }
-
-    return 0;
-}
-
-static size_t
-pm_encoding_big5_alpha_char(const uint8_t *b, ptrdiff_t n) {
-    if (pm_encoding_big5_char_width(b, n) == 1) {
-        return pm_encoding_ascii_alpha_char(b, n);
-    } else {
-        return 0;
-    }
-}
-
-static size_t
-pm_encoding_big5_alnum_char(const uint8_t *b, ptrdiff_t n) {
-    if (pm_encoding_big5_char_width(b, n) == 1) {
-        return pm_encoding_ascii_alnum_char(b, n);
-    } else {
-        return 0;
-    }
-}
-
-static bool
-pm_encoding_big5_isupper_char(const uint8_t *b, ptrdiff_t n) {
-    if (pm_encoding_big5_char_width(b, n) == 1) {
-        return pm_encoding_ascii_isupper_char(b, n);
-    } else {
-        return false;
-    }
-}
-
-/** Big5 encoding */
-pm_encoding_t pm_encoding_big5 = {
-    .name = "big5",
-    .char_width = pm_encoding_big5_char_width,
-    .alnum_char = pm_encoding_big5_alnum_char,
-    .alpha_char = pm_encoding_big5_alpha_char,
-    .isupper_char = pm_encoding_big5_isupper_char,
-    .multibyte = true
-};

data/src/enc/pm_euc_jp.c

@@ -1,59 +0,0 @@
-#include "prism/enc/pm_encoding.h"
-
-static size_t
-pm_encoding_euc_jp_char_width(const uint8_t *b, ptrdiff_t n) {
-    // These are the single byte characters.
-    if (*b < 0x80) {
-        return 1;
-    }
-
-    // These are the double byte characters.
-    if (
-        (n > 1) &&
-        (
-            ((b[0] == 0x8E) && (b[1] >= 0xA1 && b[1] <= 0xFE)) ||
-            ((b[0] >= 0xA1 && b[0] <= 0xFE) && (b[1] >= 0xA1 && b[1] <= 0xFE))
-        )
-    ) {
-        return 2;
-    }
-
-    return 0;
-}
-
-static size_t
-pm_encoding_euc_jp_alpha_char(const uint8_t *b, ptrdiff_t n) {
-    if (pm_encoding_euc_jp_char_width(b, n) == 1) {
-        return pm_encoding_ascii_alpha_char(b, n);
-    } else {
-        return 0;
-    }
-}
-
-static size_t
-pm_encoding_euc_jp_alnum_char(const uint8_t *b, ptrdiff_t n) {
-    if (pm_encoding_euc_jp_char_width(b, n) == 1) {
-        return pm_encoding_ascii_alnum_char(b, n);
-    } else {
-        return 0;
-    }
-}
-
-static bool
-pm_encoding_euc_jp_isupper_char(const uint8_t *b, ptrdiff_t n) {
-    if (pm_encoding_euc_jp_char_width(b, n) == 1) {
-        return pm_encoding_ascii_isupper_char(b, n);
-    } else {
-        return 0;
-    }
-}
-
-/** EUC-JP encoding */
-pm_encoding_t pm_encoding_euc_jp = {
-    .name = "euc-jp",
-    .char_width = pm_encoding_euc_jp_char_width,
-    .alnum_char = pm_encoding_euc_jp_alnum_char,
-    .alpha_char = pm_encoding_euc_jp_alpha_char,
-    .isupper_char = pm_encoding_euc_jp_isupper_char,
-    .multibyte = true
-};

data/src/enc/pm_gbk.c

@@ -1,62 +0,0 @@
-#include "prism/enc/pm_encoding.h"
-
-static size_t
-pm_encoding_gbk_char_width(const uint8_t *b, ptrdiff_t n) {
-    // These are the single byte characters.
-    if (*b < 0x80) {
-        return 1;
-    }
-
-    // These are the double byte characters.
-    if (
-        (n > 1) &&
-        (
-            ((b[0] >= 0xA1 && b[0] <= 0xA9) && (b[1] >= 0xA1 && b[1] <= 0xFE)) || // GBK/1
-            ((b[0] >= 0xB0 && b[0] <= 0xF7) && (b[1] >= 0xA1 && b[1] <= 0xFE)) || // GBK/2
-            ((b[0] >= 0x81 && b[0] <= 0xA0) && (b[1] >= 0x40 && b[1] <= 0xFE) && (b[1] != 0x7F)) || // GBK/3
-            ((b[0] >= 0xAA && b[0] <= 0xFE) && (b[1] >= 0x40 && b[1] <= 0xA0) && (b[1] != 0x7F)) || // GBK/4
-            ((b[0] >= 0xA8 && b[0] <= 0xA9) && (b[1] >= 0x40 && b[1] <= 0xA0) && (b[1] != 0x7F)) // GBK/5
-        )
-    ) {
-        return 2;
-    }
-
-    return 0;
-}
-
-static size_t
-pm_encoding_gbk_alpha_char(const uint8_t *b, ptrdiff_t n) {
-    if (pm_encoding_gbk_char_width(b, n) == 1) {
-        return pm_encoding_ascii_alpha_char(b, n);
-    } else {
-        return 0;
-    }
-}
-
-static size_t
-pm_encoding_gbk_alnum_char(const uint8_t *b, ptrdiff_t n) {
-    if (pm_encoding_gbk_char_width(b, n) == 1) {
-        return pm_encoding_ascii_alnum_char(b, n);
-    } else {
-        return 0;
-    }
-}
-
-static bool
-pm_encoding_gbk_isupper_char(const uint8_t *b, ptrdiff_t n) {
-    if (pm_encoding_gbk_char_width(b, n) == 1) {
-        return pm_encoding_ascii_isupper_char(b, n);
-    } else {
-        return false;
-    }
-}
-
-/** GBK encoding */
-pm_encoding_t pm_encoding_gbk = {
-    .name = "gbk",
-    .char_width = pm_encoding_gbk_char_width,
-    .alnum_char = pm_encoding_gbk_alnum_char,
-    .alpha_char = pm_encoding_gbk_alpha_char,
-    .isupper_char = pm_encoding_gbk_isupper_char,
-    .multibyte = true
-};

data/src/enc/pm_shift_jis.c

@@ -1,57 +0,0 @@
-#include "prism/enc/pm_encoding.h"
-
-static size_t
-pm_encoding_shift_jis_char_width(const uint8_t *b, ptrdiff_t n) {
-    // These are the single byte characters.
-    if (*b < 0x80 || (*b >= 0xA1 && *b <= 0xDF)) {
-        return 1;
-    }
-
-    // These are the double byte characters.
-    if (
-        (n > 1) &&
-        ((b[0] >= 0x81 && b[0] <= 0x9F) || (b[0] >= 0xE0 && b[0] <= 0xFC)) &&
-        (b[1] >= 0x40 && b[1] <= 0xFC)
-    ) {
-        return 2;
-    }
-
-    return 0;
-}
-
-static size_t
-pm_encoding_shift_jis_alpha_char(const uint8_t *b, ptrdiff_t n) {
-    if (pm_encoding_shift_jis_char_width(b, n) == 1) {
-        return pm_encoding_ascii_alpha_char(b, n);
-    } else {
-        return 0;
-    }
-}
-
-static size_t
-pm_encoding_shift_jis_alnum_char(const uint8_t *b, ptrdiff_t n) {
-    if (pm_encoding_shift_jis_char_width(b, n) == 1) {
-        return pm_encoding_ascii_alnum_char(b, n);
-    } else {
-        return 0;
-    }
-}
-
-static bool
-pm_encoding_shift_jis_isupper_char(const uint8_t *b, ptrdiff_t n) {
-    if (pm_encoding_shift_jis_char_width(b, n) == 1) {
-        return pm_encoding_ascii_isupper_char(b, n);
-    } else {
-        return 0;
-    }
-}
-
-/** Shift_JIS encoding */
-pm_encoding_t pm_encoding_shift_jis = {
-    .name = "shift_jis",
-    .char_width = pm_encoding_shift_jis_char_width,
-    .alnum_char = pm_encoding_shift_jis_alnum_char,
-    .alpha_char = pm_encoding_shift_jis_alpha_char,
-    .isupper_char = pm_encoding_shift_jis_isupper_char,
-    .multibyte = true
-};