liquid-c 4.0.1 → 4.2.0

data/ext/liquid_c/expression.h

@@ -0,0 +1,24 @@
+#if !defined(LIQUID_EXPRESSION_H)
+#define LIQUID_EXPRESSION_H
+
+#include "vm_assembler.h"
+#include "parser.h"
+
+extern VALUE cLiquidCExpression;
+extern const rb_data_type_t expression_data_type;
+
+typedef struct expression {
+    vm_assembler_t code;
+} expression_t;
+
+extern const rb_data_type_t expression_data_type;
+#define Expression_Get_Struct(obj, sval) TypedData_Get_Struct(obj, expression_t, &expression_data_type, sval)
+
+void liquid_define_expression(void);
+
+VALUE expression_new(VALUE klass, expression_t **expression_ptr);
+VALUE expression_evaluate(VALUE self, VALUE context);
+VALUE internal_expression_evaluate(expression_t *expression, VALUE context);
+
+#endif
+

data/ext/liquid_c/extconf.rb

@@ -1,14 +1,26 @@
-require 'mkmf'
-$CFLAGS << ' -std=c99 -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers'
+# frozen_string_literal: true
+
+require "mkmf"
+$CFLAGS << " -std=c11 -Wall -Wextra -Wno-unused-parameter -Wno-missing-field-initializers"
+append_cflags("-fvisibility=hidden")
 # In Ruby 2.6 and earlier, the Ruby headers did not have struct timespec defined
-valid_headers = RbConfig::CONFIG['host_os'] !~ /linux/ || Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.7")
-pedantic = !ENV['LIQUID_C_PEDANTIC'].to_s.empty?
+valid_headers = RbConfig::CONFIG["host_os"] !~ /linux/ || Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.7")
+pedantic = !ENV["LIQUID_C_PEDANTIC"].to_s.empty?
 if pedantic && valid_headers
-  $CFLAGS << ' -Werror'
+  $CFLAGS << " -Werror"
 end
-compiler = RbConfig::MAKEFILE_CONFIG['CC']
-if ENV['DEBUG'] == 'true' && compiler =~ /gcc|g\+\+/
-  $CFLAGS << ' -fbounds-check'
+if ENV["DEBUG"] == "true"
+  append_cflags("-fbounds-check")
+  CONFIG["optflags"] = " -O0"
+  # Hack to enable assertions since ruby/assert.h disables assertions unless
+  # Ruby was compiled with -DRUBY_DEBUG.
+  # https://github.com/ruby/ruby/blob/9e678cdbd054f78576a8f21b3f97cccc395ade22/include/ruby/assert.h#L36-L41
+  $CFLAGS << " -DRUBY_DEBUG"
+else
+  $CFLAGS << " -DNDEBUG"
 end
-$warnflags.gsub!(/-Wdeclaration-after-statement/, "") if $warnflags
+
+have_func "rb_hash_bulk_insert"
+
+$warnflags&.gsub!(/-Wdeclaration-after-statement/, "")
 create_makefile("liquid_c")

data/ext/liquid_c/intutil.h

@@ -0,0 +1,22 @@
+#ifndef LIQUID_INTUTIL_H
+#define LIQUID_INTUTIL_H
+
+#include <stdint.h>
+
+static inline unsigned int bytes_to_uint24(const uint8_t *bytes)
+{
+    return (bytes[0] << 16) | (bytes[1] << 8) | bytes[2];
+}
+
+static inline void uint24_to_bytes(unsigned int num, uint8_t *bytes)
+{
+    assert(num < (1 << 24));
+
+    bytes[0] = num >> 16;
+    bytes[1] = num >> 8;
+    bytes[2] = num;
+
+    assert(bytes_to_uint24(bytes) == num);
+}
+
+#endif

data/ext/liquid_c/lexer.c

@@ -1,5 +1,6 @@
 #include "liquid.h"
 #include "lexer.h"
+#include "usage.h"
 #include <stdio.h>
 
 const char *symbol_names[TOKEN_END] = {
@@ -59,8 +60,6 @@ inline static const char *scan_past(const char *cur, const char *end, char targe
     const char *tok_end = str + (n); \
     token->type = (t); \
     token->val = str; \
-    if (str != start) token->flags |= TOKEN_SPACE_PREFIX; \
-    if (tok_end < end && ISSPACE(*tok_end)) token->flags |= TOKEN_SPACE_SUFFIX; \
     return (token->val_end = tok_end); \
 }
 
@@ -109,6 +108,11 @@ const char *lex_one(const char *start, const char *end, lexer_token_t *token)
         }
     }
 
+    // Instrument for bug: https://github.com/Shopify/liquid-c/pull/120
+    if (c == '-' && str + 1 < end && str[1] == '.') {
+        usage_increment("liquid_c_negative_float_without_integer");
+    }
+
     if (ISDIGIT(c) || c == '-') {
         int has_dot = 0;
         cur = str;
@@ -140,7 +144,39 @@ const char *lex_one(const char *start, const char *end, lexer_token_t *token)
 
     if (is_special(c)) RETURN_TOKEN(c, 1);
 
-    rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Unexpected character %c", c);
+    long remaining_str_len = end - str;
+    int char_len = 0;
+
+    // read multibyte UTF-8 character
+    if ((c & 0x80) == 0) {
+        // 1-byte character
+        char_len = 1;
+    } else if ((c & 0xE0) == 0xC0) {
+        // 2-byte character
+        if (remaining_str_len >= 2) {
+            char_len = 2;
+        }
+    } else if ((c & 0xF0) == 0xE0) {
+        // 3-byte character
+        if (remaining_str_len >= 3) {
+            char_len = 3;
+        }
+    } else if ((c & 0xF8) == 0xF0) {
+        // 4-byte character
+        if (remaining_str_len >= 4) {
+            char_len = 4;
+        }
+    } else {
+        // this should never happen
+        rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Unexpected character %c", c);
+    }
+
+    if (char_len > 0) {
+        rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Unexpected character %.*s", char_len, str);
+    } else {
+        rb_raise(rb_eArgError, "invalid byte sequence in UTF-8");
+    }
+
     return NULL;
 }
 

data/ext/liquid_c/lexer.h

@@ -24,9 +24,6 @@ enum lexer_token_type {
     TOKEN_END = 256
 };
 
-#define TOKEN_SPACE_PREFIX 0x1
-#define TOKEN_SPACE_SUFFIX 0x2
-#define TOKEN_SPACE_AFFIX (TOKEN_SPACE_PREFIX | TOKEN_SPACE_SUFFIX)
 #define TOKEN_FLOAT_NUMBER 0x4
 
 typedef struct lexer_token {
@@ -42,5 +39,23 @@ inline static VALUE token_to_rstr(lexer_token_t token) {
     return rb_enc_str_new(token.val, token.val_end - token.val, utf8_encoding);
 }
 
+inline static VALUE token_check_for_symbol(lexer_token_t token) {
+    return rb_check_symbol_cstr(token.val, token.val_end - token.val, utf8_encoding);
+}
+
+inline static VALUE token_to_rstr_leveraging_existing_symbol(lexer_token_t token) {
+    VALUE sym = token_check_for_symbol(token);
+    if (RB_LIKELY(sym != Qnil))
+        return rb_sym2str(sym);
+    return token_to_rstr(token);
+}
+
+inline static VALUE token_to_rsym(lexer_token_t token) {
+    VALUE sym = token_check_for_symbol(token);
+    if (RB_LIKELY(sym != Qnil))
+        return sym;
+    return rb_str_intern(token_to_rstr(token));
+}
+
 #endif
 

data/ext/liquid_c/liquid.c

@@ -3,23 +3,93 @@
 #include "variable.h"
 #include "lexer.h"
 #include "parser.h"
+#include "raw.h"
+#include "resource_limits.h"
+#include "expression.h"
+#include "document_body.h"
 #include "block.h"
+#include "context.h"
+#include "parse_context.h"
+#include "variable_lookup.h"
+#include "vm_assembler_pool.h"
+#include "liquid_vm.h"
+#include "usage.h"
+
+ID id_evaluate;
+ID id_to_liquid;
+ID id_to_s;
+ID id_call;
+ID id_compile_evaluate;
+ID id_ivar_line_number;
+
+VALUE mLiquid, mLiquidC, cLiquidVariable, cLiquidTemplate, cLiquidBlockBody;
+VALUE cLiquidVariableLookup, cLiquidRangeLookup;
+VALUE cLiquidArgumentError, cLiquidSyntaxError, cMemoryError;
 
-VALUE mLiquid, mLiquidC, cLiquidSyntaxError, cLiquidVariable, cLiquidTemplate;
 rb_encoding *utf8_encoding;
+int utf8_encoding_index;
+
+__attribute__((noreturn)) void raise_non_utf8_encoding_error(VALUE string, const char *value_name)
+{
+    rb_raise(rb_eEncCompatError, "non-UTF8 encoded %s (%"PRIsVALUE") not supported", value_name, rb_obj_encoding(string));
+}
 
-void Init_liquid_c(void)
+RUBY_FUNC_EXPORTED void Init_liquid_c(void)
 {
+    id_evaluate = rb_intern("evaluate");
+    id_to_liquid = rb_intern("to_liquid");
+    id_to_s = rb_intern("to_s");
+    id_call = rb_intern("call");
+    id_compile_evaluate = rb_intern("compile_evaluate");
+    id_ivar_line_number = rb_intern("@line_number");
+
     utf8_encoding = rb_utf8_encoding();
+    utf8_encoding_index = rb_enc_to_index(utf8_encoding);
+
     mLiquid = rb_define_module("Liquid");
+    rb_global_variable(&mLiquid);
+
     mLiquidC = rb_define_module_under(mLiquid, "C");
+    rb_global_variable(&mLiquidC);
+
+    cLiquidArgumentError = rb_const_get(mLiquid, rb_intern("ArgumentError"));
+    rb_global_variable(&cLiquidArgumentError);
+
     cLiquidSyntaxError = rb_const_get(mLiquid, rb_intern("SyntaxError"));
+    rb_global_variable(&cLiquidSyntaxError);
+
+    cMemoryError = rb_const_get(mLiquid, rb_intern("MemoryError"));
+    rb_global_variable(&cMemoryError);
+
     cLiquidVariable = rb_const_get(mLiquid, rb_intern("Variable"));
+    rb_global_variable(&cLiquidVariable);
+
     cLiquidTemplate = rb_const_get(mLiquid, rb_intern("Template"));
+    rb_global_variable(&cLiquidTemplate);
+
+    cLiquidBlockBody = rb_const_get(mLiquid, rb_intern("BlockBody"));
+    rb_global_variable(&cLiquidBlockBody);
+
+    cLiquidVariableLookup = rb_const_get(mLiquid, rb_intern("VariableLookup"));
+    rb_global_variable(&cLiquidVariableLookup);
+
+    cLiquidRangeLookup = rb_const_get(mLiquid, rb_intern("RangeLookup"));
+    rb_global_variable(&cLiquidRangeLookup);
 
-    init_liquid_tokenizer();
-    init_liquid_parser();
-    init_liquid_variable();
-    init_liquid_block();
+    liquid_define_tokenizer();
+    liquid_define_parser();
+    liquid_define_raw();
+    liquid_define_resource_limits();
+    liquid_define_expression();
+    liquid_define_variable();
+    liquid_define_document_body();
+    liquid_define_block_body();
+    liquid_define_context();
+    liquid_define_parse_context();
+    liquid_define_variable_lookup();
+    liquid_define_vm_assembler_pool();
+    liquid_define_vm_assembler();
+    liquid_define_vm();
+    liquid_define_usage();
 }
 

data/ext/liquid_c/liquid.h

@@ -5,8 +5,31 @@
 #include <ruby/encoding.h>
 #include <stdbool.h>
 
-extern VALUE mLiquid, mLiquidC, cLiquidSyntaxError, cLiquidVariable, cLiquidTemplate;
+extern ID id_evaluate;
+extern ID id_to_liquid;
+extern ID id_to_s;
+extern ID id_call;
+extern ID id_compile_evaluate;
+extern ID id_ivar_line_number;
+
+extern VALUE mLiquid, mLiquidC, cLiquidVariable, cLiquidTemplate, cLiquidBlockBody;
+extern VALUE cLiquidVariableLookup, cLiquidRangeLookup;
+extern VALUE cLiquidArgumentError, cLiquidSyntaxError, cMemoryError;
 extern rb_encoding *utf8_encoding;
+extern int utf8_encoding_index;
+
+__attribute__((noreturn)) void raise_non_utf8_encoding_error(VALUE string, const char *string_name);
+
+static inline void check_utf8_encoding(VALUE string, const char *string_name)
+{
+    if (RB_UNLIKELY(RB_ENCODING_GET_INLINED(string) != utf8_encoding_index))
+        raise_non_utf8_encoding_error(string, string_name);
+}
+
+#ifndef RB_LIKELY
+// RB_LIKELY added in Ruby 2.4
+#define RB_LIKELY(x) (__builtin_expect(!!(x), 1))
+#endif
 
 #endif