herb 0.10.4 → 0.11.0

data/ext/herb/extension.c

@@ -1,4 +1,5 @@
 #include <ruby.h>
+#include <ruby/thread.h>
 
 #include "../../src/include/lib/hb_allocator.h"
 #include "../../src/include/lib/hb_arena_debug.h"
@@ -7,6 +8,7 @@
 #include "extension.h"
 #include "extension_helpers.h"
 #include "nodes.h"
+#include "parser_options_helpers.h"
 
 VALUE mHerb;
 VALUE cPosition;
@@ -20,8 +22,10 @@ VALUE cParserOptions;
 
 typedef struct {
   AST_DOCUMENT_NODE_T* root;
+  const char* input;
   VALUE source;
   const parser_options_T* parser_options;
+  bool print_arena_stats;
   hb_allocator_T allocator;
 } parse_args_T;
 
@@ -36,9 +40,21 @@ typedef struct {
   hb_allocator_T allocator;
 } buffer_args_T;
 
-static VALUE parse_convert_body(VALUE arg) {
+static void* parse_without_gvl(void* arg) {
   parse_args_T* args = (parse_args_T*) arg;
 
+  args->root = herb_parse(args->input, args->parser_options, &args->allocator);
+
+  return NULL;
+}
+
+static VALUE parse_body(VALUE arg) {
+  parse_args_T* args = (parse_args_T*) arg;
+
+  rb_thread_call_without_gvl(parse_without_gvl, (void*) arg, NULL, NULL);
+
+  if (args->print_arena_stats) { hb_arena_print_stats((hb_arena_T*) args->allocator.context); }
+
   return create_parse_result(args->root, args->source, args->parser_options);
 }
 
@@ -83,6 +99,34 @@ static VALUE buffer_cleanup(VALUE arg) {
   return Qnil;
 }
 
+#define HERB_MAX_ERB_OPENERS 64
+
+static size_t read_erb_openers(VALUE options, hb_string_T* buffer) {
+  VALUE erb_openers = rb_hash_lookup(options, rb_utf8_str_new_cstr("erb_openers"));
+  if (NIL_P(erb_openers)) { erb_openers = rb_hash_lookup(options, ID2SYM(rb_intern("erb_openers"))); }
+  if (NIL_P(erb_openers)) { return SIZE_MAX; }
+
+  Check_Type(erb_openers, T_ARRAY);
+
+  long length = RARRAY_LEN(erb_openers);
+
+  if (length > HERB_MAX_ERB_OPENERS) {
+    rb_raise(rb_eArgError, "erb_openers accepts at most %d entries", HERB_MAX_ERB_OPENERS);
+  }
+
+  for (long index = 0; index < length; index++) {
+    VALUE opener = rb_ary_entry(erb_openers, index);
+
+    Check_Type(opener, T_STRING);
+
+    if (RSTRING_LEN(opener) == 0) { rb_raise(rb_eArgError, "erb_openers entries must not be empty"); }
+
+    buffer[index] = hb_string_from_data(RSTRING_PTR(opener), (size_t) RSTRING_LEN(opener));
+  }
+
+  return (size_t) length;
+}
+
 static VALUE Herb_lex(int argc, VALUE* argv, VALUE self) {
   VALUE source, options;
   rb_scan_args(argc, argv, "1:", &source, &options);
@@ -90,10 +134,20 @@ static VALUE Herb_lex(int argc, VALUE* argv, VALUE self) {
   char* string = (char*) check_string(source);
   bool print_arena_stats = false;
 
+  parser_options_T parser_options = HERB_DEFAULT_PARSER_OPTIONS;
+  hb_string_T opener_buffer[HERB_MAX_ERB_OPENERS];
+
   if (!NIL_P(options)) {
     VALUE arena_stats = rb_hash_lookup(options, rb_utf8_str_new_cstr("arena_stats"));
     if (NIL_P(arena_stats)) { arena_stats = rb_hash_lookup(options, ID2SYM(rb_intern("arena_stats"))); }
     if (!NIL_P(arena_stats) && RTEST(arena_stats)) { print_arena_stats = true; }
+
+    size_t opener_count = read_erb_openers(options, opener_buffer);
+
+    if (opener_count != SIZE_MAX) {
+      parser_options.erb_openers = opener_buffer;
+      parser_options.erb_opener_count = opener_count;
+    }
   }
 
   lex_args_T args = { 0 };
@@ -101,7 +155,7 @@ static VALUE Herb_lex(int argc, VALUE* argv, VALUE self) {
 
   if (!hb_allocator_init(&args.allocator, HB_ALLOCATOR_ARENA)) { return Qnil; }
 
-  args.tokens = herb_lex(string, &args.allocator);
+  args.tokens = herb_lex_with_options(string, &parser_options, &args.allocator);
 
   if (print_arena_stats) { hb_arena_print_stats((hb_arena_T*) args.allocator.context); }
 
@@ -116,77 +170,16 @@ static VALUE Herb_parse(int argc, VALUE* argv, VALUE self) {
   bool print_arena_stats = false;
 
   parser_options_T parser_options = HERB_DEFAULT_PARSER_OPTIONS;
+  hb_string_T opener_buffer[HERB_MAX_ERB_OPENERS];
 
-  if (!NIL_P(options)) {
-    VALUE track_whitespace = rb_hash_lookup(options, rb_utf8_str_new_cstr("track_whitespace"));
-    if (NIL_P(track_whitespace)) { track_whitespace = rb_hash_lookup(options, ID2SYM(rb_intern("track_whitespace"))); }
-    if (!NIL_P(track_whitespace) && RTEST(track_whitespace)) { parser_options.track_whitespace = true; }
-
-    VALUE analyze = rb_hash_lookup(options, rb_utf8_str_new_cstr("analyze"));
-    if (NIL_P(analyze)) { analyze = rb_hash_lookup(options, ID2SYM(rb_intern("analyze"))); }
-    if (!NIL_P(analyze) && !RTEST(analyze)) { parser_options.analyze = false; }
-
-    VALUE strict = rb_hash_lookup(options, rb_utf8_str_new_cstr("strict"));
-    if (NIL_P(strict)) { strict = rb_hash_lookup(options, ID2SYM(rb_intern("strict"))); }
-    if (!NIL_P(strict)) { parser_options.strict = RTEST(strict); }
-
-    VALUE action_view_helpers = rb_hash_lookup(options, rb_utf8_str_new_cstr("action_view_helpers"));
-    if (NIL_P(action_view_helpers)) {
-      action_view_helpers = rb_hash_lookup(options, ID2SYM(rb_intern("action_view_helpers")));
-    }
-    if (!NIL_P(action_view_helpers) && RTEST(action_view_helpers)) { parser_options.action_view_helpers = true; }
-
-    VALUE transform_conditionals = rb_hash_lookup(options, rb_utf8_str_new_cstr("transform_conditionals"));
-    if (NIL_P(transform_conditionals)) {
-      transform_conditionals = rb_hash_lookup(options, ID2SYM(rb_intern("transform_conditionals")));
-    }
-    if (!NIL_P(transform_conditionals) && RTEST(transform_conditionals)) {
-      parser_options.transform_conditionals = true;
-    }
-
-    VALUE dot_notation_tags = rb_hash_lookup(options, rb_utf8_str_new_cstr("dot_notation_tags"));
-    if (NIL_P(dot_notation_tags)) {
-      dot_notation_tags = rb_hash_lookup(options, ID2SYM(rb_intern("dot_notation_tags")));
-    }
-    if (!NIL_P(dot_notation_tags) && RTEST(dot_notation_tags)) { parser_options.dot_notation_tags = true; }
-
-    VALUE render_nodes = rb_hash_lookup(options, rb_utf8_str_new_cstr("render_nodes"));
-    if (NIL_P(render_nodes)) { render_nodes = rb_hash_lookup(options, ID2SYM(rb_intern("render_nodes"))); }
-    if (!NIL_P(render_nodes) && RTEST(render_nodes)) { parser_options.render_nodes = true; }
-
-    VALUE strict_locals = rb_hash_lookup(options, rb_utf8_str_new_cstr("strict_locals"));
-    if (NIL_P(strict_locals)) { strict_locals = rb_hash_lookup(options, ID2SYM(rb_intern("strict_locals"))); }
-    if (!NIL_P(strict_locals) && RTEST(strict_locals)) { parser_options.strict_locals = true; }
-
-    VALUE prism_nodes = rb_hash_lookup(options, rb_utf8_str_new_cstr("prism_nodes"));
-    if (NIL_P(prism_nodes)) { prism_nodes = rb_hash_lookup(options, ID2SYM(rb_intern("prism_nodes"))); }
-    if (!NIL_P(prism_nodes) && RTEST(prism_nodes)) { parser_options.prism_nodes = true; }
+  herb_extract_parser_options(options, &parser_options);
 
-    VALUE prism_nodes_deep = rb_hash_lookup(options, rb_utf8_str_new_cstr("prism_nodes_deep"));
-    if (NIL_P(prism_nodes_deep)) { prism_nodes_deep = rb_hash_lookup(options, ID2SYM(rb_intern("prism_nodes_deep"))); }
-    if (!NIL_P(prism_nodes_deep) && RTEST(prism_nodes_deep)) { parser_options.prism_nodes_deep = true; }
-
-    VALUE prism_program = rb_hash_lookup(options, rb_utf8_str_new_cstr("prism_program"));
-    if (NIL_P(prism_program)) { prism_program = rb_hash_lookup(options, ID2SYM(rb_intern("prism_program"))); }
-    if (!NIL_P(prism_program) && RTEST(prism_program)) { parser_options.prism_program = true; }
-
-    VALUE html = rb_hash_lookup(options, rb_utf8_str_new_cstr("html"));
-    if (NIL_P(html)) { html = rb_hash_lookup(options, ID2SYM(rb_intern("html"))); }
-    if (!NIL_P(html) && !RTEST(html)) { parser_options.html = false; }
-
-    VALUE timeout = rb_hash_lookup(options, rb_utf8_str_new_cstr("timeout"));
-    if (NIL_P(timeout)) { timeout = rb_hash_lookup(options, ID2SYM(rb_intern("timeout"))); }
-    if (!NIL_P(timeout)) { parser_options.timeout_ms = (uint32_t) (NUM2DBL(timeout) * 1000); }
-
-    VALUE max_errors_sentinel = ID2SYM(rb_intern("__not_set__"));
-    VALUE max_errors = rb_hash_lookup2(options, rb_utf8_str_new_cstr("max_errors"), max_errors_sentinel);
-
-    if (max_errors == max_errors_sentinel) {
-      max_errors = rb_hash_lookup2(options, ID2SYM(rb_intern("max_errors")), max_errors_sentinel);
-    }
+  if (!NIL_P(options)) {
+    size_t opener_count = read_erb_openers(options, opener_buffer);
 
-    if (max_errors != max_errors_sentinel) {
-      parser_options.max_errors = NIL_P(max_errors) ? 0 : (uint32_t) NUM2UINT(max_errors);
+    if (opener_count != SIZE_MAX) {
+      parser_options.erb_openers = opener_buffer;
+      parser_options.erb_opener_count = opener_count;
     }
 
     VALUE arena_stats = rb_hash_lookup(options, rb_utf8_str_new_cstr("arena_stats"));
@@ -194,17 +187,26 @@ static VALUE Herb_parse(int argc, VALUE* argv, VALUE self) {
     if (!NIL_P(arena_stats) && RTEST(arena_stats)) { print_arena_stats = true; }
   }
 
+  uint32_t error_count = 0;
+  parser_options.error_count = &error_count;
+
   parse_args_T args = { 0 };
   args.source = source;
   args.parser_options = &parser_options;
+  args.print_arena_stats = print_arena_stats;
 
   if (!hb_allocator_init(&args.allocator, HB_ALLOCATOR_ARENA)) { return Qnil; }
 
-  args.root = herb_parse(string, &parser_options, &args.allocator);
+  if (string != NULL) {
+    args.input = hb_allocator_strndup(&args.allocator, string, (size_t) RSTRING_LEN(source));
 
-  if (print_arena_stats) { hb_arena_print_stats((hb_arena_T*) args.allocator.context); }
+    if (args.input == NULL) {
+      hb_allocator_destroy(&args.allocator);
+      rb_raise(rb_eNoMemError, "failed to allocate a parser input buffer");
+    }
+  }
 
-  return rb_ensure(parse_convert_body, (VALUE) &args, parse_cleanup, (VALUE) &args);
+  return rb_ensure(parse_body, (VALUE) &args, parse_cleanup, (VALUE) &args);
 }
 
 static VALUE Herb_extract_ruby(int argc, VALUE* argv, VALUE self) {
@@ -220,8 +222,20 @@ static VALUE Herb_extract_ruby(int argc, VALUE* argv, VALUE self) {
   if (!hb_buffer_init(&output, strlen(string), &allocator)) { return Qnil; }
 
   herb_extract_ruby_options_T extract_options = HERB_EXTRACT_RUBY_DEFAULT_OPTIONS;
+  hb_string_T opener_buffer[HERB_MAX_ERB_OPENERS];
 
   if (!NIL_P(options)) {
+    size_t opener_count = read_erb_openers(options, opener_buffer);
+
+    if (opener_count != SIZE_MAX) {
+      extract_options.erb_openers = opener_buffer;
+      extract_options.erb_opener_count = opener_count;
+    }
+
+    VALUE custom_tags_value = rb_hash_lookup(options, rb_utf8_str_new_cstr("custom_tags"));
+    if (NIL_P(custom_tags_value)) { custom_tags_value = rb_hash_lookup(options, ID2SYM(rb_intern("custom_tags"))); }
+    if (!NIL_P(custom_tags_value)) { extract_options.custom_tags = RTEST(custom_tags_value); }
+
     VALUE semicolons_value = rb_hash_lookup(options, rb_utf8_str_new_cstr("semicolons"));
     if (NIL_P(semicolons_value)) { semicolons_value = rb_hash_lookup(options, ID2SYM(rb_intern("semicolons"))); }
     if (!NIL_P(semicolons_value)) { extract_options.semicolons = RTEST(semicolons_value); }
@@ -396,8 +410,22 @@ static VALUE Herb_leak_check(VALUE self, VALUE source) {
   return result;
 }
 
+static VALUE Herb_default_erb_openings(VALUE self) {
+  VALUE openings = rb_ary_new_capa((long) HERB_DEFAULT_ERB_OPENINGS_COUNT);
+
+  for (size_t index = 0; index < HERB_DEFAULT_ERB_OPENINGS_COUNT; index++) {
+    hb_string_T opening = HERB_DEFAULT_ERB_OPENINGS[index];
+
+    rb_ary_push(openings, rb_str_freeze(rb_utf8_str_new(opening.data, (long) opening.length)));
+  }
+
+  return openings;
+}
+
 static VALUE Herb_version(VALUE self) {
-  VALUE gem_version = rb_const_get(self, rb_intern("VERSION"));
+  (void) self;
+
+  VALUE gem_version = rb_const_get(mHerb, rb_intern("VERSION"));
   VALUE libherb_version = rb_utf8_str_new_cstr(herb_version());
   VALUE libprism_version = rb_utf8_str_new_cstr(herb_prism_version());
 
@@ -437,7 +465,8 @@ typedef struct {
 } diff_args_T;
 
 static VALUE rb_create_diff_operation(const herb_diff_operation_T* operation) {
-  VALUE cDiffOperation = rb_const_get(mHerb, rb_intern("DiffOperation"));
+  VALUE mDiff = rb_const_get(mHerb, rb_intern("Diff"));
+  VALUE cDiffOperation = rb_const_get(mDiff, rb_intern("Operation"));
 
   VALUE type = ID2SYM(rb_intern(herb_diff_operation_type_to_string(operation->type)));
 
@@ -446,8 +475,12 @@ static VALUE rb_create_diff_operation(const herb_diff_operation_T* operation) {
     rb_ary_push(path_array, UINT2NUM(operation->path.indices[index]));
   }
 
-  VALUE old_node = operation->old_node != NULL ? rb_node_from_c_struct((AST_NODE_T*) operation->old_node) : Qnil;
-  VALUE new_node = operation->new_node != NULL ? rb_node_from_c_struct((AST_NODE_T*) operation->new_node) : Qnil;
+  VALUE old_node = operation->old_node != NULL
+                   ? rb_node_from_c_struct((AST_NODE_T*) operation->old_node, &HERB_DEFAULT_PARSER_OPTIONS)
+                   : Qnil;
+  VALUE new_node = operation->new_node != NULL
+                   ? rb_node_from_c_struct((AST_NODE_T*) operation->new_node, &HERB_DEFAULT_PARSER_OPTIONS)
+                   : Qnil;
 
   return rb_funcall(
     cDiffOperation,
@@ -466,7 +499,8 @@ static VALUE diff_convert_body(VALUE arg) {
   diff_args_T* args = (diff_args_T*) arg;
   herb_diff_result_T* diff_result = args->diff_result;
 
-  VALUE cDiffResult = rb_const_get(mHerb, rb_intern("DiffResult"));
+  VALUE mDiff = rb_const_get(mHerb, rb_intern("Diff"));
+  VALUE cDiffResult = rb_const_get(mDiff, rb_intern("Result"));
 
   size_t operation_count = herb_diff_operation_count(diff_result);
   VALUE operations_array = rb_ary_new_capa((long) operation_count);
@@ -495,8 +529,8 @@ static VALUE diff_cleanup(VALUE arg) {
 }
 
 static VALUE Herb_diff(int argc, VALUE* argv, VALUE self) {
-  VALUE old_source, new_source;
-  rb_scan_args(argc, argv, "2", &old_source, &new_source);
+  VALUE old_source, new_source, options;
+  rb_scan_args(argc, argv, "2:", &old_source, &new_source, &options);
 
   char* old_string = (char*) check_string(old_source);
   char* new_string = (char*) check_string(new_source);
@@ -504,6 +538,17 @@ static VALUE Herb_diff(int argc, VALUE* argv, VALUE self) {
   diff_args_T args = { 0 };
 
   parser_options_T parser_options = HERB_DEFAULT_PARSER_OPTIONS;
+  herb_diff_options_T diff_options = HERB_DEFAULT_DIFF_OPTIONS;
+
+  if (!NIL_P(options)) {
+    VALUE track_whitespace_changes = rb_hash_lookup(options, rb_utf8_str_new_cstr("track_whitespace_changes"));
+    if (NIL_P(track_whitespace_changes)) {
+      track_whitespace_changes = rb_hash_lookup(options, ID2SYM(rb_intern("track_whitespace_changes")));
+    }
+    if (!NIL_P(track_whitespace_changes) && RTEST(track_whitespace_changes)) {
+      diff_options.track_whitespace_changes = true;
+    }
+  }
 
   if (!hb_allocator_init(&args.old_allocator, HB_ALLOCATOR_ARENA)) { return Qnil; }
 
@@ -529,7 +574,7 @@ static VALUE Herb_diff(int argc, VALUE* argv, VALUE self) {
     return Qnil;
   }
 
-  args.diff_result = herb_diff(args.old_root, args.new_root, &args.diff_allocator);
+  args.diff_result = herb_diff(args.old_root, args.new_root, &diff_options, &args.diff_allocator);
 
   return rb_ensure(diff_convert_body, (VALUE) &args, diff_cleanup, (VALUE) &args);
 }
@@ -548,12 +593,15 @@ __attribute__((__visibility__("default"))) void Init_herb(void) {
   rb_init_node_classes();
   rb_init_error_classes();
 
-  rb_define_singleton_method(mHerb, "parse", Herb_parse, -1);
-  rb_define_singleton_method(mHerb, "lex", Herb_lex, -1);
-  rb_define_singleton_method(mHerb, "extract_ruby", Herb_extract_ruby, -1);
-  rb_define_singleton_method(mHerb, "extract_html", Herb_extract_html, 1);
-  rb_define_singleton_method(mHerb, "arena_stats", Herb_arena_stats, -1);
-  rb_define_singleton_method(mHerb, "leak_check", Herb_leak_check, 1);
-  rb_define_singleton_method(mHerb, "version", Herb_version, 0);
-  rb_define_singleton_method(mHerb, "diff", Herb_diff, -1);
+  VALUE mBackend = rb_define_module_under(mHerb, "Backend");
+
+  rb_define_singleton_method(mBackend, "parse", Herb_parse, -1);
+  rb_define_singleton_method(mBackend, "lex", Herb_lex, -1);
+  rb_define_singleton_method(mBackend, "extract_ruby", Herb_extract_ruby, -1);
+  rb_define_singleton_method(mBackend, "extract_html", Herb_extract_html, 1);
+  rb_define_singleton_method(mBackend, "arena_stats", Herb_arena_stats, -1);
+  rb_define_singleton_method(mBackend, "leak_check", Herb_leak_check, 1);
+  rb_define_singleton_method(mBackend, "default_erb_openings", Herb_default_erb_openings, 0);
+  rb_define_singleton_method(mBackend, "version", Herb_version, 0);
+  rb_define_singleton_method(mBackend, "diff", Herb_diff, -1);
 }

data/ext/herb/extension_helpers.c

@@ -1,8 +1,12 @@
 #include <ruby.h>
+#include <ruby/encoding.h>
+
+#include <stdbool.h>
 
 #include "extension.h"
 #include "extension_helpers.h"
 #include "nodes.h"
+#include "parser_options_helpers.h"
 
 #include "../../src/include/herb.h"
 #include "../../src/include/lexer/token.h"
@@ -21,28 +25,54 @@ const char* check_string(VALUE value) {
   return RSTRING_PTR(value);
 }
 
+static ID id_line, id_column, id_start, id_end, id_from, id_to, id_value, id_range, id_location, id_type;
+static bool ast_value_ivar_ids_initialized = false;
+
+static void init_ast_value_ivar_ids(void) {
+  if (ast_value_ivar_ids_initialized) { return; }
+
+  id_line = rb_intern("@line");
+  id_column = rb_intern("@column");
+  id_start = rb_intern("@start");
+  id_end = rb_intern("@end");
+  id_from = rb_intern("@from");
+  id_to = rb_intern("@to");
+  id_value = rb_intern("@value");
+  id_range = rb_intern("@range");
+  id_location = rb_intern("@location");
+  id_type = rb_intern("@type");
+
+  ast_value_ivar_ids_initialized = true;
+}
+
 VALUE rb_position_from_c_struct(position_T position) {
-  VALUE args[2];
-  args[0] = UINT2NUM(position.line);
-  args[1] = UINT2NUM(position.column);
+  init_ast_value_ivar_ids();
 
-  return rb_class_new_instance(2, args, cPosition);
+  VALUE object = rb_obj_alloc(cPosition);
+  rb_ivar_set(object, id_line, UINT2NUM(position.line));
+  rb_ivar_set(object, id_column, UINT2NUM(position.column));
+
+  return object;
 }
 
 VALUE rb_location_from_c_struct(location_T location) {
-  VALUE args[2];
-  args[0] = rb_position_from_c_struct(location.start);
-  args[1] = rb_position_from_c_struct(location.end);
+  init_ast_value_ivar_ids();
+
+  VALUE object = rb_obj_alloc(cLocation);
+  rb_ivar_set(object, id_start, rb_position_from_c_struct(location.start));
+  rb_ivar_set(object, id_end, rb_position_from_c_struct(location.end));
 
-  return rb_class_new_instance(2, args, cLocation);
+  return object;
 }
 
 VALUE rb_range_from_c_struct(range_T range) {
-  VALUE args[2];
-  args[0] = UINT2NUM(range.from);
-  args[1] = UINT2NUM(range.to);
+  init_ast_value_ivar_ids();
+
+  VALUE object = rb_obj_alloc(cRange);
+  rb_ivar_set(object, id_from, UINT2NUM(range.from));
+  rb_ivar_set(object, id_to, UINT2NUM(range.to));
 
-  return rb_class_new_instance(2, args, cRange);
+  return object;
 }
 
 VALUE rb_string_from_hb_string(hb_string_T string) {
@@ -51,17 +81,42 @@ VALUE rb_string_from_hb_string(hb_string_T string) {
   return rb_utf8_str_new(string.data, string.length);
 }
 
-VALUE rb_token_from_c_struct(token_T* token) {
+VALUE rb_interned_string_from_hb_string(hb_string_T string) {
+  if (hb_string_is_null(string)) { return Qnil; }
+
+  return rb_enc_interned_str(string.data, string.length, rb_utf8_encoding());
+}
+
+static VALUE token_type_value_cache[TOKEN_EOF + 1] = { 0 };
+
+static VALUE rb_token_type_value(token_type_T type) {
+  if ((unsigned int) type > (unsigned int) TOKEN_EOF) {
+    return rb_interned_string_from_hb_string(token_type_to_string(type));
+  }
+
+  VALUE cached = token_type_value_cache[type];
+
+  if (cached == 0) {
+    cached = rb_interned_string_from_hb_string(token_type_to_string(type));
+    rb_gc_register_mark_object(cached);
+    token_type_value_cache[type] = cached;
+  }
+
+  return cached;
+}
+
+VALUE rb_token_from_c_struct(token_T* token, const parser_options_T* options) {
   if (!token) { return Qnil; }
 
-  VALUE value = rb_string_from_hb_string(token->value);
-  VALUE range = rb_range_from_c_struct(token->range);
-  VALUE location = rb_location_from_c_struct(token->location);
-  VALUE type = rb_string_from_hb_string(token_type_to_string(token->type));
+  init_ast_value_ivar_ids();
 
-  VALUE args[4] = { value, range, location, type };
+  VALUE object = rb_obj_alloc(cToken);
+  rb_ivar_set(object, id_value, rb_string_from_hb_string(token->value));
+  rb_ivar_set(object, id_range, options->track_locations ? rb_range_from_c_struct(token->range) : Qnil);
+  rb_ivar_set(object, id_location, options->track_locations ? rb_location_from_c_struct(token->location) : Qnil);
+  rb_ivar_set(object, id_type, rb_token_type_value(token->type));
 
-  return rb_class_new_instance(4, args, cToken);
+  return object;
 }
 
 VALUE create_lex_result(hb_array_T* tokens, VALUE source) {
@@ -71,7 +126,7 @@ VALUE create_lex_result(hb_array_T* tokens, VALUE source) {
 
   for (size_t i = 0; i < hb_array_size(tokens); i++) {
     token_T* token = hb_array_get(tokens, i);
-    if (token != NULL) { rb_ary_push(value, rb_token_from_c_struct(token)); }
+    if (token != NULL) { rb_ary_push(value, rb_token_from_c_struct(token, &HERB_DEFAULT_PARSER_OPTIONS)); }
   }
 
   VALUE args[4] = { value, source, warnings, errors };
@@ -80,33 +135,17 @@ VALUE create_lex_result(hb_array_T* tokens, VALUE source) {
 }
 
 VALUE create_parse_result(AST_DOCUMENT_NODE_T* root, VALUE source, const parser_options_T* options) {
-  VALUE value = rb_node_from_c_struct((AST_NODE_T*) root);
+  VALUE value = rb_node_from_c_struct((AST_NODE_T*) root, options);
   VALUE warnings = rb_ary_new();
   VALUE errors = rb_ary_new();
-
-  VALUE kwargs = rb_hash_new();
-  rb_hash_aset(kwargs, ID2SYM(rb_intern("strict")), options->strict ? Qtrue : Qfalse);
-  rb_hash_aset(kwargs, ID2SYM(rb_intern("track_whitespace")), options->track_whitespace ? Qtrue : Qfalse);
-  rb_hash_aset(kwargs, ID2SYM(rb_intern("analyze")), options->analyze ? Qtrue : Qfalse);
-  rb_hash_aset(kwargs, ID2SYM(rb_intern("action_view_helpers")), options->action_view_helpers ? Qtrue : Qfalse);
-  rb_hash_aset(kwargs, ID2SYM(rb_intern("transform_conditionals")), options->transform_conditionals ? Qtrue : Qfalse);
-  rb_hash_aset(kwargs, ID2SYM(rb_intern("render_nodes")), options->render_nodes ? Qtrue : Qfalse);
-  rb_hash_aset(kwargs, ID2SYM(rb_intern("strict_locals")), options->strict_locals ? Qtrue : Qfalse);
-  rb_hash_aset(kwargs, ID2SYM(rb_intern("prism_nodes")), options->prism_nodes ? Qtrue : Qfalse);
-  rb_hash_aset(kwargs, ID2SYM(rb_intern("prism_nodes_deep")), options->prism_nodes_deep ? Qtrue : Qfalse);
-  rb_hash_aset(kwargs, ID2SYM(rb_intern("prism_program")), options->prism_program ? Qtrue : Qfalse);
-  rb_hash_aset(kwargs, ID2SYM(rb_intern("timeout")), DBL2NUM((double) options->timeout_ms / 1000.0));
-
-  rb_hash_aset(
-    kwargs,
-    ID2SYM(rb_intern("max_errors")),
-    options->max_errors == 0 ? Qnil : UINT2NUM(options->max_errors)
-  );
+  VALUE kwargs = herb_build_parser_options_hash(options);
 
   VALUE parser_options_args[1] = { kwargs };
   VALUE parser_options = rb_class_new_instance_kw(1, parser_options_args, cParserOptions, RB_PASS_KEYWORDS);
 
-  VALUE args[5] = { value, source, warnings, errors, parser_options };
+  VALUE error_count = options->error_count != NULL ? UINT2NUM(*options->error_count) : Qnil;
+
+  VALUE args[6] = { value, source, warnings, errors, parser_options, error_count };
 
-  return rb_class_new_instance(5, args, cParseResult);
+  return rb_class_new_instance(6, args, cParseResult);
 }

data/ext/herb/extension_helpers.h

@@ -3,6 +3,8 @@
 
 #include <ruby.h>
 
+#include <stdbool.h>
+
 #include "../../src/include/herb.h"
 #include "../../src/include/lexer/token.h"
 #include "../../src/include/location/location.h"
@@ -11,11 +13,12 @@
 
 const char* check_string(VALUE value);
 VALUE rb_string_from_hb_string(hb_string_T string);
+VALUE rb_interned_string_from_hb_string(hb_string_T string);
 
 VALUE rb_position_from_c_struct(position_T position);
 VALUE rb_location_from_c_struct(location_T location);
 
-VALUE rb_token_from_c_struct(token_T* token);
+VALUE rb_token_from_c_struct(token_T* token, const parser_options_T* options);
 VALUE rb_range_from_c_struct(range_T range);
 
 VALUE create_lex_result(hb_array_T* tokens, VALUE source);