herb 0.8.10-arm-linux-gnu → 0.9.0-arm-linux-gnu

data/ext/herb/error_helpers.h

@@ -9,6 +9,7 @@
 
 #include <ruby.h>
 
+void rb_init_error_classes(void);
 VALUE rb_error_from_c_struct(ERROR_T* error);
 VALUE rb_errors_array_from_c_array(hb_array_T* array);
 

data/ext/herb/extconf.rb

@@ -1,9 +1,42 @@
 # frozen_string_literal: true
 
 require "mkmf"
+require_relative "../../lib/herb/bootstrap"
 
 extension_name = "herb"
 
+if Herb::Bootstrap.git_source? && !Herb::Bootstrap.templates_generated?
+  puts "Building from source — running bootstrap..."
+  Herb::Bootstrap.generate_templates
+
+  unless Herb::Bootstrap.prism_vendored?
+    prism_path = Herb::Bootstrap.find_prism_gem_path
+
+    abort <<~MSG unless prism_path
+      ERROR: Could not find Prism C source files.
+
+      When installing Herb from a git source, a git-sourced Prism is required
+      (the released gem does not include C source files).
+
+      Add it to your Gemfile before the herb git reference:
+
+        gem "prism", github: "ruby/prism", tag: "v1.9.0"
+        gem "herb", github: "...", branch: "..."
+
+      Then run `bundle install` again.
+    MSG
+
+    puts "Vendoring Prism from #{prism_path}..."
+    Herb::Bootstrap.vendor_prism(prism_gem_path: prism_path)
+  end
+
+  root_path = Herb::Bootstrap::ROOT_PATH
+  sha = `git -C #{root_path} rev-parse --short HEAD 2>/dev/null`.strip
+
+  $CFLAGS << " -DHERB_GIT_BUILD"
+  $CFLAGS << " -DHERB_GIT_SHA=\\\"#{sha}\\\"" unless sha.empty?
+end
+
 include_path = File.expand_path("../../src/include", __dir__)
 prism_path = File.expand_path("../../vendor/prism", __dir__)
 
@@ -11,6 +44,8 @@ prism_src_path = "#{prism_path}/src"
 prism_include_path = "#{prism_path}/include"
 
 $VPATH << "$(srcdir)/../../src"
+$VPATH << "$(srcdir)/../../src/analyze"
+$VPATH << "$(srcdir)/../../src/analyze/action_view"
 $VPATH << "$(srcdir)/../../src/util"
 $VPATH << prism_src_path
 $VPATH << "#{prism_src_path}/util"
@@ -21,41 +56,45 @@ $INCFLAGS << " -I#{prism_src_path}"
 $INCFLAGS << " -I#{prism_src_path}/util"
 
 $CFLAGS << " -fvisibility=hidden"
+$CFLAGS << " -DHERB_EXCLUDE_PRETTYPRINT"
+$CFLAGS << " -DPRISM_EXCLUDE_PRETTYPRINT"
+$CFLAGS << " -DPRISM_EXCLUDE_JSON"
+$CFLAGS << " -DPRISM_EXCLUDE_PACK"
 
 herb_src_files = Dir.glob("#{$srcdir}/../../src/**/*.c").map { |file| file.delete_prefix("../../../../ext/herb/") }.sort
 
-prism_main_files = %w[
-  diagnostic.c
-  encoding.c
-  node.c
-  options.c
-  pack.c
-  prettyprint.c
-  prism.c
-  regexp.c
-  serialize.c
-  static_literals.c
-  token_type.c
+prism_main_files = [
+  "diagnostic.c",
+  "encoding.c",
+  "node.c",
+  "options.c",
+  "pack.c",
+  "prettyprint.c",
+  "prism.c",
+  "regexp.c",
+  "serialize.c",
+  "static_literals.c",
+  "token_type.c"
 ]
 
-prism_util_files = %w[
-  pm_buffer.c
-  pm_char.c
-  pm_constant_pool.c
-  pm_integer.c
-  pm_list.c
-  pm_memchr.c
-  pm_newline_list.c
-  pm_string.c
-  pm_strncasecmp.c
-  pm_strpbrk.c
+prism_util_files = [
+  "pm_buffer.c",
+  "pm_char.c",
+  "pm_constant_pool.c",
+  "pm_integer.c",
+  "pm_list.c",
+  "pm_memchr.c",
+  "pm_newline_list.c",
+  "pm_string.c",
+  "pm_strncasecmp.c",
+  "pm_strpbrk.c"
 ]
 
-core_src_files = %w[
-  extension.c
-  nodes.c
-  error_helpers.c
-  extension_helpers.c
+core_src_files = [
+  "extension.c",
+  "nodes.c",
+  "error_helpers.c",
+  "extension_helpers.c"
 ]
 
 $srcs = core_src_files + herb_src_files + prism_main_files + prism_util_files

data/ext/herb/extension.c

@@ -1,12 +1,13 @@
 #include <ruby.h>
 
+#include "../../src/include/util/hb_allocator.h"
+#include "../../src/include/util/hb_arena_debug.h"
+
 #include "error_helpers.h"
 #include "extension.h"
 #include "extension_helpers.h"
 #include "nodes.h"
 
-#include "../../src/include/analyze.h"
-
 VALUE mHerb;
 VALUE cPosition;
 VALUE cLocation;
@@ -15,29 +16,96 @@ VALUE cToken;
 VALUE cResult;
 VALUE cLexResult;
 VALUE cParseResult;
+VALUE cParserOptions;
+
+typedef struct {
+  AST_DOCUMENT_NODE_T* root;
+  VALUE source;
+  const parser_options_T* parser_options;
+  hb_allocator_T allocator;
+} parse_args_T;
+
+typedef struct {
+  hb_array_T* tokens;
+  VALUE source;
+  hb_allocator_T allocator;
+} lex_args_T;
+
+typedef struct {
+  char* buffer_value;
+  hb_allocator_T allocator;
+} buffer_args_T;
+
+static VALUE parse_convert_body(VALUE arg) {
+  parse_args_T* args = (parse_args_T*) arg;
+
+  return create_parse_result(args->root, args->source, args->parser_options);
+}
 
-static VALUE Herb_lex(VALUE self, VALUE source) {
-  char* string = (char*) check_string(source);
+static VALUE parse_cleanup(VALUE arg) {
+  parse_args_T* args = (parse_args_T*) arg;
 
-  hb_array_T* tokens = herb_lex(string);
+  if (args->root != NULL) { ast_node_free((AST_NODE_T*) args->root, &args->allocator); }
 
-  VALUE result = create_lex_result(tokens, source);
+  hb_allocator_destroy(&args->allocator);
 
-  herb_free_tokens(&tokens);
+  return Qnil;
+}
 
-  return result;
+static VALUE lex_convert_body(VALUE arg) {
+  lex_args_T* args = (lex_args_T*) arg;
+
+  return create_lex_result(args->tokens, args->source);
 }
 
-static VALUE Herb_lex_file(VALUE self, VALUE path) {
-  char* file_path = (char*) check_string(path);
-  hb_array_T* tokens = herb_lex_file(file_path);
+static VALUE lex_cleanup(VALUE arg) {
+  lex_args_T* args = (lex_args_T*) arg;
 
-  VALUE source_value = read_file_to_ruby_string(file_path);
-  VALUE result = create_lex_result(tokens, source_value);
+  if (args->tokens != NULL) { herb_free_tokens(&args->tokens, &args->allocator); }
 
-  herb_free_tokens(&tokens);
+  hb_allocator_destroy(&args->allocator);
 
-  return result;
+  return Qnil;
+}
+
+static VALUE buffer_to_string_body(VALUE arg) {
+  buffer_args_T* args = (buffer_args_T*) arg;
+
+  return rb_utf8_str_new_cstr(args->buffer_value);
+}
+
+static VALUE buffer_cleanup(VALUE arg) {
+  buffer_args_T* args = (buffer_args_T*) arg;
+
+  hb_allocator_dealloc(&args->allocator, args->buffer_value);
+  hb_allocator_destroy(&args->allocator);
+
+  return Qnil;
+}
+
+static VALUE Herb_lex(int argc, VALUE* argv, VALUE self) {
+  VALUE source, options;
+  rb_scan_args(argc, argv, "1:", &source, &options);
+
+  char* string = (char*) check_string(source);
+  bool print_arena_stats = false;
+
+  if (!NIL_P(options)) {
+    VALUE arena_stats = rb_hash_lookup(options, rb_utf8_str_new_cstr("arena_stats"));
+    if (NIL_P(arena_stats)) { arena_stats = rb_hash_lookup(options, ID2SYM(rb_intern("arena_stats"))); }
+    if (!NIL_P(arena_stats) && RTEST(arena_stats)) { print_arena_stats = true; }
+  }
+
+  lex_args_T args = { 0 };
+  args.source = source;
+
+  if (!hb_allocator_init(&args.allocator, HB_ALLOCATOR_ARENA)) { return Qnil; }
+
+  args.tokens = herb_lex(string, &args.allocator);
+
+  if (print_arena_stats) { hb_arena_print_stats((hb_arena_T*) args.allocator.context); }
+
+  return rb_ensure(lex_convert_body, (VALUE) &args, lex_cleanup, (VALUE) &args);
 }
 
 static VALUE Herb_parse(int argc, VALUE* argv, VALUE self) {
@@ -45,72 +113,244 @@ static VALUE Herb_parse(int argc, VALUE* argv, VALUE self) {
   rb_scan_args(argc, argv, "1:", &source, &options);
 
   char* string = (char*) check_string(source);
+  bool print_arena_stats = false;
 
-  parser_options_T* parser_options = NULL;
-  parser_options_T opts = { 0 };
+  parser_options_T parser_options = HERB_DEFAULT_PARSER_OPTIONS;
 
   if (!NIL_P(options)) {
     VALUE track_whitespace = rb_hash_lookup(options, rb_utf8_str_new_cstr("track_whitespace"));
     if (NIL_P(track_whitespace)) { track_whitespace = rb_hash_lookup(options, ID2SYM(rb_intern("track_whitespace"))); }
+    if (!NIL_P(track_whitespace) && RTEST(track_whitespace)) { parser_options.track_whitespace = true; }
+
+    VALUE analyze = rb_hash_lookup(options, rb_utf8_str_new_cstr("analyze"));
+    if (NIL_P(analyze)) { analyze = rb_hash_lookup(options, ID2SYM(rb_intern("analyze"))); }
+    if (!NIL_P(analyze) && !RTEST(analyze)) { parser_options.analyze = false; }
 
-    if (!NIL_P(track_whitespace) && RTEST(track_whitespace)) {
-      opts.track_whitespace = true;
-      parser_options = &opts;
+    VALUE strict = rb_hash_lookup(options, rb_utf8_str_new_cstr("strict"));
+    if (NIL_P(strict)) { strict = rb_hash_lookup(options, ID2SYM(rb_intern("strict"))); }
+    if (!NIL_P(strict)) { parser_options.strict = RTEST(strict); }
+
+    VALUE action_view_helpers = rb_hash_lookup(options, rb_utf8_str_new_cstr("action_view_helpers"));
+    if (NIL_P(action_view_helpers)) {
+      action_view_helpers = rb_hash_lookup(options, ID2SYM(rb_intern("action_view_helpers")));
     }
+    if (!NIL_P(action_view_helpers) && RTEST(action_view_helpers)) { parser_options.action_view_helpers = true; }
+
+    VALUE prism_nodes = rb_hash_lookup(options, rb_utf8_str_new_cstr("prism_nodes"));
+    if (NIL_P(prism_nodes)) { prism_nodes = rb_hash_lookup(options, ID2SYM(rb_intern("prism_nodes"))); }
+    if (!NIL_P(prism_nodes) && RTEST(prism_nodes)) { parser_options.prism_nodes = true; }
+
+    VALUE prism_nodes_deep = rb_hash_lookup(options, rb_utf8_str_new_cstr("prism_nodes_deep"));
+    if (NIL_P(prism_nodes_deep)) { prism_nodes_deep = rb_hash_lookup(options, ID2SYM(rb_intern("prism_nodes_deep"))); }
+    if (!NIL_P(prism_nodes_deep) && RTEST(prism_nodes_deep)) { parser_options.prism_nodes_deep = true; }
+
+    VALUE prism_program = rb_hash_lookup(options, rb_utf8_str_new_cstr("prism_program"));
+    if (NIL_P(prism_program)) { prism_program = rb_hash_lookup(options, ID2SYM(rb_intern("prism_program"))); }
+    if (!NIL_P(prism_program) && RTEST(prism_program)) { parser_options.prism_program = true; }
+
+    VALUE arena_stats = rb_hash_lookup(options, rb_utf8_str_new_cstr("arena_stats"));
+    if (NIL_P(arena_stats)) { arena_stats = rb_hash_lookup(options, ID2SYM(rb_intern("arena_stats"))); }
+    if (!NIL_P(arena_stats) && RTEST(arena_stats)) { print_arena_stats = true; }
   }
 
-  AST_DOCUMENT_NODE_T* root = herb_parse(string, parser_options);
+  parse_args_T args = { 0 };
+  args.source = source;
+  args.parser_options = &parser_options;
 
-  herb_analyze_parse_tree(root, string);
+  if (!hb_allocator_init(&args.allocator, HB_ALLOCATOR_ARENA)) { return Qnil; }
 
-  VALUE result = create_parse_result(root, source);
+  args.root = herb_parse(string, &parser_options, &args.allocator);
 
-  ast_node_free((AST_NODE_T*) root);
+  if (print_arena_stats) { hb_arena_print_stats((hb_arena_T*) args.allocator.context); }
 
-  return result;
+  return rb_ensure(parse_convert_body, (VALUE) &args, parse_cleanup, (VALUE) &args);
 }
 
-static VALUE Herb_parse_file(VALUE self, VALUE path) {
-  char* file_path = (char*) check_string(path);
+static VALUE Herb_extract_ruby(int argc, VALUE* argv, VALUE self) {
+  VALUE source, options;
+  rb_scan_args(argc, argv, "1:", &source, &options);
 
-  VALUE source_value = read_file_to_ruby_string(file_path);
-  char* string = (char*) check_string(source_value);
+  char* string = (char*) check_string(source);
 
-  AST_DOCUMENT_NODE_T* root = herb_parse(string, NULL);
+  hb_allocator_T allocator;
+  if (!hb_allocator_init(&allocator, HB_ALLOCATOR_ARENA)) { return Qnil; }
 
-  herb_analyze_parse_tree(root, string);
+  hb_buffer_T output;
+  if (!hb_buffer_init(&output, strlen(string), &allocator)) { return Qnil; }
 
-  VALUE result = create_parse_result(root, source_value);
+  herb_extract_ruby_options_T extract_options = HERB_EXTRACT_RUBY_DEFAULT_OPTIONS;
 
-  ast_node_free((AST_NODE_T*) root);
+  if (!NIL_P(options)) {
+    VALUE semicolons_value = rb_hash_lookup(options, rb_utf8_str_new_cstr("semicolons"));
+    if (NIL_P(semicolons_value)) { semicolons_value = rb_hash_lookup(options, ID2SYM(rb_intern("semicolons"))); }
+    if (!NIL_P(semicolons_value)) { extract_options.semicolons = RTEST(semicolons_value); }
 
-  return result;
+    VALUE comments_value = rb_hash_lookup(options, rb_utf8_str_new_cstr("comments"));
+    if (NIL_P(comments_value)) { comments_value = rb_hash_lookup(options, ID2SYM(rb_intern("comments"))); }
+    if (!NIL_P(comments_value)) { extract_options.comments = RTEST(comments_value); }
+
+    VALUE preserve_positions_value = rb_hash_lookup(options, rb_utf8_str_new_cstr("preserve_positions"));
+    if (NIL_P(preserve_positions_value)) {
+      preserve_positions_value = rb_hash_lookup(options, ID2SYM(rb_intern("preserve_positions")));
+    }
+    if (!NIL_P(preserve_positions_value)) { extract_options.preserve_positions = RTEST(preserve_positions_value); }
+  }
+
+  herb_extract_ruby_to_buffer_with_options(string, &output, &extract_options, &allocator);
+
+  buffer_args_T args = { .buffer_value = output.value, .allocator = allocator };
+
+  return rb_ensure(buffer_to_string_body, (VALUE) &args, buffer_cleanup, (VALUE) &args);
 }
 
-static VALUE Herb_extract_ruby(VALUE self, VALUE source) {
+static VALUE Herb_extract_html(VALUE self, VALUE source) {
   char* string = (char*) check_string(source);
+
+  hb_allocator_T allocator;
+  if (!hb_allocator_init(&allocator, HB_ALLOCATOR_ARENA)) { return Qnil; }
+
   hb_buffer_T output;
+  if (!hb_buffer_init(&output, strlen(string), &allocator)) { return Qnil; }
 
-  if (!hb_buffer_init(&output, strlen(string))) { return Qnil; }
+  herb_extract_html_to_buffer(string, &output, &allocator);
 
-  herb_extract_ruby_to_buffer(string, &output);
+  buffer_args_T args = { .buffer_value = output.value, .allocator = allocator };
 
-  VALUE result = rb_utf8_str_new_cstr(output.value);
-  free(output.value);
+  return rb_ensure(buffer_to_string_body, (VALUE) &args, buffer_cleanup, (VALUE) &args);
+}
 
-  return result;
+static VALUE Herb_arena_stats(int argc, VALUE* argv, VALUE self) {
+  VALUE source, options;
+  rb_scan_args(argc, argv, "1:", &source, &options);
+
+  char* string = (char*) check_string(source);
+
+  parser_options_T parser_options = HERB_DEFAULT_PARSER_OPTIONS;
+
+  if (!NIL_P(options)) {
+    VALUE track_whitespace = rb_hash_lookup(options, rb_utf8_str_new_cstr("track_whitespace"));
+    if (NIL_P(track_whitespace)) { track_whitespace = rb_hash_lookup(options, ID2SYM(rb_intern("track_whitespace"))); }
+    if (!NIL_P(track_whitespace) && RTEST(track_whitespace)) { parser_options.track_whitespace = true; }
+
+    VALUE analyze = rb_hash_lookup(options, rb_utf8_str_new_cstr("analyze"));
+    if (NIL_P(analyze)) { analyze = rb_hash_lookup(options, ID2SYM(rb_intern("analyze"))); }
+    if (!NIL_P(analyze) && !RTEST(analyze)) { parser_options.analyze = false; }
+
+    VALUE strict = rb_hash_lookup(options, rb_utf8_str_new_cstr("strict"));
+    if (NIL_P(strict)) { strict = rb_hash_lookup(options, ID2SYM(rb_intern("strict"))); }
+    if (!NIL_P(strict)) { parser_options.strict = RTEST(strict); }
+  }
+
+  hb_allocator_T allocator;
+  if (!hb_allocator_init(&allocator, HB_ALLOCATOR_ARENA)) { return Qnil; }
+
+  AST_DOCUMENT_NODE_T* root = herb_parse(string, &parser_options, &allocator);
+
+  hb_arena_stats_T stats = hb_arena_get_stats((hb_arena_T*) allocator.context);
+
+  if (root != NULL) { ast_node_free((AST_NODE_T*) root, &allocator); }
+  hb_allocator_destroy(&allocator);
+
+  VALUE hash = rb_hash_new();
+  rb_hash_aset(hash, ID2SYM(rb_intern("pages")), SIZET2NUM(stats.pages));
+  rb_hash_aset(hash, ID2SYM(rb_intern("total_capacity")), SIZET2NUM(stats.total_capacity));
+  rb_hash_aset(hash, ID2SYM(rb_intern("total_used")), SIZET2NUM(stats.total_used));
+  rb_hash_aset(hash, ID2SYM(rb_intern("total_available")), SIZET2NUM(stats.total_available));
+  rb_hash_aset(hash, ID2SYM(rb_intern("allocations")), SIZET2NUM(stats.allocations));
+  rb_hash_aset(hash, ID2SYM(rb_intern("fragmentation")), SIZET2NUM(stats.fragmentation));
+  rb_hash_aset(hash, ID2SYM(rb_intern("default_page_size")), SIZET2NUM(stats.default_page_size));
+
+  return hash;
 }
 
-static VALUE Herb_extract_html(VALUE self, VALUE source) {
+static VALUE make_tracking_hash(hb_allocator_tracking_stats_T* stats) {
+  VALUE hash = rb_hash_new();
+  rb_hash_aset(hash, ID2SYM(rb_intern("allocations")), SIZET2NUM(stats->allocation_count));
+  rb_hash_aset(hash, ID2SYM(rb_intern("deallocations")), SIZET2NUM(stats->deallocation_count));
+  rb_hash_aset(hash, ID2SYM(rb_intern("bytes_allocated")), SIZET2NUM(stats->bytes_allocated));
+  rb_hash_aset(hash, ID2SYM(rb_intern("bytes_deallocated")), SIZET2NUM(stats->bytes_deallocated));
+  rb_hash_aset(hash, ID2SYM(rb_intern("untracked_deallocations")), SIZET2NUM(stats->untracked_deallocation_count));
+
+  VALUE leaks = rb_ary_new();
+  for (size_t i = 0; i < stats->buckets_capacity; i++) {
+    if (stats->buckets[i].pointer != NULL && stats->buckets[i].pointer != (void*) 1) {
+      rb_ary_push(leaks, SIZET2NUM(stats->buckets[i].size));
+    }
+  }
+  rb_hash_aset(hash, ID2SYM(rb_intern("leaks")), leaks);
+
+  VALUE untracked = rb_ary_new_capa((long) stats->untracked_pointers_size);
+  for (size_t i = 0; i < stats->untracked_pointers_size; i++) {
+    rb_ary_push(untracked, rb_sprintf("%p", stats->untracked_pointers[i]));
+  }
+  rb_hash_aset(hash, ID2SYM(rb_intern("untracked_pointers")), untracked);
+
+  return hash;
+}
+
+static VALUE Herb_leak_check(VALUE self, VALUE source) {
   char* string = (char*) check_string(source);
-  hb_buffer_T output;
+  VALUE result = rb_hash_new();
+
+  {
+    hb_allocator_T allocator;
+    if (!hb_allocator_init(&allocator, HB_ALLOCATOR_TRACKING)) { return Qnil; }
+
+    hb_array_T* tokens = herb_lex(string, &allocator);
+    if (tokens != NULL) { herb_free_tokens(&tokens, &allocator); }
+
+    hb_allocator_tracking_stats_T* stats = hb_allocator_tracking_stats(&allocator);
+    rb_hash_aset(result, ID2SYM(rb_intern("lex")), make_tracking_hash(stats));
+
+    hb_allocator_destroy(&allocator);
+  }
+
+  {
+    hb_allocator_T allocator;
+    if (!hb_allocator_init(&allocator, HB_ALLOCATOR_TRACKING)) { return Qnil; }
+
+    parser_options_T parser_options = HERB_DEFAULT_PARSER_OPTIONS;
+    AST_DOCUMENT_NODE_T* root = herb_parse(string, &parser_options, &allocator);
+    if (root != NULL) { ast_node_free((AST_NODE_T*) root, &allocator); }
+
+    hb_allocator_tracking_stats_T* stats = hb_allocator_tracking_stats(&allocator);
+    rb_hash_aset(result, ID2SYM(rb_intern("parse")), make_tracking_hash(stats));
+
+    hb_allocator_destroy(&allocator);
+  }
 
-  if (!hb_buffer_init(&output, strlen(string))) { return Qnil; }
+  {
+    hb_allocator_T allocator;
+    if (!hb_allocator_init(&allocator, HB_ALLOCATOR_TRACKING)) { return Qnil; }
 
-  herb_extract_html_to_buffer(string, &output);
+    hb_buffer_T output;
+    if (!hb_buffer_init(&output, strlen(string), &allocator)) { return Qnil; }
 
-  VALUE result = rb_utf8_str_new_cstr(output.value);
-  free(output.value);
+    herb_extract_ruby_options_T extract_options = HERB_EXTRACT_RUBY_DEFAULT_OPTIONS;
+    herb_extract_ruby_to_buffer_with_options(string, &output, &extract_options, &allocator);
+
+    hb_allocator_tracking_stats_T* stats = hb_allocator_tracking_stats(&allocator);
+    rb_hash_aset(result, ID2SYM(rb_intern("extract_ruby")), make_tracking_hash(stats));
+
+    hb_buffer_free(&output);
+    hb_allocator_destroy(&allocator);
+  }
+
+  {
+    hb_allocator_T allocator;
+    if (!hb_allocator_init(&allocator, HB_ALLOCATOR_TRACKING)) { return Qnil; }
+
+    hb_buffer_T output;
+    if (!hb_buffer_init(&output, strlen(string), &allocator)) { return Qnil; }
+
+    herb_extract_html_to_buffer(string, &output, &allocator);
+
+    hb_allocator_tracking_stats_T* stats = hb_allocator_tracking_stats(&allocator);
+    rb_hash_aset(result, ID2SYM(rb_intern("extract_html")), make_tracking_hash(stats));
+
+    hb_buffer_free(&output);
+    hb_allocator_destroy(&allocator);
+  }
 
   return result;
 }
@@ -119,9 +359,31 @@ static VALUE Herb_version(VALUE self) {
   VALUE gem_version = rb_const_get(self, rb_intern("VERSION"));
   VALUE libherb_version = rb_utf8_str_new_cstr(herb_version());
   VALUE libprism_version = rb_utf8_str_new_cstr(herb_prism_version());
-  VALUE format_string = rb_utf8_str_new_cstr("herb gem v%s, libprism v%s, libherb v%s (Ruby C native extension)");
+
+#ifdef HERB_GIT_BUILD
+#  ifdef HERB_GIT_SHA
+  VALUE format_string = rb_utf8_str_new_cstr(
+    "herb gem " HERB_GIT_SHA ", libprism v%s, libherb " HERB_GIT_SHA " (Ruby C native extension, built from source)"
+  );
+
+  return rb_funcall(rb_mKernel, rb_intern("sprintf"), 2, format_string, libprism_version);
+#  else
+  VALUE format_string =
+    rb_utf8_str_new_cstr("herb gem v%s, libprism v%s, libherb v%s (Ruby C native extension, built from source)");
 
   return rb_funcall(rb_mKernel, rb_intern("sprintf"), 4, format_string, gem_version, libprism_version, libherb_version);
+#  endif
+#else
+  return rb_funcall(
+    rb_mKernel,
+    rb_intern("sprintf"),
+    4,
+    rb_utf8_str_new_cstr("herb gem v%s, libprism v%s, libherb v%s (Ruby C native extension)"),
+    gem_version,
+    libprism_version,
+    libherb_version
+  );
+#endif
 }
 
 __attribute__((__visibility__("default"))) void Init_herb(void) {
@@ -133,12 +395,16 @@ __attribute__((__visibility__("default"))) void Init_herb(void) {
   cResult = rb_define_class_under(mHerb, "Result", rb_cObject);
   cLexResult = rb_define_class_under(mHerb, "LexResult", cResult);
   cParseResult = rb_define_class_under(mHerb, "ParseResult", cResult);
+  cParserOptions = rb_define_class_under(mHerb, "ParserOptions", rb_cObject);
+
+  rb_init_node_classes();
+  rb_init_error_classes();
 
   rb_define_singleton_method(mHerb, "parse", Herb_parse, -1);
-  rb_define_singleton_method(mHerb, "lex", Herb_lex, 1);
-  rb_define_singleton_method(mHerb, "parse_file", Herb_parse_file, 1);
-  rb_define_singleton_method(mHerb, "lex_file", Herb_lex_file, 1);
-  rb_define_singleton_method(mHerb, "extract_ruby", Herb_extract_ruby, 1);
+  rb_define_singleton_method(mHerb, "lex", Herb_lex, -1);
+  rb_define_singleton_method(mHerb, "extract_ruby", Herb_extract_ruby, -1);
   rb_define_singleton_method(mHerb, "extract_html", Herb_extract_html, 1);
+  rb_define_singleton_method(mHerb, "arena_stats", Herb_arena_stats, -1);
+  rb_define_singleton_method(mHerb, "leak_check", Herb_leak_check, 1);
   rb_define_singleton_method(mHerb, "version", Herb_version, 0);
 }

data/ext/herb/extension.h

@@ -11,5 +11,6 @@ extern VALUE cToken;
 extern VALUE cResult;
 extern VALUE cLexResult;
 extern VALUE cParseResult;
+extern VALUE cParserOptions;
 
 #endif

data/ext/herb/extension_helpers.c

@@ -5,10 +5,11 @@
 #include "nodes.h"
 
 #include "../../src/include/herb.h"
-#include "../../src/include/io.h"
 #include "../../src/include/location.h"
 #include "../../src/include/position.h"
 #include "../../src/include/token.h"
+#include "../../src/include/util/hb_allocator.h"
+#include "../../src/include/util/hb_string.h"
 
 const char* check_string(VALUE value) {
   if (NIL_P(value)) { return NULL; }
@@ -44,14 +45,19 @@ VALUE rb_range_from_c_struct(range_T range) {
   return rb_class_new_instance(2, args, cRange);
 }
 
+VALUE rb_string_from_hb_string(hb_string_T string) {
+  if (hb_string_is_null(string)) { return Qnil; }
+
+  return rb_utf8_str_new(string.data, string.length);
+}
+
 VALUE rb_token_from_c_struct(token_T* token) {
   if (!token) { return Qnil; }
 
-  VALUE value = token->value ? rb_utf8_str_new_cstr(token->value) : Qnil;
-
+  VALUE value = rb_string_from_hb_string(token->value);
   VALUE range = rb_range_from_c_struct(token->range);
   VALUE location = rb_location_from_c_struct(token->location);
-  VALUE type = rb_utf8_str_new_cstr(token_type_to_string(token->type));
+  VALUE type = rb_string_from_hb_string(token_type_to_string(token->type));
 
   VALUE args[4] = { value, range, location, type };
 
@@ -73,21 +79,24 @@ VALUE create_lex_result(hb_array_T* tokens, VALUE source) {
   return rb_class_new_instance(4, args, cLexResult);
 }
 
-VALUE create_parse_result(AST_DOCUMENT_NODE_T* root, VALUE source) {
+VALUE create_parse_result(AST_DOCUMENT_NODE_T* root, VALUE source, const parser_options_T* options) {
   VALUE value = rb_node_from_c_struct((AST_NODE_T*) root);
   VALUE warnings = rb_ary_new();
   VALUE errors = rb_ary_new();
 
-  VALUE args[4] = { value, source, warnings, errors };
-
-  return rb_class_new_instance(4, args, cParseResult);
-}
+  VALUE kwargs = rb_hash_new();
+  rb_hash_aset(kwargs, ID2SYM(rb_intern("strict")), options->strict ? Qtrue : Qfalse);
+  rb_hash_aset(kwargs, ID2SYM(rb_intern("track_whitespace")), options->track_whitespace ? Qtrue : Qfalse);
+  rb_hash_aset(kwargs, ID2SYM(rb_intern("analyze")), options->analyze ? Qtrue : Qfalse);
+  rb_hash_aset(kwargs, ID2SYM(rb_intern("action_view_helpers")), options->action_view_helpers ? Qtrue : Qfalse);
+  rb_hash_aset(kwargs, ID2SYM(rb_intern("prism_nodes")), options->prism_nodes ? Qtrue : Qfalse);
+  rb_hash_aset(kwargs, ID2SYM(rb_intern("prism_nodes_deep")), options->prism_nodes_deep ? Qtrue : Qfalse);
+  rb_hash_aset(kwargs, ID2SYM(rb_intern("prism_program")), options->prism_program ? Qtrue : Qfalse);
 
-VALUE read_file_to_ruby_string(const char* file_path) {
-  char* source = herb_read_file(file_path);
-  VALUE source_value = rb_utf8_str_new_cstr(source);
+  VALUE parser_options_args[1] = { kwargs };
+  VALUE parser_options = rb_class_new_instance_kw(1, parser_options_args, cParserOptions, RB_PASS_KEYWORDS);
 
-  free(source);
+  VALUE args[5] = { value, source, warnings, errors, parser_options };
 
-  return source_value;
+  return rb_class_new_instance(5, args, cParseResult);
 }