natalie_parser 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: '048e048ede42652f8a81298a3310313f8bf9929ba5cd952a89d3269d8333b363'
+  data.tar.gz: 5c2388c8125b1444c454c0ee3a0d4a07f305cc3fd668d8e03dbc4404201a50f7
+SHA512:
+  metadata.gz: 55932c3dc24ceeeab109a4a980ddf96babf276ced15dffb68d7c013ca5cc246d2551b59acc94f82c1ec5c0073859267ace3a264a3d8b9470465672ea5aebb59d
+  data.tar.gz: bcbaaad396877bcbf6453e8d55ad3f46dcb68ba09bcfe2f6cf0f1fa6d96966c646f530e3872d77736ef40b5818feed888d69fe33e62beeb95dcfd1c6f5115203

data/CHANGELOG.md

@@ -0,0 +1,22 @@
+# Changelog
+
+## 1.0.0 (2022-06-03)
+
+### Summary
+
+This is the initial public release. The 1.0 milestone was chosen as soon
+as NatalieParser was useful for integration back with the upstream Natalie
+compiler project, i.e. it could fully replace RubyParser as the parser
+in use by Natalie.
+
+That is not to say that NatalieParser is _complete_ -- it is merely _useful_.
+
+These are the features known to still be missing in this release:
+
+- [ ] Support different source encodings
+- [ ] Support more of the Ruby 3.0 syntax
+  - [ ] Argument forwarding (`...`)
+  - [ ] Pattern matching
+  - [ ] Numbered block parameters (`_1`, `_2`, etc.)
+  - [ ] Non-ASCII identifiers
+  - [ ] Rational and Complex literals (`1r` and `2i`)

data/Dockerfile

@@ -0,0 +1,26 @@
+ARG IMAGE=ruby:3.0
+FROM $IMAGE
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y -q build-essential clang
+RUN gem install bundler --no-doc
+
+ENV LC_ALL C.UTF-8
+
+WORKDIR natalie_parser
+
+COPY Gemfile /natalie_parser/ 
+RUN bundle install
+
+ARG CC=gcc
+ENV CC=$CC
+ARG CXX=g++
+ENV CXX=$CXX
+
+COPY Rakefile Rakefile
+COPY ext ext
+COPY lib lib
+COPY src src
+COPY include include
+RUN rake
+
+COPY test test

data/Gemfile

@@ -0,0 +1,10 @@
+# NOTE: This Gemfile is used for testing the project.
+# These dependencies are not needed to consume the library as a C extension for MRI.
+
+source 'https://rubygems.org'
+
+gem 'minitest'
+gem 'minitest-focus'
+gem 'minitest-reporters'
+gem 'ruby_parser'
+gem 'rake'

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 Tim Morgan and contributors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,55 @@
+# Natalie Parser
+
+[![github build status](https://github.com/natalie-lang/natalie_parser/actions/workflows/build.yml/badge.svg)](https://github.com/natalie-lang/natalie_parser/actions?query=workflow%3ABuild+branch%3Amaster)
+[![MIT License](https://img.shields.io/badge/license-MIT-blue)](https://github.com/natalie-lang/natalie_parser/blob/master/LICENSE)
+
+This is a parser for the Ruby programming language, written in C++.
+It was extracted from the [Natalie](https://github.com/natalie-lang/natalie) project.
+
+You can use this library directly from a C/C++ project, or you can
+build it as a Ruby gem and use it from Ruby itself.
+
+We are currently targeting Ruby 3.0 syntax, but that will probably
+change over time, depending on what things we want to support and
+what kind of help we get from the community.
+
+NOTE: This project is still very new and there are certainly bugs.
+See the list below for things we already know about, but expect there
+are more we don't know about yet. **We don't recommend you use this in
+production applications.**
+
+## To Do
+
+- [x] Parse the [Natalie](https://github.com/natalie-lang/natalie) compiler and standard library
+- [x] Pass (mostly) the [RubyParser](https://github.com/seattlerb/ruby_parser) test suite
+- [ ] Support different source encodings
+- [ ] Support more of the Ruby 3.0 syntax
+  - [x] "Endless" method definition (`def foo = bar`)
+  - [ ] Argument forwarding (`...`)
+  - [ ] Pattern matching
+  - [ ] Numbered block parameters (`_1`, `_2`, etc.)
+  - [ ] Non-ASCII identifiers
+  - [ ] Rational and Complex literals (`1r` and `2i`)
+
+## Development
+
+```sh
+rake
+ruby -I lib:ext -r natalie_parser -e "p NatalieParser.parse('1 + 2')"
+# => s(:block, s(:call, s(:lit, 1), :+, s(:lit, 2)))
+```
+
+### Running Tests
+
+```sh
+rake test
+```
+
+## Copyright & License
+
+Natalie is copyright 2022, Tim Morgan and contributors. Natalie is licensed
+under the MIT License; see the `LICENSE` file in this directory for the full text.
+
+### Note about Outside Sources
+
+The file `test/test_ruby_parser.rb` is copyright Ryan Davis and is licensed MIT.

data/Rakefile

@@ -0,0 +1,242 @@
+task default: :build
+
+desc 'Build Natalie Parser library and MRI C extension'
+task build: %i[
+  bundle_install
+  build_dir
+  library
+  parser_c_ext
+  write_compile_database
+]
+
+so_ext = RUBY_PLATFORM =~ /darwin/ ? 'bundle' : 'so'
+
+desc 'Build Natalie Parser library'
+task library: [:build_dir, "build/libnatalie_parser.a"]
+
+desc 'Build Natalie Parser MRI C extension'
+task parser_c_ext: [:build_dir, "ext/natalie_parser/natalie_parser.#{so_ext}"]
+
+desc 'Remove temporary files created during build'
+task :clean do
+  Rake::FileList[%w[
+    build/build.log
+    build/*.o
+    build/node
+    build/asan_test
+    ext/natalie_parser/*.{h,log,o}
+  ]].each { |path| rm_rf path if File.exist?(path) }
+end
+
+desc 'Remove all generated files'
+task :clobber do
+  Rake::FileList[%w[
+    build
+    ext/natalie_parser/*.{so,bundle,h,log,o}
+  ]].each { |path| rm_rf path if File.exist?(path) }
+end
+
+task distclean: :clobber
+
+desc 'Run the test suite'
+task test: [:build, 'build/asan_test'] do
+  sh 'bundle exec ruby test/all.rb'
+  sh 'build/asan_test'
+  sh 'bundle exec ruby test/test_ruby_parser.rb'
+end
+
+desc 'Run test test suite when changes are made (requires entr binary)'
+task :watch do
+  files = Rake::FileList['**/*.cpp', '**/*.hpp', '**/*.rb']
+  sh "ls #{files} | entr -c -s 'rake test'"
+end
+
+desc 'Show line counts for the project'
+task :cloc do
+  sh 'cloc include lib src test'
+end
+
+desc 'Generate tags file for development'
+task :ctags do
+  sh 'ctags -R --exclude=.cquery_cache --exclude=ext --exclude=build --append=no .'
+end
+task tags: :ctags
+
+desc 'Format C++ code with clang-format'
+task :format do
+  sh "find include -type f -name '*.hpp' -exec clang-format -i --style=file {} +"
+  sh "find src -type f -name '*.cpp' -exec clang-format -i --style=file {} +"
+end
+
+desc 'Show TODO and FIXME comments in the project'
+task :todo do
+  sh "egrep -r 'FIXME|TODO' src include lib"
+end
+
+desc 'Run the benchmark script'
+task benchmark: :build do
+  require_relative './test/benchmark'
+end
+
+# # # # Docker Tasks (used for CI) # # # #
+
+DOCKER_FLAGS =
+  if !ENV['CI'] && STDOUT.isatty
+    '-i -t'
+  elsif ENV['CI']
+    "-e CI=#{ENV['CI']}"
+  end
+
+task :docker_build do
+  sh 'docker build -t natalie-parser .'
+end
+
+task docker_bash: :docker_build do
+  sh 'docker run -it --rm --entrypoint bash natalie-parser'
+end
+
+task :docker_build_clang do
+  sh 'docker build -t natalie-parser-clang --build-arg CC=clang --build-arg CXX=clang++ .'
+end
+
+task :docker_build_ruby27 do
+  sh 'docker build -t natalie-parser-ruby27 --build-arg IMAGE="ruby:2.7" .'
+end
+
+task docker_test: %i[docker_test_gcc docker_test_clang]
+
+task docker_test_gcc: :docker_build do
+  sh "docker run #{DOCKER_FLAGS} --rm --entrypoint rake natalie-parser test"
+end
+
+task docker_test_clang: :docker_build_clang do
+  sh "docker run #{DOCKER_FLAGS} --rm --entrypoint rake natalie-parser-clang test"
+end
+
+# # # # Build Compile Database # # # #
+
+if system('which compiledb 2>&1 >/dev/null')
+  $compiledb_out = []
+
+  def $stderr.puts(str)
+    write(str + "\n")
+    $compiledb_out << str
+  end
+
+  task :write_compile_database do
+    if $compiledb_out.any?
+      File.write('build/build.log', $compiledb_out.join("\n"))
+      sh 'compiledb < build/build.log'
+    end
+  end
+else
+  task :write_compile_database do
+    # noop
+  end
+end
+
+# # # # Internal Tasks and Rules # # # #
+
+STANDARD = 'c++17'
+HEADERS = Rake::FileList['include/**/{*.h,*.hpp}']
+SOURCES = Rake::FileList['src/**/*.{c,cpp}']
+OBJECT_FILES = SOURCES.sub('src/', 'build/').pathmap('%p.o')
+
+require 'tempfile'
+
+task :build_dir do
+  mkdir_p 'build/lexer' unless File.exist?('build/lexer')
+  mkdir_p 'build/node' unless File.exist?('build/node')
+end
+
+rule '.cpp.o' => ['src/%n'] + HEADERS do |t|
+  sh "#{cxx} #{cxx_flags.join(' ')} -std=#{STANDARD} -c -o #{t.name} #{t.source}"
+end
+
+rule %r{lexer/.*\.cpp\.o$} => ['src/lexer/%n'] + HEADERS do |t|
+  sh "#{cxx} #{cxx_flags.join(' ')} -std=#{STANDARD} -c -o #{t.name} #{t.source}"
+end
+
+rule %r{node/.*\.cpp\.o$} => ['src/node/%n'] + HEADERS do |t|
+  sh "#{cxx} #{cxx_flags.join(' ')} -std=#{STANDARD} -c -o #{t.name} #{t.source}"
+end
+
+multitask objects: OBJECT_FILES
+
+file 'build/libnatalie_parser.a' => HEADERS + [:objects] do |t|
+  sh "ar rcs #{t.name} #{OBJECT_FILES}"
+end
+
+file "ext/natalie_parser/natalie_parser.#{so_ext}" => [
+  'ext/natalie_parser/natalie_parser.cpp',
+  'ext/natalie_parser/mri_creator.hpp',
+] + SOURCES + HEADERS do |t|
+  build_dir = File.expand_path('ext/natalie_parser', __dir__)
+  Rake::FileList['ext/natalie_parser/*.o'].each { |path| rm path }
+  rm_rf 'ext/natalie_parser/natalie_parser.so'
+  sh <<-SH
+    cd #{build_dir} && \
+    ruby extconf.rb && \
+    make -j
+  SH
+end
+
+  file 'build/fragments.hpp' => ['test/parser_test.rb', 'test/support/extract_parser_test_fragments.rb'] do
+  sh 'ruby -I lib:ext test/support/extract_parser_test_fragments.rb'
+end
+
+file 'build/asan_test' => ['test/asan_test.cpp', 'build/fragments.hpp', :library] do |t|
+  sh "#{cxx} #{cxx_flags.join(' ')} -std=#{STANDARD} -I build -I include -o #{t.name} #{t.source} -L build -lnatalie_parser"
+end
+
+task :bundle_install do
+  sh 'bundle check || bundle install'
+end
+
+def cc
+  @cc ||=
+    if ENV['CC']
+      ENV['CC']
+    elsif system('which ccache 2>&1 > /dev/null')
+      'ccache cc'
+    else
+      'cc'
+    end
+end
+
+def cxx
+  @cxx ||=
+    if ENV['CXX']
+      ENV['CXX']
+    elsif system('which ccache 2>&1 > /dev/null')
+      'ccache c++'
+    else
+      'c++'
+    end
+end
+
+def cxx_flags
+  base_flags =
+    case ENV['BUILD']
+    when 'release'
+      %w[
+        -fPIC
+        -g
+        -O2
+      ]
+    else
+      %w[
+        -fPIC
+        -g
+        -Wall
+        -Wextra
+        -Werror
+        -fsanitize=address
+      ]
+    end
+  base_flags + include_paths.map { |path| "-I #{path}" }
+end
+
+def include_paths
+  [File.expand_path('include', __dir__)]
+end

data/ext/natalie_parser/extconf.rb

@@ -0,0 +1,9 @@
+require 'mkmf'
+$CXXFLAGS += ' -g -std=c++17'
+$INCFLAGS += ' -I ../../include'
+$srcs = Dir['../../src/**/*.cpp', 'natalie_parser.cpp']
+$VPATH << "$(srcdir)/../../src"
+$VPATH << "$(srcdir)/../../src/lexer"
+$VPATH << "$(srcdir)/../../src/node"
+create_header
+create_makefile 'natalie_parser'

data/ext/natalie_parser/mri_creator.hpp

@@ -0,0 +1,139 @@
+#include "ruby.h"
+#include "ruby/encoding.h"
+#include "ruby/intern.h"
+
+#include "natalie_parser/creator.hpp"
+#include "natalie_parser/node.hpp"
+
+extern VALUE Sexp;
+
+namespace NatalieParser {
+
+class MRICreator : public Creator {
+public:
+    MRICreator(const Node &node)
+        : Creator { node.file().static_cast_as<const String>(), node.line(), node.column() } {
+        reset_sexp();
+    }
+
+    MRICreator(const MRICreator &other)
+        : Creator { other.file(), other.line(), other.column() } {
+        reset_sexp();
+    }
+
+    virtual ~MRICreator() { }
+
+    virtual void reset_sexp() override {
+        m_sexp = rb_class_new_instance(0, nullptr, Sexp);
+        rb_ivar_set(m_sexp, rb_intern("@file"), get_file_string(file()));
+        rb_ivar_set(m_sexp, rb_intern("@line"), rb_int_new(line() + 1));
+        rb_ivar_set(m_sexp, rb_intern("@column"), rb_int_new(column() + 1));
+    }
+
+    virtual void set_comments(const TM::String &comments) override {
+        auto string_obj = rb_utf8_str_new(comments.c_str(), comments.length());
+        rb_ivar_set(m_sexp, rb_intern("@comments"), string_obj);
+    }
+
+    virtual void set_type(const char *type) override {
+        rb_ary_store(m_sexp, 0, ID2SYM(rb_intern(type)));
+    }
+
+    virtual void append(const Node &node) override {
+        if (node.type() == Node::Type::Nil) {
+            rb_ary_push(m_sexp, Qnil);
+            return;
+        }
+        MRICreator creator { node };
+        creator.set_assignment(assignment());
+        node.transform(&creator);
+        rb_ary_push(m_sexp, creator.sexp());
+    }
+
+    virtual void append_array(const ArrayNode &array) override {
+        MRICreator creator { array };
+        creator.set_assignment(assignment());
+        array.ArrayNode::transform(&creator);
+        rb_ary_push(m_sexp, creator.sexp());
+    }
+
+    virtual void append_false() override {
+        rb_ary_push(m_sexp, Qfalse);
+    }
+
+    virtual void append_float(double number) override {
+        rb_ary_push(m_sexp, rb_float_new(number));
+    }
+
+    virtual void append_integer(long long number) override {
+        rb_ary_push(m_sexp, rb_int_new(number));
+    }
+
+    virtual void append_integer(TM::String &number) override {
+        auto string_obj = rb_utf8_str_new(number.c_str(), number.length());
+        rb_ary_push(m_sexp, rb_Integer(string_obj));
+    }
+
+    virtual void append_nil() override {
+        rb_ary_push(m_sexp, Qnil);
+    }
+
+    virtual void append_range(long long first, long long last, bool exclude_end) override {
+        rb_ary_push(m_sexp, rb_range_new(rb_int_new(first), rb_int_new(last), exclude_end ? Qtrue : Qfalse));
+    }
+
+    virtual void append_regexp(TM::String &pattern, int options) override {
+        auto encoding = pattern.contains_utf8_encoded_multibyte_characters() ? rb_utf8_encoding() : rb_ascii8bit_encoding();
+        auto regexp = rb_enc_reg_new(pattern.c_str(), pattern.size(), encoding, options);
+        rb_ary_push(m_sexp, regexp);
+    }
+
+    virtual void append_sexp(std::function<void(Creator *)> fn) override {
+        MRICreator creator { *this };
+        fn(&creator);
+        rb_ary_push(m_sexp, creator.sexp());
+    }
+
+    virtual void append_string(TM::String &string) override {
+        auto encoding = string.contains_seemingly_valid_utf8_encoded_characters() ? rb_utf8_encoding() : rb_ascii8bit_encoding();
+        rb_ary_push(m_sexp, rb_enc_str_new(string.c_str(), string.length(), encoding));
+    }
+
+    virtual void append_symbol(TM::String &name) override {
+        auto encoding = name.contains_utf8_encoded_multibyte_characters() ? rb_utf8_encoding() : rb_ascii8bit_encoding();
+        auto symbol = ID2SYM(rb_intern3(name.c_str(), name.size(), encoding));
+        rb_ary_push(m_sexp, symbol);
+    }
+
+    virtual void append_true() override {
+        rb_ary_push(m_sexp, Qtrue);
+    }
+
+    virtual void wrap(const char *type) override {
+        auto inner = m_sexp;
+        reset_sexp();
+        set_type(type);
+        rb_ary_push(m_sexp, inner);
+    }
+
+    VALUE sexp() const { return m_sexp; }
+
+private:
+    VALUE m_sexp { Qnil };
+
+    static VALUE get_file_string(SharedPtr<const String> file) {
+        auto file_string = s_file_cache.get(*file);
+        if (!file_string) {
+            file_string = rb_str_new(file->c_str(), file->length());
+            // FIXME: Seems there is no way to un-register and object. :-(
+            rb_gc_register_mark_object(file_string);
+            s_file_cache.put(*file, file_string);
+        }
+        return file_string;
+    }
+
+    // TODO: Move this to the Parser object, pass it in, clean it up when finished with it.
+    // (Otherwise we leak memory if the user parses lots of different files in a long-running process.)
+    inline static TM::Hashmap<const String, VALUE> s_file_cache { TM::HashType::TMString };
+};
+}

data/ext/natalie_parser/natalie_parser.cpp

@@ -0,0 +1,144 @@
+#include "extconf.h"
+#include "ruby.h"
+#include "ruby/encoding.h"
+#include "ruby/intern.h"
+#include "stdio.h"
+
+// this includes MUST come after
+#include "mri_creator.hpp"
+#include "natalie_parser/parser.hpp"
+
+VALUE Parser;
+VALUE Sexp;
+
+extern "C" {
+
+VALUE initialize(int argc, VALUE *argv, VALUE self) {
+    if (argc < 1 || argc > 2)
+        rb_raise(rb_eSyntaxError,
+            "wrong number of arguments (given %d, expected 1..2)", argc);
+    rb_ivar_set(self, rb_intern("@code"), argv[0]);
+    VALUE path;
+    if (argc > 1)
+        path = argv[1];
+    else
+        path = rb_str_new_cstr("(string)");
+    rb_ivar_set(self, rb_intern("@path"), path);
+    return self;
+}
+
+VALUE node_to_ruby(TM::SharedPtr<NatalieParser::Node> node) {
+    NatalieParser::MRICreator creator { node.ref() };
+    node->transform(&creator);
+    return creator.sexp();
+}
+
+VALUE parse_on_instance(VALUE self) {
+    VALUE code = rb_ivar_get(self, rb_intern("@code"));
+    VALUE path = rb_ivar_get(self, rb_intern("@path"));
+    auto code_string = new TM::String { StringValueCStr(code) };
+    auto path_string = new TM::String { StringValueCStr(path) };
+    auto parser = NatalieParser::Parser { code_string, path_string };
+    try {
+        auto tree = parser.tree();
+        VALUE ast = node_to_ruby(tree);
+        return ast;
+    } catch (NatalieParser::Parser::SyntaxError &error) {
+        rb_raise(rb_eSyntaxError, "%s", error.message());
+    }
+}
+
+VALUE parse(int argc, VALUE *argv, VALUE self) {
+    VALUE parser = rb_class_new_instance(argc, argv, Parser);
+    return parse_on_instance(parser);
+}
+
+VALUE token_to_ruby(NatalieParser::Token token, bool include_location_info) {
+    if (token.is_eof())
+        return Qnil;
+    try {
+        token.validate();
+    } catch (NatalieParser::Parser::SyntaxError &error) {
+        rb_raise(rb_eSyntaxError, "%s", error.message());
+    }
+    const char *type = token.type_value();
+    if (!type) abort(); // FIXME: assert no workie?
+    auto hash = rb_hash_new();
+    rb_hash_aset(hash, ID2SYM(rb_intern("type")), ID2SYM(rb_intern(type)));
+    auto lit = token.literal_or_blank();
+    switch (token.type()) {
+    case NatalieParser::Token::Type::Bignum:
+    case NatalieParser::Token::Type::Doc:
+    case NatalieParser::Token::Type::String: {
+        auto literal = token.literal_string();
+        rb_hash_aset(hash, ID2SYM(rb_intern("literal")), rb_utf8_str_new(literal->c_str(), literal->length()));
+        break;
+    }
+    case NatalieParser::Token::Type::BackRef:
+    case NatalieParser::Token::Type::BareName:
+    case NatalieParser::Token::Type::ClassVariable:
+    case NatalieParser::Token::Type::Constant:
+    case NatalieParser::Token::Type::GlobalVariable:
+    case NatalieParser::Token::Type::InstanceVariable:
+    case NatalieParser::Token::Type::Symbol:
+    case NatalieParser::Token::Type::SymbolKey: {
+        auto literal = token.literal_string();
+        rb_hash_aset(hash, ID2SYM(rb_intern("literal")), ID2SYM(rb_intern3(literal->c_str(), literal->size(), rb_utf8_encoding())));
+        break;
+    }
+    case NatalieParser::Token::Type::Fixnum:
+    case NatalieParser::Token::Type::NthRef:
+        rb_hash_aset(hash, ID2SYM(rb_intern("literal")), rb_int_new(token.get_fixnum()));
+        break;
+    case NatalieParser::Token::Type::Float:
+        rb_hash_aset(hash, ID2SYM(rb_intern("literal")), rb_float_new(token.get_double()));
+        break;
+    case NatalieParser::Token::Type::InterpolatedRegexpEnd:
+        if (token.has_literal()) {
+            auto options = token.literal_string();
+            rb_hash_aset(hash, ID2SYM(rb_intern("options")), rb_str_new(options->c_str(), options->length()));
+        }
+        break;
+    default:
+        void();
+    }
+    if (include_location_info) {
+        rb_hash_aset(hash, ID2SYM(rb_intern("line")), rb_int_new(token.line()));
+        rb_hash_aset(hash, ID2SYM(rb_intern("column")), rb_int_new(token.column()));
+    }
+    return hash;
+}
+
+VALUE tokens_on_instance(VALUE self, VALUE include_location_info = Qfalse) {
+    VALUE code = rb_ivar_get(self, rb_intern("@code"));
+    VALUE path = rb_ivar_get(self, rb_intern("@path"));
+    auto code_string = new TM::String { StringValueCStr(code) };
+    auto path_string = new TM::String { StringValueCStr(path) };
+    auto lexer = NatalieParser::Lexer { code_string, path_string };
+    auto array = rb_ary_new();
+    auto the_tokens = lexer.tokens();
+    for (auto token : *the_tokens) {
+        auto token_value = token_to_ruby(token, RTEST(include_location_info));
+        if (token_value != Qnil && token_value != Qfalse)
+            rb_ary_push(array, token_value);
+    }
+    return array;
+}
+
+VALUE tokens(int argc, VALUE *argv, VALUE self) {
+    VALUE parser = rb_class_new_instance(1, argv, Parser);
+    VALUE include_location_info = argc > 1 ? argv[1] : Qfalse;
+    return tokens_on_instance(parser, include_location_info);
+}
+
+void Init_natalie_parser() {
+    int error;
+    Sexp = rb_const_get(rb_cObject, rb_intern("Sexp"));
+    Parser = rb_define_class("NatalieParser", rb_cObject);
+    rb_define_method(Parser, "initialize", initialize, -1);
+    rb_define_method(Parser, "parse", parse_on_instance, 0);
+    rb_define_method(Parser, "tokens", tokens_on_instance, 1);
+    rb_define_singleton_method(Parser, "parse", parse, -1);
+    rb_define_singleton_method(Parser, "tokens", tokens, -1);
+}
+}