natalie_parser 1.0.0

data/lib/natalie_parser/sexp.rb

@@ -0,0 +1,36 @@
+class Sexp < Array
+  def initialize(*items)
+    items.each { |i| self << i }
+  end
+
+  def self.from_array(ary)
+    Sexp.new(*ary)
+  end
+
+  attr_accessor :file, :line, :column, :comments
+
+  def inspect
+    "s(#{map(&:inspect).join(', ')})"
+  end
+
+  alias sexp_type first
+
+  def new(*items)
+    s = Sexp.new(*items)
+    s.file = file
+    s.line = line
+    s.column = column
+    s
+  end
+
+  def pretty_print q
+    nnd = ")"
+    q.group(1, "s(", nnd) do
+      q.seplist(self) { |v| q.pp v }
+    end
+  end
+end
+
+def s(*items)
+  Sexp.new(*items)
+end

data/lib/natalie_parser/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class NatalieParser
+  VERSION = '1.0.0'
+end

data/lib/natalie_parser.rb

@@ -0,0 +1,3 @@
+require_relative './natalie_parser/sexp'
+require_relative './natalie_parser/version'
+require 'natalie_parser/natalie_parser'

data/natalie_parser.gemspec

@@ -0,0 +1,23 @@
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'natalie_parser/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'natalie_parser'
+  spec.version       = NatalieParser::VERSION
+  spec.authors       = ['Tim Morgan']
+  spec.email         = ['tim@timmorgan.org']
+
+  spec.summary       = 'A Parser for the Ruby Programming Language'
+  spec.description   = 'NatalieParser is a zero-dependency, from-scratch, hand-written recursive descent parser for the Ruby Programming Language.'
+  spec.homepage      = 'https://github.com/natalie-lang/natalie_parser'
+  spec.license       = 'MIT'
+
+  spec.files         = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files`.split("\n").reject { |f| f.match(%r{^(test|\.)}) }
+  end
+
+  spec.require_paths = ['lib', 'ext']
+  spec.extensions    = %w[ext/natalie_parser/extconf.rb]
+end
+

data/src/lexer/interpolated_string_lexer.cpp

@@ -0,0 +1,88 @@
+#include "natalie_parser/lexer/interpolated_string_lexer.hpp"
+#include "natalie_parser/token.hpp"
+
+namespace NatalieParser {
+
+Token InterpolatedStringLexer::build_next_token() {
+    switch (m_state) {
+    case State::InProgress:
+        return consume_string();
+    case State::EvaluateBegin:
+        return start_evaluation();
+    case State::EvaluateEnd:
+        return stop_evaluation();
+    case State::EndToken:
+        return finish();
+    case State::Done:
+        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column };
+    }
+    TM_UNREACHABLE();
+}
+
+Token InterpolatedStringLexer::consume_string() {
+    SharedPtr<String> buf = new String;
+    while (auto c = current_char()) {
+        if (c == '\\') {
+            advance(); // backslash
+            auto result = consume_escaped_byte(*buf);
+            if (!result.first)
+                return Token { result.second, current_char(), m_file, m_cursor_line, m_cursor_column };
+        } else if (c == '#' && peek() == '{') {
+            if (buf->is_empty()) {
+                advance(2);
+                return start_evaluation();
+            }
+            auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+            advance(2);
+            m_state = State::EvaluateBegin;
+            return token;
+        } else if (c == m_start_char && m_start_char != m_stop_char) {
+            m_pair_depth++;
+            advance();
+            buf->append_char(c);
+        } else if (c == m_stop_char) {
+            advance();
+            if (m_pair_depth > 0) {
+                m_pair_depth--;
+                buf->append_char(c);
+            } else if (buf->is_empty()) {
+                return finish();
+            } else {
+                m_state = State::EndToken;
+                return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+            }
+        } else {
+            buf->append_char(c);
+            advance();
+        }
+    }
+
+    // Heredocs don't use a stop char --
+    // they just give us the whole input and we consume everything.
+    if (m_stop_char == 0) {
+        advance();
+        m_state = State::EndToken;
+        return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+    }
+
+    return Token { Token::Type::UnterminatedString, buf, m_file, m_token_line, m_token_column };
+}
+
+Token InterpolatedStringLexer::start_evaluation() {
+    m_nested_lexer = new Lexer { *this, '{', '}' };
+    m_state = State::EvaluateEnd;
+    return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column };
+}
+
+Token InterpolatedStringLexer::stop_evaluation() {
+    advance(); // }
+    m_state = State::InProgress;
+    return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column };
+}
+
+Token InterpolatedStringLexer::finish() {
+    m_state = State::Done;
+    return Token { m_end_type, m_file, m_cursor_line, m_cursor_column };
+}
+
+};

data/src/lexer/regexp_lexer.cpp

@@ -0,0 +1,95 @@
+#include "natalie_parser/lexer/regexp_lexer.hpp"
+#include "natalie_parser/token.hpp"
+
+namespace NatalieParser {
+
+Token RegexpLexer::build_next_token() {
+    switch (m_state) {
+    case State::InProgress:
+        return consume_regexp();
+    case State::EvaluateBegin:
+        m_nested_lexer = new Lexer { *this };
+        m_nested_lexer->set_stop_char('}');
+        m_state = State::EvaluateEnd;
+        return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column };
+    case State::EvaluateEnd:
+        advance(); // }
+        if (current_char() == m_stop_char) {
+            advance();
+            m_options = consume_options();
+            m_state = State::EndToken;
+        } else {
+            m_state = State::InProgress;
+        }
+        return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column };
+    case State::EndToken: {
+        m_state = State::Done;
+        auto token = Token { Token::Type::InterpolatedRegexpEnd, m_file, m_cursor_line, m_cursor_column };
+        if (m_options && !m_options->is_empty())
+            token.set_literal(m_options);
+        return token;
+    }
+    case State::Done:
+        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column };
+    }
+    TM_UNREACHABLE();
+}
+
+Token RegexpLexer::consume_regexp() {
+    SharedPtr<String> buf = new String;
+    while (auto c = current_char()) {
+        if (c == '\\') {
+            c = next();
+            switch (c) {
+            case '/':
+                buf->append_char(c);
+                break;
+            default:
+                if (c == m_stop_char) {
+                    buf->append_char(c);
+                } else {
+                    buf->append_char('\\');
+                    buf->append_char(c);
+                }
+                break;
+            }
+            advance();
+        } else if (c == '#' && peek() == '{') {
+            auto token = Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+            buf = new String;
+            advance(2);
+            m_state = State::EvaluateBegin;
+            return token;
+        } else if (c == m_start_char && m_start_char != m_stop_char) {
+            m_pair_depth++;
+            advance();
+            buf->append_char(c);
+        } else if (c == m_stop_char) {
+            advance();
+            if (m_pair_depth > 0) {
+                m_pair_depth--;
+                buf->append_char(c);
+            } else {
+                m_options = consume_options();
+                m_state = State::EndToken;
+                return Token { Token::Type::String, buf, m_file, m_token_line, m_token_column };
+            }
+        } else {
+            buf->append_char(c);
+            advance();
+        }
+    }
+    return Token { Token::Type::UnterminatedRegexp, buf, m_file, m_token_line, m_token_column };
+}
+
+String *RegexpLexer::consume_options() {
+    char c = current_char();
+    auto options = new String;
+    while (c == 'i' || c == 'm' || c == 'x' || c == 'o' || c == 'u' || c == 'e' || c == 's' || c == 'n') {
+        options->append_char(c);
+        c = next();
+    }
+    return options;
+}
+
+};

data/src/lexer/word_array_lexer.cpp

@@ -0,0 +1,134 @@
+#include "natalie_parser/lexer/word_array_lexer.hpp"
+#include "natalie_parser/token.hpp"
+
+namespace NatalieParser {
+
+Token WordArrayLexer::build_next_token() {
+    switch (m_state) {
+    case State::InProgress:
+        return consume_array();
+    case State::DynamicStringInProgress:
+        return consume_array();
+    case State::DynamicStringBegin:
+        m_state = State::EvaluateBegin;
+        return Token { Token::Type::String, m_buffer, m_file, m_token_line, m_token_column };
+    case State::DynamicStringEnd:
+        if (current_char() == m_stop_char) {
+            advance();
+            m_state = State::EndToken;
+        } else {
+            m_state = State::InProgress;
+        }
+        return Token { Token::Type::InterpolatedStringEnd, m_file, m_token_line, m_token_column };
+    case State::EvaluateBegin:
+        return start_evaluation();
+    case State::EvaluateEnd:
+        advance(); // }
+        m_state = State::DynamicStringInProgress;
+        return Token { Token::Type::EvaluateToStringEnd, m_file, m_token_line, m_token_column };
+    case State::EndToken:
+        m_state = State::Done;
+        return Token { Token::Type::RBracket, m_file, m_cursor_line, m_cursor_column };
+    case State::Done:
+        return Token { Token::Type::Eof, m_file, m_cursor_line, m_cursor_column };
+    }
+    TM_UNREACHABLE();
+}
+
+Token WordArrayLexer::consume_array() {
+    m_buffer = new String;
+    while (auto c = current_char()) {
+        if (c == '\\') {
+            c = next();
+            advance();
+            if (c == ' ') {
+                m_buffer->append_char(c);
+            } else if (m_interpolated) {
+                // FIXME: need to use logic from InterpolatedStringLexer
+                switch (c) {
+                case 'n':
+                    m_buffer->append_char('\n');
+                    break;
+                case 't':
+                    m_buffer->append_char('\t');
+                    break;
+                default:
+                    m_buffer->append_char(c);
+                    break;
+                }
+            } else {
+                if (isspace(c)) {
+                    m_buffer->append_char(c);
+                } else {
+                    m_buffer->append_char('\\');
+                    m_buffer->append_char(c);
+                }
+            }
+        } else if (isspace(c)) {
+            if (m_state == State::DynamicStringInProgress) {
+                advance();
+                return dynamic_string_finish();
+            }
+            if (!m_buffer->is_empty()) {
+                auto token = Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column };
+                advance();
+                return token;
+            }
+            advance(); // space
+        } else if (m_interpolated && c == '#' && peek() == '{') {
+            return in_progress_start_dynamic_string();
+        } else if (c == m_start_char && m_start_char != m_stop_char) {
+            m_pair_depth++;
+            advance();
+            m_buffer->append_char(c);
+        } else if (c == m_stop_char) {
+            if (m_pair_depth > 0) {
+                m_pair_depth--;
+                m_buffer->append_char(c);
+                advance();
+            } else if (m_state == State::DynamicStringInProgress) {
+                return dynamic_string_finish();
+            } else {
+                return in_progress_finish();
+            }
+        } else {
+            m_buffer->append_char(c);
+            advance();
+        }
+    }
+
+    return Token { Token::Type::UnterminatedWordArray, m_buffer, m_file, m_token_line, m_token_column };
+}
+
+Token WordArrayLexer::in_progress_start_dynamic_string() {
+    advance(2); // #{
+    m_state = State::DynamicStringBegin;
+    return Token { Token::Type::InterpolatedStringBegin, m_file, m_cursor_line, m_cursor_column };
+}
+
+Token WordArrayLexer::start_evaluation() {
+    m_nested_lexer = new Lexer { *this, '{', '}' };
+    m_state = State::EvaluateEnd;
+    return Token { Token::Type::EvaluateToStringBegin, m_file, m_token_line, m_token_column };
+}
+
+Token WordArrayLexer::dynamic_string_finish() {
+    if (!m_buffer->is_empty()) {
+        m_state = State::DynamicStringEnd;
+        return Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column };
+    }
+    m_state = State::InProgress;
+    return Token { Token::Type::InterpolatedStringEnd, m_file, m_token_line, m_token_column };
+}
+
+Token WordArrayLexer::in_progress_finish() {
+    advance(); // ) or ] or } or whatever
+    if (!m_buffer->is_empty()) {
+        m_state = State::EndToken;
+        return Token { Token::Type::String, m_buffer, m_file, m_cursor_line, m_cursor_column };
+    }
+    m_state = State::Done;
+    return Token { Token::Type::RBracket, m_file, m_cursor_line, m_cursor_column };
+}
+
+};