natalie_parser 1.0.0

data/include/natalie_parser/creator/debug_creator.hpp

@@ -0,0 +1,113 @@
+#pragma once
+
+#include "natalie_parser/creator.hpp"
+#include "natalie_parser/node.hpp"
+
+namespace NatalieParser {
+
+class DebugCreator : public Creator {
+public:
+    virtual ~DebugCreator() { }
+
+    virtual void set_comments(const TM::String &) override {
+        // ignore for now
+    }
+
+    virtual void set_type(const char *type) override {
+        if (m_nodes.size() >= 1)
+            m_nodes[0] = String::format(":{}", type);
+        else
+            m_nodes.push_front(String::format(":{}", type));
+    }
+
+    virtual void append(const Node &node) override {
+        if (node.type() == Node::Type::Nil) {
+            m_nodes.push("nil");
+            return;
+        }
+        DebugCreator creator;
+        creator.set_assignment(assignment());
+        node.transform(&creator);
+        m_nodes.push(creator.to_string());
+    }
+
+    virtual void append_array(const ArrayNode &array) override {
+        DebugCreator creator;
+        creator.set_assignment(assignment());
+        array.ArrayNode::transform(&creator);
+        m_nodes.push(creator.to_string());
+    }
+
+    virtual void append_false() override {
+        m_nodes.push("false");
+    }
+
+    virtual void append_float(double number) override {
+        m_nodes.push(String(number));
+    }
+
+    virtual void append_integer(long long number) override {
+        m_nodes.push(String(number));
+    }
+
+    virtual void append_integer(TM::String &number) override {
+        m_nodes.push(String(number));
+    }
+
+    virtual void append_nil() override {
+        m_nodes.push("nil");
+    }
+
+    virtual void append_range(long long first, long long last, bool exclude_end) override {
+        m_nodes.push(String(first));
+        m_nodes.push(exclude_end ? "..." : "..");
+        m_nodes.push(String(last));
+    }
+
+    virtual void append_regexp(TM::String &pattern, int options) override {
+        TM_UNUSED(options);
+        m_nodes.push('/');
+        m_nodes.push(pattern);
+        m_nodes.push('/');
+    }
+
+    virtual void append_sexp(std::function<void(Creator *)> fn) override {
+        DebugCreator creator;
+        fn(&creator);
+        m_nodes.push(creator.to_string());
+    }
+
+    virtual void append_string(TM::String &string) override {
+        m_nodes.push(String::format("\"{}\"", string));
+    }
+
+    virtual void append_symbol(TM::String &name) override {
+        m_nodes.push(String::format(":{}", name));
+    }
+
+    virtual void append_true() override {
+        m_nodes.push("true");
+    }
+
+    virtual void wrap(const char *type) override {
+        auto inner = to_string();
+        m_nodes.clear();
+        set_type(type);
+        m_nodes.push(inner);
+    }
+
+    TM::String to_string() {
+        TM::String buf = "(";
+        for (size_t i = 0; i < m_nodes.size(); ++i) {
+            buf.append(m_nodes[i]);
+            if (i + 1 < m_nodes.size())
+                buf.append(", ");
+        }
+        buf.append_char(')');
+        return buf;
+    }
+
+private:
+    TM::Vector<TM::String> m_nodes {};
+};
+}

data/include/natalie_parser/creator.hpp

@@ -0,0 +1,108 @@
+#pragma once
+
+#include "tm/shared_ptr.hpp"
+#include "tm/string.hpp"
+#include "tm/vector.hpp"
+#include <functional>
+#include <initializer_list>
+
+namespace NatalieParser {
+
+class Node;
+class ArrayNode;
+
+class Creator {
+public:
+    Creator() { }
+
+    Creator(TM::SharedPtr<const TM::String> file, size_t line, size_t column)
+        : m_file { file }
+        , m_line { line }
+        , m_column { column } { }
+
+    virtual void set_comments(const TM::String &comments) = 0;
+    virtual void set_type(const char *type) = 0;
+    virtual void append(const TM::SharedPtr<Node> node) { append(*node); }
+    virtual void append(const Node &node) = 0;
+    virtual void append_array(const TM::SharedPtr<ArrayNode> array) { append_array(*array); }
+    virtual void append_array(const ArrayNode &array) = 0;
+    virtual void append_false() = 0;
+    virtual void append_float(double number) = 0;
+    virtual void append_integer(long long number) = 0;
+    virtual void append_integer(TM::String &number) = 0;
+    virtual void append_nil() = 0;
+    virtual void append_range(long long first, long long last, bool exclude_end) = 0;
+    virtual void append_regexp(TM::String &pattern, int options) = 0;
+    virtual void append_sexp(std::function<void(Creator *)> fn) = 0;
+    virtual void append_string(TM::String &string) = 0;
+    virtual void append_symbol(TM::String &symbol) = 0;
+    virtual void append_true() = 0;
+    virtual void wrap(const char *type) = 0;
+
+    virtual ~Creator() { }
+
+    void append_nil_sexp() {
+        append_sexp([&](Creator *c) { c->set_type("nil"); });
+    }
+
+    void append_regexp(TM::SharedPtr<TM::String> pattern_ptr, int options) {
+        if (!pattern_ptr) {
+            auto p = TM::String("");
+            append_regexp(p, options);
+        }
+        append_regexp(*pattern_ptr, options);
+    }
+
+    void append_string(const char *string) {
+        auto s = TM::String(string);
+        append_string(s);
+    }
+
+    void append_string(TM::SharedPtr<TM::String> string_ptr) {
+        if (!string_ptr) {
+            auto s = TM::String("");
+            append_string(s);
+        }
+        append_string(*string_ptr);
+    }
+
+    void append_symbol(const char *symbol) {
+        auto s = TM::String(symbol);
+        append_symbol(s);
+    }
+
+    void append_symbol(TM::SharedPtr<TM::String> symbol_ptr) {
+        if (!symbol_ptr) {
+            auto s = TM::String("");
+            append_symbol(s);
+        }
+        append_symbol(*symbol_ptr);
+    }
+
+    bool assignment() { return m_assignment; }
+    void set_assignment(bool assignment) { m_assignment = assignment; }
+
+    void with_assignment(bool assignment, std::function<void()> fn) {
+        auto assignment_was = m_assignment;
+        m_assignment = assignment;
+        fn();
+        m_assignment = assignment_was;
+    }
+
+    TM::SharedPtr<const TM::String> file() const { return m_file; }
+    size_t line() const { return m_line; }
+    size_t column() const { return m_column; }
+
+    void set_line(size_t line) { m_line = line; }
+    void set_column(size_t column) { m_column = column; }
+
+    virtual void reset_sexp() { }
+
+private:
+    bool m_assignment { false };
+    TM::SharedPtr<const TM::String> m_file {};
+    size_t m_line { 0 };
+    size_t m_column { 0 };
+};
+
+}

data/include/natalie_parser/lexer/interpolated_string_lexer.hpp

@@ -0,0 +1,64 @@
+#pragma once
+
+#include "natalie_parser/lexer.hpp"
+#include "natalie_parser/token.hpp"
+#include "tm/shared_ptr.hpp"
+#include "tm/vector.hpp"
+
+namespace NatalieParser {
+
+class InterpolatedStringLexer : public Lexer {
+public:
+    InterpolatedStringLexer(Lexer &parent_lexer, char start_char, char stop_char, Token::Type end_type)
+        : Lexer { parent_lexer }
+        , m_end_type { end_type } {
+        set_nested_lexer(nullptr);
+        set_start_char(start_char == stop_char ? 0 : start_char);
+        set_stop_char(stop_char);
+    }
+
+    // used for lexing a Heredoc
+    InterpolatedStringLexer(Lexer &parent_lexer, Token string_token, Token::Type end_type)
+        : Lexer { string_token.literal_string(), parent_lexer.file() }
+        , m_end_type { end_type }
+        , m_alters_parent_cursor_position { false } {
+        set_cursor_line(parent_lexer.cursor_line() + 1); // the line after the heredoc delimiter
+        set_nested_lexer(nullptr);
+        set_stop_char(0);
+    }
+
+    virtual bool alters_parent_cursor_position() override { return m_alters_parent_cursor_position; }
+
+private:
+    virtual Token build_next_token() override;
+    Token consume_string();
+    Token start_evaluation();
+    Token stop_evaluation();
+    Token finish();
+
+    virtual bool skip_whitespace() override { return false; }
+
+    /**
+     * a little state machine
+     * stateDiagram-v2
+     *     [*] --> InProgress
+     *     InProgress --> EvaluateBegin
+     *     InProgress --> EndToken
+     *     EvaluateBegin --> EvaluateEnd
+     *     EvaluateEnd --> InProgress
+     *     EndToken --> Done
+     *     Done --> [*]
+     */
+    enum class State {
+        InProgress,
+        EvaluateBegin,
+        EvaluateEnd,
+        EndToken,
+        Done,
+    };
+
+    State m_state { State::InProgress };
+    Token::Type m_end_type;
+    bool m_alters_parent_cursor_position { true };
+};
+}

data/include/natalie_parser/lexer/regexp_lexer.hpp

@@ -0,0 +1,37 @@
+#pragma once
+
+#include "natalie_parser/lexer.hpp"
+#include "natalie_parser/token.hpp"
+#include "tm/shared_ptr.hpp"
+#include "tm/vector.hpp"
+
+namespace NatalieParser {
+
+class RegexpLexer : public Lexer {
+public:
+    RegexpLexer(Lexer &parent_lexer, char start_char, char stop_char)
+        : Lexer { parent_lexer } {
+        set_nested_lexer(nullptr);
+        set_start_char(start_char == stop_char ? 0 : start_char);
+        set_stop_char(stop_char);
+    }
+
+private:
+    virtual Token build_next_token() override;
+    Token consume_regexp();
+    String *consume_options();
+
+    virtual bool skip_whitespace() override { return false; }
+
+    enum class State {
+        InProgress,
+        EvaluateBegin,
+        EvaluateEnd,
+        EndToken,
+        Done,
+    };
+
+    State m_state { State::InProgress };
+    SharedPtr<String> m_options {};
+};
+}

data/include/natalie_parser/lexer/word_array_lexer.hpp

@@ -0,0 +1,57 @@
+#pragma once
+
+#include "natalie_parser/lexer.hpp"
+#include "natalie_parser/token.hpp"
+#include "tm/shared_ptr.hpp"
+#include "tm/vector.hpp"
+
+namespace NatalieParser {
+
+class WordArrayLexer : public Lexer {
+public:
+    WordArrayLexer(Lexer &parent_lexer, char start_char, char stop_char, bool interpolated)
+        : Lexer { parent_lexer }
+        , m_interpolated { interpolated }
+        , m_start_char { start_char } {
+        set_nested_lexer(nullptr);
+        set_stop_char(stop_char);
+    }
+
+private:
+    virtual Token build_next_token() override;
+    Token consume_array();
+
+    virtual bool skip_whitespace() override { return false; }
+
+    bool interpolated() const { return m_interpolated; }
+
+    // states
+    enum class State {
+        InProgress,
+        DynamicStringBegin,
+        DynamicStringInProgress,
+        DynamicStringEnd,
+        EvaluateBegin,
+        EvaluateEnd,
+        EndToken,
+        Done,
+    };
+
+    // transitions
+    Token in_progress_start_dynamic_string();
+    Token start_evaluation();
+    Token dynamic_string_finish();
+    Token in_progress_finish();
+
+    State m_state { State::InProgress };
+
+    // if this is true, then process #{...} interpolation
+    bool m_interpolated { false };
+
+    // if we encounter the m_start_char within the array,
+    // then increment m_pair_depth
+    char m_start_char { 0 };
+    int m_pair_depth { 0 };
+    SharedPtr<String> m_buffer;
+};
+}

data/include/natalie_parser/lexer.hpp

@@ -0,0 +1,135 @@
+#pragma once
+
+#include "natalie_parser/token.hpp"
+#include "tm/shared_ptr.hpp"
+#include "tm/vector.hpp"
+
+namespace NatalieParser {
+
+class Lexer {
+public:
+    Lexer(SharedPtr<String> input, SharedPtr<String> file)
+        : m_input { input }
+        , m_file { file }
+        , m_size { input->length() } { }
+
+    Lexer(const Lexer &other, char start_char, char stop_char)
+        : m_input { other.m_input }
+        , m_file { other.m_file }
+        , m_size { other.m_size }
+        , m_index { other.m_index }
+        , m_cursor_line { other.m_cursor_line }
+        , m_cursor_column { other.m_cursor_column }
+        , m_token_line { other.m_token_line }
+        , m_token_column { other.m_token_column }
+        , m_stop_char { stop_char }
+        , m_start_char { start_char } { }
+
+    SharedPtr<Vector<Token>> tokens();
+    Token next_token();
+
+    virtual ~Lexer() {
+        delete m_nested_lexer;
+    }
+
+    SharedPtr<String> file() const { return m_file; }
+
+    size_t cursor_line() const { return m_cursor_line; }
+    void set_cursor_line(size_t cursor_line) { m_cursor_line = cursor_line; }
+
+    void set_nested_lexer(Lexer *lexer) { m_nested_lexer = lexer; }
+    void set_start_char(char c) { m_start_char = c; }
+    void set_stop_char(char c) { m_stop_char = c; }
+
+    virtual bool alters_parent_cursor_position() { return true; }
+
+protected:
+    char current_char() {
+        if (m_index >= m_size)
+            return 0;
+        char c = m_input->at(m_index);
+        return c;
+    }
+
+    bool match(size_t bytes, const char *compare);
+    void advance();
+    void advance(size_t bytes);
+    void rewind(size_t bytes = 1);
+
+    char next() {
+        advance();
+        return current_char();
+    }
+
+    char peek() {
+        if (m_index + 1 >= m_size)
+            return 0;
+        return m_input->at(m_index + 1);
+    }
+
+    virtual bool skip_whitespace();
+    virtual Token build_next_token();
+    Token consume_symbol();
+    Token consume_word(Token::Type type);
+    Token consume_bare_name();
+    Token consume_constant();
+    Token consume_global_variable();
+    Token consume_heredoc();
+    Token consume_numeric();
+    Token consume_numeric_as_float(SharedPtr<String>);
+    Token consume_nth_ref();
+    long long consume_hex_number(int max_length = 0, bool allow_underscore = false);
+    long long consume_octal_number(int max_length = 0, bool allow_underscore = false);
+    Token consume_double_quoted_string(char, char, Token::Type begin_type = Token::Type::InterpolatedStringBegin, Token::Type end_type = Token::Type::InterpolatedStringEnd);
+    Token consume_single_quoted_string(char, char);
+    Token consume_quoted_array_without_interpolation(char start_char, char stop_char, Token::Type type);
+    Token consume_quoted_array_with_interpolation(char start_char, char stop_char, Token::Type type);
+    Token consume_regexp(char start_char, char stop_char);
+    SharedPtr<String> consume_non_whitespace();
+
+    void utf32_codepoint_to_utf8(String &buf, long long codepoint);
+    std::pair<bool, Token::Type> consume_escaped_byte(String &buf);
+
+    Token chars_to_fixnum_or_bignum_token(SharedPtr<String> chars, int base, int offset);
+
+    bool token_is_first_on_line() const;
+
+    bool char_can_be_string_or_regexp_delimiter(char c) const {
+        return (c >= '!' && c <= '/') || c == ':' || c == '?' || c == '@' || c == '~' || c == '|' || (c >= '^' && c <= '`');
+    }
+
+    SharedPtr<String> m_input;
+    SharedPtr<String> m_file;
+    size_t m_size { 0 };
+    size_t m_index { 0 };
+
+    // where we should jump after each heredoc
+    Vector<size_t> m_heredoc_stack {};
+
+    // current character position
+    size_t m_cursor_line { 0 };
+    size_t m_cursor_column { 0 };
+
+    // start of current token
+    size_t m_token_line { 0 };
+    size_t m_token_column { 0 };
+
+    // if the current token is preceded by whitespace
+    bool m_whitespace_precedes { false };
+
+    // the previously-matched token
+    Token m_last_token {};
+
+    // we have an open ternary '?' that needs a matching ':'
+    bool m_open_ternary { false };
+
+    Lexer *m_nested_lexer { nullptr };
+
+    char m_stop_char { 0 };
+
+    // if we encounter the m_start_char within the string,
+    // then increment m_pair_depth
+    char m_start_char { 0 };
+    int m_pair_depth { 0 };
+};
+}

data/include/natalie_parser/node/alias_node.hpp

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "natalie_parser/node/node.hpp"
+#include "natalie_parser/node/node_with_args.hpp"
+#include "natalie_parser/node/symbol_node.hpp"
+#include "tm/hashmap.hpp"
+#include "tm/owned_ptr.hpp"
+#include "tm/string.hpp"
+
+namespace NatalieParser {
+
+using namespace TM;
+
+class AliasNode : public Node {
+public:
+    AliasNode(const Token &token, SharedPtr<SymbolNode> new_name, SharedPtr<SymbolNode> existing_name)
+        : Node { token }
+        , m_new_name { new_name }
+        , m_existing_name { existing_name } {
+        assert(m_new_name);
+        assert(m_existing_name);
+    }
+
+    virtual Type type() const override { return Type::Alias; }
+
+    const SharedPtr<SymbolNode> new_name() const { return m_new_name; }
+    const SharedPtr<SymbolNode> existing_name() const { return m_existing_name; }
+
+    virtual void transform(Creator *creator) const override;
+
+private:
+    SharedPtr<SymbolNode> m_new_name {};
+    SharedPtr<SymbolNode> m_existing_name {};
+};
+}

data/include/natalie_parser/node/arg_node.hpp

@@ -0,0 +1,74 @@
+#pragma once
+
+#include "natalie_parser/node/node.hpp"
+#include "natalie_parser/node/node_with_args.hpp"
+#include "tm/hashmap.hpp"
+#include "tm/owned_ptr.hpp"
+#include "tm/string.hpp"
+
+namespace NatalieParser {
+
+using namespace TM;
+
+class ArgNode : public Node {
+public:
+    ArgNode(const Token &token)
+        : Node { token } { }
+
+    ArgNode(const Token &token, SharedPtr<String> name)
+        : Node { token }
+        , m_name { name } { }
+
+    virtual Type type() const override { return Type::Arg; }
+
+    const SharedPtr<String> name() const { return m_name; }
+
+    void append_name(Creator *creator) const {
+        String n;
+        if (m_name)
+            n = m_name->clone();
+        if (m_splat) {
+            n.prepend_char('*');
+        } else if (m_kwsplat) {
+            n.prepend_char('*');
+            n.prepend_char('*');
+        } else if (m_block_arg) {
+            n.prepend_char('&');
+        }
+        creator->append_symbol(n);
+    }
+
+    bool splat_or_kwsplat() const { return m_splat || m_kwsplat; }
+
+    bool splat() const { return m_splat; }
+    void set_splat(bool splat) { m_splat = splat; }
+
+    bool kwsplat() const { return m_kwsplat; }
+    void set_kwsplat(bool kwsplat) { m_kwsplat = kwsplat; }
+
+    bool block_arg() const { return m_block_arg; }
+    void set_block_arg(bool block_arg) { m_block_arg = block_arg; }
+
+    const SharedPtr<Node> value() const { return m_value; }
+
+    void set_value(SharedPtr<Node> value) { m_value = value; }
+
+    void add_to_locals(TM::Hashmap<TM::String> &locals) {
+        locals.set(m_name->c_str());
+    }
+
+    virtual void transform(Creator *creator) const override {
+        creator->set_type("lasgn");
+        append_name(creator);
+        if (m_value)
+            creator->append(m_value.ref());
+    }
+
+protected:
+    SharedPtr<String> m_name {};
+    bool m_block_arg { false };
+    bool m_splat { false };
+    bool m_kwsplat { false };
+    SharedPtr<Node> m_value {};
+};
+}

data/include/natalie_parser/node/array_node.hpp

@@ -0,0 +1,34 @@
+#pragma once
+
+#include "natalie_parser/node/node.hpp"
+#include "natalie_parser/node/node_with_args.hpp"
+#include "tm/hashmap.hpp"
+#include "tm/string.hpp"
+
+namespace NatalieParser {
+
+using namespace TM;
+
+class ArrayNode : public Node {
+public:
+    ArrayNode(const Token &token)
+        : Node { token } { }
+
+    virtual Type type() const override { return Type::Array; }
+
+    void add_node(SharedPtr<Node> node) {
+        m_nodes.push(node);
+    }
+
+    const Vector<SharedPtr<Node>> &nodes() const { return m_nodes; }
+
+    virtual void transform(Creator *creator) const override {
+        creator->set_type("array");
+        for (auto node : m_nodes)
+            creator->append(node);
+    }
+
+protected:
+    Vector<SharedPtr<Node>> m_nodes {};
+};
+}

data/include/natalie_parser/node/array_pattern_node.hpp

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "natalie_parser/node/array_node.hpp"
+#include "natalie_parser/node/node.hpp"
+#include "natalie_parser/node/node_with_args.hpp"
+#include "tm/hashmap.hpp"
+#include "tm/string.hpp"
+
+namespace NatalieParser {
+
+using namespace TM;
+
+class ArrayPatternNode : public ArrayNode {
+public:
+    ArrayPatternNode(const Token &token)
+        : ArrayNode { token } { }
+
+    virtual Type type() const override { return Type::ArrayPattern; }
+
+    virtual void transform(Creator *creator) const override {
+        creator->set_type("array_pat");
+        if (!m_nodes.is_empty())
+            creator->append_nil(); // NOTE: I don't know what this nil is for
+        for (auto node : m_nodes)
+            creator->append(node);
+    }
+};
+}