yarp 0.6.0

data/lib/yarp.rb

@@ -0,0 +1,330 @@
+# frozen_string_literal: true
+
+module YARP
+  # This represents a source of Ruby code that has been parsed. It is used in
+  # conjunction with locations to allow them to resolve line numbers and source
+  # ranges.
+  class Source
+    attr_reader :source, :offsets
+
+    def initialize(source, offsets)
+      @source = source
+      @offsets = offsets
+    end
+
+    def slice(offset, length)
+      source.byteslice(offset, length)
+    end
+
+    def line(value)
+      offsets.bsearch_index { |offset| offset > value } || offsets.length
+    end
+
+    def column(value)
+      value - offsets[line(value) - 1]
+    end
+  end
+
+  # This represents a location in the source.
+  class Location
+    # A Source object that is used to determine more information from the given
+    # offset and length.
+    private attr_reader :source
+
+    # The byte offset from the beginning of the source where this location
+    # starts.
+    attr_reader :start_offset
+
+    # The length of this location in bytes.
+    attr_reader :length
+
+    def initialize(source, start_offset, length)
+      @source = source
+      @start_offset = start_offset
+      @length = length
+    end
+
+    def inspect
+      "#<YARP::Location @start_offset=#{@start_offset} @length=#{@length}>"
+    end
+
+    # The source code that this location represents.
+    def slice
+      source.slice(start_offset, length)
+    end
+
+    # The byte offset from the beginning of the source where this location ends.
+    def end_offset
+      start_offset + length
+    end
+
+    # The line number where this location starts.
+    def start_line
+      source.line(start_offset)
+    end
+
+    # The line number where this location ends.
+    def end_line
+      source.line(end_offset - 1)
+    end
+
+    # The column number in bytes where this location starts from the start of
+    # the line.
+    def start_column
+      source.column(start_offset)
+    end
+
+    # The column number in bytes where this location ends from the start of the
+    # line.
+    def end_column
+      source.column(end_offset - 1)
+    end
+
+    def deconstruct_keys(keys)
+      { start_offset: start_offset, end_offset: end_offset }
+    end
+
+    def pretty_print(q)
+      q.text("(#{start_offset}...#{end_offset})")
+    end
+
+    def ==(other)
+      other.is_a?(Location) &&
+        other.start_offset == start_offset &&
+        other.end_offset == end_offset
+    end
+
+    def self.null
+      new(0, 0)
+    end
+  end
+
+  # This represents a comment that was encountered during parsing.
+  class Comment
+    attr_reader :type, :location
+
+    def initialize(type, location)
+      @type = type
+      @location = location
+    end
+
+    def deconstruct_keys(keys)
+      { type: type, location: location }
+    end
+  end
+
+  # This represents an error that was encountered during parsing.
+  class ParseError
+    attr_reader :message, :location
+
+    def initialize(message, location)
+      @message = message
+      @location = location
+    end
+
+    def deconstruct_keys(keys)
+      { message: message, location: location }
+    end
+  end
+
+  # This represents a warning that was encountered during parsing.
+  class ParseWarning
+    attr_reader :message, :location
+
+    def initialize(message, location)
+      @message = message
+      @location = location
+    end
+
+    def deconstruct_keys(keys)
+      { message: message, location: location }
+    end
+  end
+
+  # A class that knows how to walk down the tree. None of the individual visit
+  # methods are implemented on this visitor, so it forces the consumer to
+  # implement each one that they need. For a default implementation that
+  # continues walking the tree, see the Visitor class.
+  class BasicVisitor
+    def visit(node)
+      node&.accept(self)
+    end
+
+    def visit_all(nodes)
+      nodes.map { |node| visit(node) }
+    end
+
+    def visit_child_nodes(node)
+      visit_all(node.child_nodes)
+    end
+  end
+
+  class Visitor < BasicVisitor
+  end
+
+  # This represents the result of a call to ::parse or ::parse_file. It contains
+  # the AST, any comments that were encounters, and any errors that were
+  # encountered.
+  class ParseResult
+    attr_reader :value, :comments, :errors, :warnings, :source
+
+    def initialize(value, comments, errors, warnings, source)
+      @value = value
+      @comments = comments
+      @errors = errors
+      @warnings = warnings
+      @source = source
+    end
+
+    def deconstruct_keys(keys)
+      { value: value, comments: comments, errors: errors, warnings: warnings }
+    end
+
+    def success?
+      errors.empty?
+    end
+
+    def failure?
+      !success?
+    end
+
+    # Keep in sync with Java MarkNewlinesVisitor
+    class MarkNewlinesVisitor < YARP::Visitor
+      def initialize(newline_marked)
+        @newline_marked = newline_marked
+      end
+
+      def visit_block_node(node)
+        old_newline_marked = @newline_marked
+        @newline_marked = Array.new(old_newline_marked.size, false)
+        begin
+          super(node)
+        ensure
+          @newline_marked = old_newline_marked
+        end
+      end
+      alias_method :visit_lambda_node, :visit_block_node
+
+      def visit_if_node(node)
+        node.set_newline_flag(@newline_marked)
+        super(node)
+      end
+      alias_method :visit_unless_node, :visit_if_node
+
+      def visit_statements_node(node)
+        node.body.each do |child|
+          child.set_newline_flag(@newline_marked)
+        end
+        super(node)
+      end
+    end
+    private_constant :MarkNewlinesVisitor
+
+    def mark_newlines
+      newline_marked = Array.new(1 + @source.offsets.size, false)
+      visitor = MarkNewlinesVisitor.new(newline_marked)
+      value.accept(visitor)
+      value
+    end
+  end
+
+  # This represents a token from the Ruby source.
+  class Token
+    attr_reader :type, :value, :location
+
+    def initialize(type, value, location)
+      @type = type
+      @value = value
+      @location = location
+    end
+
+    def deconstruct_keys(keys)
+      { type: type, value: value, location: location }
+    end
+
+    def pretty_print(q)
+      q.group do
+        q.text(type.to_s)
+        self.location.pretty_print(q)
+        q.text("(")
+        q.nest(2) do
+          q.breakable("")
+          q.pp(value)
+        end
+        q.breakable("")
+        q.text(")")
+      end
+    end
+
+    def ==(other)
+      other.is_a?(Token) &&
+        other.type == type &&
+        other.value == value
+    end
+  end
+
+  # This represents a node in the tree.
+  class Node
+    attr_reader :location
+
+    def newline?
+      @newline ? true : false
+    end
+
+    def set_newline_flag(newline_marked)
+      line = location.start_line
+      unless newline_marked[line]
+        newline_marked[line] = true
+        @newline = true
+      end
+    end
+
+    def pretty_print(q)
+      q.group do
+        q.text(self.class.name.split("::").last)
+        location.pretty_print(q)
+        q.text("[Li:#{location.start_line}]") if newline?
+        q.text("(")
+        q.nest(2) do
+          deconstructed = deconstruct_keys([])
+          deconstructed.delete(:location)
+
+          q.breakable("")
+          q.seplist(deconstructed, lambda { q.comma_breakable }, :each_value) { |value| q.pp(value) }
+        end
+        q.breakable("")
+        q.text(")")
+      end
+    end
+  end
+
+  # Load the serialized AST using the source as a reference into a tree.
+  def self.load(source, serialized)
+    Serialize.load(source, serialized)
+  end
+
+  # This module is used for testing and debugging and is not meant to be used by
+  # consumers of this library.
+  module Debug
+    def self.newlines(source)
+      YARP.parse(source).source.offsets
+    end
+
+    def self.parse_serialize_file(filepath)
+      parse_serialize_file_metadata(filepath, [filepath.bytesize, filepath.b, 0].pack("LA*L"))
+    end
+  end
+
+  # Marking this as private so that consumers don't see it. It makes it a little
+  # annoying for testing since you have to const_get it to access the methods,
+  # but at least this way it's clear it's not meant for consumers.
+  private_constant :Debug
+end
+
+require_relative "yarp/lex_compat"
+require_relative "yarp/node"
+require_relative "yarp/ripper_compat"
+require_relative "yarp/serialize"
+require_relative "yarp/pack"
+
+require "yarp/yarp"

data/src/diagnostic.c

@@ -0,0 +1,25 @@
+#include "yarp/diagnostic.h"
+
+// Append an error to the given list of diagnostic.
+bool
+yp_diagnostic_list_append(yp_list_t *list, const char *start, const char *end, const char *message) {
+    yp_diagnostic_t *diagnostic = (yp_diagnostic_t *) malloc(sizeof(yp_diagnostic_t));
+    if (diagnostic == NULL) return false;
+
+    *diagnostic = (yp_diagnostic_t) { .start = start, .end = end, .message = message };
+    yp_list_append(list, (yp_list_node_t *) diagnostic);
+    return true;
+}
+
+// Deallocate the internal state of the given diagnostic list.
+void
+yp_diagnostic_list_free(yp_list_t *list) {
+    yp_list_node_t *node, *next;
+
+    for (node = list->head; node != NULL; node = next) {
+        next = node->next;
+
+        yp_diagnostic_t *diagnostic = (yp_diagnostic_t *) node;
+        free(diagnostic);
+    }
+}

data/src/enc/yp_big5.c

@@ -0,0 +1,79 @@
+#include "yarp/enc/yp_encoding.h"
+
+typedef uint16_t yp_big5_codepoint_t;
+
+static yp_big5_codepoint_t
+yp_big5_codepoint(const char *c, ptrdiff_t n, size_t *width) {
+    const unsigned char *uc = (const unsigned char *) c;
+
+    // These are the single byte characters.
+    if (*uc < 0x80) {
+        *width = 1;
+        return *uc;
+    }
+
+    // These are the double byte characters.
+    if ((n > 1) && (uc[0] >= 0xA1 && uc[0] <= 0xFE) && (uc[1] >= 0x40 && uc[1] <= 0xFE)) {
+        *width = 2;
+        return (yp_big5_codepoint_t) (uc[0] << 8 | uc[1]);
+    }
+
+    *width = 0;
+    return 0;
+}
+
+static size_t
+yp_encoding_big5_char_width(const char *c, ptrdiff_t n) {
+    size_t width;
+    yp_big5_codepoint(c, n, &width);
+
+    return width;
+}
+
+static size_t
+yp_encoding_big5_alpha_char(const char *c, ptrdiff_t n) {
+    size_t width;
+    yp_big5_codepoint_t codepoint = yp_big5_codepoint(c, n, &width);
+
+    if (width == 1) {
+        const char value = (const char) codepoint;
+        return yp_encoding_ascii_alpha_char(&value, n);
+    } else {
+        return 0;
+    }
+}
+
+static size_t
+yp_encoding_big5_alnum_char(const char *c, ptrdiff_t n) {
+    size_t width;
+    yp_big5_codepoint_t codepoint = yp_big5_codepoint(c, n, &width);
+
+    if (width == 1) {
+        const char value = (const char) codepoint;
+        return yp_encoding_ascii_alnum_char(&value, n);
+    } else {
+        return 0;
+    }
+}
+
+static bool
+yp_encoding_big5_isupper_char(const char *c, ptrdiff_t n) {
+    size_t width;
+    yp_big5_codepoint_t codepoint = yp_big5_codepoint(c, n, &width);
+
+    if (width == 1) {
+        const char value = (const char) codepoint;
+        return yp_encoding_ascii_isupper_char(&value, n);
+    } else {
+        return false;
+    }
+}
+
+yp_encoding_t yp_encoding_big5 = {
+    .name = "big5",
+    .char_width = yp_encoding_big5_char_width,
+    .alnum_char = yp_encoding_big5_alnum_char,
+    .alpha_char = yp_encoding_big5_alpha_char,
+    .isupper_char = yp_encoding_big5_isupper_char,
+    .multibyte = true
+};

data/src/enc/yp_euc_jp.c

@@ -0,0 +1,85 @@
+#include "yarp/enc/yp_encoding.h"
+
+typedef uint16_t yp_euc_jp_codepoint_t;
+
+static yp_euc_jp_codepoint_t
+yp_euc_jp_codepoint(const char *c, ptrdiff_t n, size_t *width) {
+    const unsigned char *uc = (const unsigned char *) c;
+
+    // These are the single byte characters.
+    if (*uc < 0x80) {
+        *width = 1;
+        return *uc;
+    }
+
+    // These are the double byte characters.
+    if (
+        (n > 1) &&
+        (
+            ((uc[0] == 0x8E) && (uc[1] >= 0xA1 && uc[1] <= 0xFE)) ||
+            ((uc[0] >= 0xA1 && uc[0] <= 0xFE) && (uc[1] >= 0xA1 && uc[1] <= 0xFE))
+        )
+    ) {
+        *width = 2;
+        return (yp_euc_jp_codepoint_t) (uc[0] << 8 | uc[1]);
+    }
+
+    *width = 0;
+    return 0;
+}
+
+static size_t
+yp_encoding_euc_jp_char_width(const char *c, ptrdiff_t n) {
+    size_t width;
+    yp_euc_jp_codepoint(c, n, &width);
+
+    return width;
+}
+
+static size_t
+yp_encoding_euc_jp_alpha_char(const char *c, ptrdiff_t n) {
+    size_t width;
+    yp_euc_jp_codepoint_t codepoint = yp_euc_jp_codepoint(c, n, &width);
+
+    if (width == 1) {
+        const char value = (const char) codepoint;
+        return yp_encoding_ascii_alpha_char(&value, n);
+    } else {
+        return 0;
+    }
+}
+
+static size_t
+yp_encoding_euc_jp_alnum_char(const char *c, ptrdiff_t n) {
+    size_t width;
+    yp_euc_jp_codepoint_t codepoint = yp_euc_jp_codepoint(c, n, &width);
+
+    if (width == 1) {
+        const char value = (const char) codepoint;
+        return yp_encoding_ascii_alnum_char(&value, n);
+    } else {
+        return 0;
+    }
+}
+
+static bool
+yp_encoding_euc_jp_isupper_char(const char *c, ptrdiff_t n) {
+    size_t width;
+    yp_euc_jp_codepoint_t codepoint = yp_euc_jp_codepoint(c, n, &width);
+
+    if (width == 1) {
+        const char value = (const char) codepoint;
+        return yp_encoding_ascii_isupper_char(&value, n);
+    } else {
+        return 0;
+    }
+}
+
+yp_encoding_t yp_encoding_euc_jp = {
+    .name = "euc-jp",
+    .char_width = yp_encoding_euc_jp_char_width,
+    .alnum_char = yp_encoding_euc_jp_alnum_char,
+    .alpha_char = yp_encoding_euc_jp_alpha_char,
+    .isupper_char = yp_encoding_euc_jp_isupper_char,
+    .multibyte = true
+};

data/src/enc/yp_gbk.c

@@ -0,0 +1,88 @@
+#include "yarp/enc/yp_encoding.h"
+
+typedef uint16_t yp_gbk_codepoint_t;
+
+static yp_gbk_codepoint_t
+yp_gbk_codepoint(const char *c, ptrdiff_t n, size_t *width) {
+    const unsigned char *uc = (const unsigned char *) c;
+
+    // These are the single byte characters.
+    if (*uc < 0x80) {
+        *width = 1;
+        return *uc;
+    }
+
+    // These are the double byte characters.
+    if (
+        (n > 1) &&
+        (
+            ((uc[0] >= 0xA1 && uc[0] <= 0xA9) && (uc[1] >= 0xA1 && uc[1] <= 0xFE)) || // GBK/1
+            ((uc[0] >= 0xB0 && uc[0] <= 0xF7) && (uc[1] >= 0xA1 && uc[1] <= 0xFE)) || // GBK/2
+            ((uc[0] >= 0x81 && uc[0] <= 0xA0) && (uc[1] >= 0x40 && uc[1] <= 0xFE) && (uc[1] != 0x7F)) || // GBK/3
+            ((uc[0] >= 0xAA && uc[0] <= 0xFE) && (uc[1] >= 0x40 && uc[1] <= 0xA0) && (uc[1] != 0x7F)) || // GBK/4
+            ((uc[0] >= 0xA8 && uc[0] <= 0xA9) && (uc[1] >= 0x40 && uc[1] <= 0xA0) && (uc[1] != 0x7F)) // GBK/5
+        )
+    ) {
+        *width = 2;
+        return (yp_gbk_codepoint_t) (uc[0] << 8 | uc[1]);
+    }
+
+    *width = 0;
+    return 0;
+}
+
+static size_t
+yp_encoding_gbk_char_width(const char *c, ptrdiff_t n) {
+    size_t width;
+    yp_gbk_codepoint(c, n, &width);
+
+    return width;
+}
+
+static size_t
+yp_encoding_gbk_alpha_char(const char *c, ptrdiff_t n) {
+    size_t width;
+    yp_gbk_codepoint_t codepoint = yp_gbk_codepoint(c, n, &width);
+
+    if (width == 1) {
+        const char value = (const char) codepoint;
+        return yp_encoding_ascii_alpha_char(&value, n);
+    } else {
+        return 0;
+    }
+}
+
+static size_t
+yp_encoding_gbk_alnum_char(const char *c, ptrdiff_t n) {
+    size_t width;
+    yp_gbk_codepoint_t codepoint = yp_gbk_codepoint(c, n, &width);
+
+    if (width == 1) {
+        const char value = (const char) codepoint;
+        return yp_encoding_ascii_alnum_char(&value, n);
+    } else {
+        return 0;
+    }
+}
+
+static bool
+yp_encoding_gbk_isupper_char(const char *c, ptrdiff_t n) {
+    size_t width;
+    yp_gbk_codepoint_t codepoint = yp_gbk_codepoint(c, n, &width);
+
+    if (width == 1) {
+        const char value = (const char) codepoint;
+        return yp_encoding_ascii_isupper_char(&value, n);
+    } else {
+        return false;
+    }
+}
+
+yp_encoding_t yp_encoding_gbk = {
+    .name = "gbk",
+    .char_width = yp_encoding_gbk_char_width,
+    .alnum_char = yp_encoding_gbk_alnum_char,
+    .alpha_char = yp_encoding_gbk_alpha_char,
+    .isupper_char = yp_encoding_gbk_isupper_char,
+    .multibyte = true
+};

data/src/enc/yp_shift_jis.c

@@ -0,0 +1,83 @@
+#include "yarp/enc/yp_encoding.h"
+
+typedef uint16_t yp_shift_jis_codepoint_t;
+
+static yp_shift_jis_codepoint_t
+yp_shift_jis_codepoint(const char *c, ptrdiff_t n, size_t *width) {
+    const unsigned char *uc = (const unsigned char *) c;
+
+    // These are the single byte characters.
+    if (*uc < 0x80 || (*uc >= 0xA1 && *uc <= 0xDF)) {
+        *width = 1;
+        return *uc;
+    }
+
+    // These are the double byte characters.
+    if (
+        (n > 1) &&
+        ((uc[0] >= 0x81 && uc[0] <= 0x9F) || (uc[0] >= 0xE0 && uc[0] <= 0xFC)) &&
+        (uc[1] >= 0x40 && uc[1] <= 0xFC)
+    ) {
+        *width = 2;
+        return (yp_shift_jis_codepoint_t) (uc[0] << 8 | uc[1]);
+    }
+
+    *width = 0;
+    return 0;
+}
+
+static size_t
+yp_encoding_shift_jis_char_width(const char *c, ptrdiff_t n) {
+    size_t width;
+    yp_shift_jis_codepoint(c, n, &width);
+
+    return width;
+}
+
+static size_t
+yp_encoding_shift_jis_alpha_char(const char *c, ptrdiff_t n) {
+    size_t width;
+    yp_shift_jis_codepoint_t codepoint = yp_shift_jis_codepoint(c, n, &width);
+
+    if (width == 1) {
+        const char value = (const char) codepoint;
+        return yp_encoding_ascii_alpha_char(&value, n);
+    } else {
+        return 0;
+    }
+}
+
+static size_t
+yp_encoding_shift_jis_alnum_char(const char *c, ptrdiff_t n) {
+    size_t width;
+    yp_shift_jis_codepoint_t codepoint = yp_shift_jis_codepoint(c, n, &width);
+
+    if (width == 1) {
+        const char value = (const char) codepoint;
+        return yp_encoding_ascii_alnum_char(&value, n);
+    } else {
+        return 0;
+    }
+}
+
+static bool
+yp_encoding_shift_jis_isupper_char(const char *c, ptrdiff_t n) {
+    size_t width;
+    yp_shift_jis_codepoint_t codepoint = yp_shift_jis_codepoint(c, n, &width);
+
+    if (width == 1) {
+        const char value = (const char) codepoint;
+        return yp_encoding_ascii_isupper_char(&value, n);
+    } else {
+        return 0;
+    }
+}
+
+yp_encoding_t yp_encoding_shift_jis = {
+    .name = "shift_jis",
+    .char_width = yp_encoding_shift_jis_char_width,
+    .alnum_char = yp_encoding_shift_jis_alnum_char,
+    .alpha_char = yp_encoding_shift_jis_alpha_char,
+    .isupper_char = yp_encoding_shift_jis_isupper_char,
+    .multibyte = true
+};