prism 0.24.0 → 0.29.0

data/src/util/pm_string.c

@@ -58,7 +58,7 @@ pm_string_constant_init(pm_string_t *string, const char *source, size_t length)
  * `MapViewOfFile`, on POSIX systems that have access to `mmap` we'll use
  * `mmap`, and on other POSIX systems we'll use `read`.
  */
-bool
+PRISM_EXPORTED_FUNCTION bool
 pm_string_mapped_init(pm_string_t *string, const char *filepath) {
 #ifdef _WIN32
     // Open the file for reading.
@@ -102,7 +102,7 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
 
     *string = (pm_string_t) { .type = PM_STRING_MAPPED, .source = source, .length = (size_t) file_size };
     return true;
-#else
+#elif defined(_POSIX_MAPPED_FILES)
     // Open the file for reading
     int fd = open(filepath, O_RDONLY);
     if (fd == -1) {
@@ -135,6 +135,113 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
     close(fd);
     *string = (pm_string_t) { .type = PM_STRING_MAPPED, .source = source, .length = size };
     return true;
+#else
+    (void) string;
+    (void) filepath;
+    perror("pm_string_mapped_init is not implemented for this platform");
+    return false;
+#endif
+}
+
+/**
+ * Read the file indicated by the filepath parameter into source and load its
+ * contents and size into the given `pm_string_t`. The given `pm_string_t`
+ * should be freed using `pm_string_free` when it is no longer used.
+ */
+PRISM_EXPORTED_FUNCTION bool
+pm_string_file_init(pm_string_t *string, const char *filepath) {
+#ifdef _WIN32
+    // Open the file for reading.
+    HANDLE file = CreateFile(filepath, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+
+    if (file == INVALID_HANDLE_VALUE) {
+        return false;
+    }
+
+    // Get the file size.
+    DWORD file_size = GetFileSize(file, NULL);
+    if (file_size == INVALID_FILE_SIZE) {
+        CloseHandle(file);
+        return false;
+    }
+
+    // If the file is empty, then we don't need to do anything else, we'll set
+    // the source to a constant empty string and return.
+    if (file_size == 0) {
+        CloseHandle(file);
+        const uint8_t source[] = "";
+        *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = source, .length = 0 };
+        return true;
+    }
+
+    // Create a buffer to read the file into.
+    uint8_t *source = xmalloc(file_size);
+    if (source == NULL) {
+        CloseHandle(file);
+        return false;
+    }
+
+    // Read the contents of the file
+    DWORD bytes_read;
+    if (!ReadFile(file, source, file_size, &bytes_read, NULL)) {
+        CloseHandle(file);
+        return false;
+    }
+
+    // Check the number of bytes read
+    if (bytes_read != file_size) {
+        xfree(source);
+        CloseHandle(file);
+        return false;
+    }
+
+    CloseHandle(file);
+    *string = (pm_string_t) { .type = PM_STRING_OWNED, .source = source, .length = (size_t) file_size };
+    return true;
+#elif defined(_POSIX_MAPPED_FILES)
+    FILE *file = fopen(filepath, "rb");
+    if (file == NULL) {
+        return false;
+    }
+
+    fseek(file, 0, SEEK_END);
+    long file_size = ftell(file);
+
+    if (file_size == -1) {
+        fclose(file);
+        return false;
+    }
+
+    if (file_size == 0) {
+        fclose(file);
+        const uint8_t source[] = "";
+        *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = source, .length = 0 };
+        return true;
+    }
+
+    size_t length = (size_t) file_size;
+    uint8_t *source = xmalloc(length);
+    if (source == NULL) {
+        fclose(file);
+        return false;
+    }
+
+    fseek(file, 0, SEEK_SET);
+    size_t bytes_read = fread(source, length, 1, file);
+    fclose(file);
+
+    if (bytes_read != 1) {
+        xfree(source);
+        return false;
+    }
+
+    *string = (pm_string_t) { .type = PM_STRING_OWNED, .source = source, .length = length };
+    return true;
+#else
+    (void) string;
+    (void) filepath;
+    perror("pm_string_file_init is not implemented for this platform");
+    return false;
 #endif
 }
 
@@ -161,13 +268,33 @@ pm_string_ensure_owned(pm_string_t *string) {
     size_t length = pm_string_length(string);
     const uint8_t *source = pm_string_source(string);
 
-    uint8_t *memory = malloc(length);
+    uint8_t *memory = xmalloc(length);
     if (!memory) return;
 
     pm_string_owned_init(string, memory, length);
     memcpy((void *) string->source, source, length);
 }
 
+/**
+ * Compare the underlying lengths and bytes of two strings. Returns 0 if the
+ * strings are equal, a negative number if the left string is less than the
+ * right string, and a positive number if the left string is greater than the
+ * right string.
+ */
+int
+pm_string_compare(const pm_string_t *left, const pm_string_t *right) {
+    size_t left_length = pm_string_length(left);
+    size_t right_length = pm_string_length(right);
+
+    if (left_length < right_length) {
+        return -1;
+    } else if (left_length > right_length) {
+        return 1;
+    }
+
+    return memcmp(pm_string_source(left), pm_string_source(right), left_length);
+}
+
 /**
  * Returns the length associated with the string.
  */
@@ -192,12 +319,14 @@ pm_string_free(pm_string_t *string) {
     void *memory = (void *) string->source;
 
     if (string->type == PM_STRING_OWNED) {
-        free(memory);
+        xfree(memory);
+#ifdef PRISM_HAS_MMAP
     } else if (string->type == PM_STRING_MAPPED && string->length) {
 #if defined(_WIN32)
         UnmapViewOfFile(memory);
-#else
+#elif defined(_POSIX_MAPPED_FILES)
         munmap(memory, string->length);
 #endif
+#endif /* PRISM_HAS_MMAP */
     }
 }

data/src/util/pm_string_list.c

@@ -12,7 +12,7 @@ pm_string_list_append(pm_string_list_t *string_list, pm_string_t *string) {
             string_list->capacity *= 2;
         }
 
-        string_list->strings = realloc(string_list->strings, string_list->capacity * sizeof(pm_string_t));
+        string_list->strings = xrealloc(string_list->strings, string_list->capacity * sizeof(pm_string_t));
         if (string_list->strings == NULL) abort();
     }
 
@@ -24,5 +24,5 @@ pm_string_list_append(pm_string_list_t *string_list, pm_string_t *string) {
  */
 void
 pm_string_list_free(pm_string_list_t *string_list) {
-    free(string_list->strings);
+    xfree(string_list->strings);
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prism
 version: !ruby/object:Gem::Version
-  version: 0.24.0
+  version: 0.29.0
 platform: ruby
 authors:
 - Shopify
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-15 00:00:00.000000000 Z
+date: 2024-05-10 00:00:00.000000000 Z
 dependencies: []
 description:
 email:
@@ -18,6 +18,7 @@ extensions:
 - ext/prism/extconf.rb
 extra_rdoc_files: []
 files:
+- BSDmakefile
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - CONTRIBUTING.md
@@ -38,7 +39,7 @@ files:
 - docs/parser_translation.md
 - docs/parsing_rules.md
 - docs/releasing.md
-- docs/ripper.md
+- docs/ripper_translation.md
 - docs/ruby_api.md
 - docs/ruby_parser_translation.md
 - docs/serialization.md
@@ -59,13 +60,14 @@ files:
 - include/prism/parser.h
 - include/prism/prettyprint.h
 - include/prism/regexp.h
+- include/prism/static_literals.h
 - include/prism/util/pm_buffer.h
 - include/prism/util/pm_char.h
 - include/prism/util/pm_constant_pool.h
+- include/prism/util/pm_integer.h
 - include/prism/util/pm_list.h
 - include/prism/util/pm_memchr.h
 - include/prism/util/pm_newline_list.h
-- include/prism/util/pm_state_stack.h
 - include/prism/util/pm_string.h
 - include/prism/util/pm_string_list.h
 - include/prism/util/pm_strncasecmp.h
@@ -79,16 +81,19 @@ files:
 - lib/prism/dot_visitor.rb
 - lib/prism/dsl.rb
 - lib/prism/ffi.rb
+- lib/prism/inspect_visitor.rb
 - lib/prism/lex_compat.rb
 - lib/prism/mutation_compiler.rb
 - lib/prism/node.rb
 - lib/prism/node_ext.rb
-- lib/prism/node_inspector.rb
 - lib/prism/pack.rb
 - lib/prism/parse_result.rb
 - lib/prism/parse_result/comments.rb
 - lib/prism/parse_result/newlines.rb
 - lib/prism/pattern.rb
+- lib/prism/polyfill/byteindex.rb
+- lib/prism/polyfill/unpack1.rb
+- lib/prism/reflection.rb
 - lib/prism/serialize.rb
 - lib/prism/translation.rb
 - lib/prism/translation/parser.rb
@@ -98,13 +103,39 @@ files:
 - lib/prism/translation/parser33.rb
 - lib/prism/translation/parser34.rb
 - lib/prism/translation/ripper.rb
+- lib/prism/translation/ripper/sexp.rb
+- lib/prism/translation/ripper/shim.rb
 - lib/prism/translation/ruby_parser.rb
 - lib/prism/visitor.rb
 - prism.gemspec
 - rbi/prism.rbi
-- rbi/prism_static.rbi
+- rbi/prism/compiler.rbi
+- rbi/prism/inspect_visitor.rbi
+- rbi/prism/node.rbi
+- rbi/prism/node_ext.rbi
+- rbi/prism/parse_result.rbi
+- rbi/prism/reflection.rbi
+- rbi/prism/translation/parser.rbi
+- rbi/prism/translation/parser33.rbi
+- rbi/prism/translation/parser34.rbi
+- rbi/prism/translation/ripper.rbi
+- rbi/prism/visitor.rbi
 - sig/prism.rbs
-- sig/prism_static.rbs
+- sig/prism/compiler.rbs
+- sig/prism/dispatcher.rbs
+- sig/prism/dot_visitor.rbs
+- sig/prism/dsl.rbs
+- sig/prism/inspect_visitor.rbs
+- sig/prism/lex_compat.rbs
+- sig/prism/mutation_compiler.rbs
+- sig/prism/node.rbs
+- sig/prism/node_ext.rbs
+- sig/prism/pack.rbs
+- sig/prism/parse_result.rbs
+- sig/prism/pattern.rbs
+- sig/prism/reflection.rbs
+- sig/prism/serialize.rbs
+- sig/prism/visitor.rbs
 - src/diagnostic.c
 - src/encoding.c
 - src/node.c
@@ -114,14 +145,15 @@ files:
 - src/prism.c
 - src/regexp.c
 - src/serialize.c
+- src/static_literals.c
 - src/token_type.c
 - src/util/pm_buffer.c
 - src/util/pm_char.c
 - src/util/pm_constant_pool.c
+- src/util/pm_integer.c
 - src/util/pm_list.c
 - src/util/pm_memchr.c
 - src/util/pm_newline_list.c
-- src/util/pm_state_stack.c
 - src/util/pm_string.c
 - src/util/pm_string_list.c
 - src/util/pm_strncasecmp.c
@@ -148,7 +180,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
+rubygems_version: 3.6.0.dev
 signing_key:
 specification_version: 4
 summary: Prism Ruby parser

data/docs/ripper.md

@@ -1,36 +0,0 @@
-# Ripper
-
-To test the parser, we compare against the output from `Ripper`, both for testing the lexer and testing the parser. The lexer test suite is much more feature complete at the moment.
-
-To lex source code using `prism`, you typically would run `Prism.lex(source)`. If you want to instead get output that `Ripper` would normally produce, you can run `Prism.lex_compat(source)`. This will produce tokens that should be equivalent to `Ripper`.
-
-To parse source code using `prism`, you typically would run `Prism.parse(source)`. If you want to instead using the `Ripper` streaming interface, you can inherit from `Prism::RipperCompat` and override the `on_*` methods. This will produce a syntax tree that should be equivalent to `Ripper`. That would look like:
-
-```ruby
-class ArithmeticRipper < Prism::RipperCompat
-  def on_binary(left, operator, right)
-    left.public_send(operator, right)
-  end
-
-  def on_int(value)
-    value.to_i
-  end
-
-  def on_program(stmts)
-    stmts
-  end
-
-  def on_stmts_new
-    []
-  end
-
-  def on_stmts_add(stmts, stmt)
-    stmts << stmt
-    stmts
-  end
-end
-
-ArithmeticRipper.new("1 + 2 - 3").parse # => [0]
-```
-
-There are also APIs for building trees similar to the s-expression builders in `Ripper`. The method names are the same. These include `Prism::RipperCompat.sexp_raw(source)` and `Prism::RipperCompat.sexp(source)`.

data/include/prism/util/pm_state_stack.h

@@ -1,42 +0,0 @@
-/**
- * @file pm_state_stack.h
- *
- * A stack of boolean values.
- */
-#ifndef PRISM_STATE_STACK_H
-#define PRISM_STATE_STACK_H
-
-#include "prism/defines.h"
-
-#include <stdbool.h>
-#include <stdint.h>
-
-/**
- * A struct that represents a stack of boolean values.
- */
-typedef uint32_t pm_state_stack_t;
-
-/**
- * Pushes a value onto the stack.
- *
- * @param stack The stack to push the value onto.
- * @param value The value to push onto the stack.
- */
-void pm_state_stack_push(pm_state_stack_t *stack, bool value);
-
-/**
- * Pops a value off the stack.
- *
- * @param stack The stack to pop the value off of.
- */
-void pm_state_stack_pop(pm_state_stack_t *stack);
-
-/**
- * Returns the value at the top of the stack.
- *
- * @param stack The stack to get the value from.
- * @return The value at the top of the stack.
- */
-bool pm_state_stack_p(pm_state_stack_t *stack);
-
-#endif

data/lib/prism/node_inspector.rb

@@ -1,68 +0,0 @@
-# frozen_string_literal: true
-
-module Prism
-  # This object is responsible for generating the output for the inspect method
-  # implementations of child nodes.
-  class NodeInspector # :nodoc:
-    attr_reader :prefix, :output
-
-    def initialize(prefix = "")
-      @prefix = prefix
-      @output = +""
-    end
-
-    # Appends a line to the output with the current prefix.
-    def <<(line)
-      output << "#{prefix}#{line}"
-    end
-
-    # This generates a string that is used as the header of the inspect output
-    # for any given node.
-    def header(node)
-      output = +"@ #{node.class.name.split("::").last} ("
-      output << "location: (#{node.location.start_line},#{node.location.start_column})-(#{node.location.end_line},#{node.location.end_column})"
-      output << ", newline: true" if node.newline?
-      output << ")\n"
-      output
-    end
-
-    # Generates a string that represents a list of nodes. It handles properly
-    # using the box drawing characters to make the output look nice.
-    def list(prefix, nodes)
-      output = +"(length: #{nodes.length})\n"
-      last_index = nodes.length - 1
-
-      nodes.each_with_index do |node, index|
-        pointer, preadd = (index == last_index) ? ["└── ", "    "] : ["├── ", "│   "]
-        node_prefix = "#{prefix}#{preadd}"
-        output << node.inspect(NodeInspector.new(node_prefix)).sub(node_prefix, "#{prefix}#{pointer}")
-      end
-
-      output
-    end
-
-    # Generates a string that represents a location field on a node.
-    def location(value)
-      if value
-        "(#{value.start_line},#{value.start_column})-(#{value.end_line},#{value.end_column}) = #{value.slice.inspect}"
-      else
-        "∅"
-      end
-    end
-
-    # Generates a string that represents a child node.
-    def child_node(node, append)
-      node.inspect(child_inspector(append)).delete_prefix(prefix)
-    end
-
-    # Returns a new inspector that can be used to inspect a child node.
-    def child_inspector(append)
-      NodeInspector.new("#{prefix}#{append}")
-    end
-
-    # Returns the output as a string.
-    def to_str
-      output
-    end
-  end
-end

data/rbi/prism_static.rbi

@@ -1,207 +0,0 @@
-class Prism::ParseResult
-  sig { returns(Prism::ProgramNode) }
-  def value; end
-
-  sig { returns(T::Array[Prism::Comment]) }
-  def comments; end
-
-  sig { returns(T::Array[Prism::ParseError]) }
-  def errors; end
-
-  sig { returns(T::Array[Prism::ParseWarning]) }
-  def warnings; end
-
-  sig { returns(Prism::Source) }
-  def source; end
-end
-
-class Prism::ParseError
-  sig { returns(String) }
-  def message; end
-
-  sig { returns(Prism::Location) }
-  def location; end
-end
-
-class Prism::ParseWarning
-  sig { returns(String) }
-  def message; end
-
-  sig { returns(Prism::Location) }
-  def location; end
-end
-
-class Prism::Node
-  sig { returns(Prism::Location) }
-  def location; end
-
-  sig { returns(String) }
-  def slice; end
-
-  sig { returns(String) }
-  def to_dot; end
-
-  sig { params(visitor: Prism::Visitor).void }
-  def accept(visitor); end
-
-  sig { returns(T::Array[T.nilable(Prism::Node)]) }
-  def child_nodes; end
-
-  sig { returns(T::Array[Prism::Node]) }
-  def compact_child_nodes; end
-
-  sig { returns(T::Array[T.nilable(Prism::Node)]) }
-  def deconstruct; end
-
-  sig { returns(Symbol) }
-  def type; end
-end
-
-class Prism::Comment
-  sig { returns(Prism::Location) }
-  def location; end
-
-  sig { returns(T::Boolean) }
-  def trailing?; end
-end
-
-class Prism::InlineComment < Prism::Comment
-  sig { override.returns(T::Boolean) }
-  def trailing?; end
-end
-
-class Prism::EmbDocComment < Prism::Comment
-end
-
-class Prism::DATAComment < Prism::Comment
-end
-
-class Prism::Location
-  sig { params(source: Prism::Source, start_offset: Integer, length: Integer).void }
-  def initialize(source, start_offset, length); end
-
-  sig { returns(String) }
-  def slice; end
-
-  sig { returns(T::Array[Prism::Comment]) }
-  def comments; end
-
-  sig { params(options: T.untyped).returns(Prism::Location) }
-  def copy(**options); end
-
-  sig { returns(Integer) }
-  def start_offset; end
-
-  sig { returns(Integer) }
-  def end_offset; end
-
-  sig { returns(Integer) }
-  def start_line; end
-
-  sig { returns(Integer) }
-  def end_line; end
-
-  sig { returns(Integer) }
-  def start_column; end
-
-  sig { returns(Integer) }
-  def end_column; end
-end
-
-class Prism::Source
-  sig { params(source: String, start_line: Integer, offsets: T::Array[Integer]).void }
-  def initialize(source, start_line, offsets); end
-
-  sig { params(offset: Integer, length: Integer).returns(String) }
-  def slice(offset, length); end
-
-  sig { params(value: Integer).returns(Integer) }
-  def line(value); end
-
-  sig { params(value: Integer).returns(Integer) }
-  def line_offset(value); end
-
-  sig { params(value: Integer).returns(Integer) }
-  def column(value); end
-
-  sig { returns(String) }
-  def source; end
-
-  sig { returns(T::Array[Integer]) }
-  def offsets; end
-end
-
-class Prism::Token
-  sig { params(type: T.untyped, value: String, location: Prism::Location).void }
-  def initialize(type, value, location); end
-
-  sig { params(keys: T.untyped).returns(T.untyped) }
-  def deconstruct_keys(keys); end
-
-  sig { params(q: T.untyped).returns(T.untyped) }
-  def pretty_print(q); end
-
-  sig { params(other: T.untyped).returns(T::Boolean) }
-  def ==(other); end
-
-  sig { returns(T.untyped) }
-  def type; end
-
-  sig { returns(String) }
-  def value; end
-
-  sig { returns(Prism::Location) }
-  def location; end
-end
-
-class Prism::NodeInspector
-  sig { params(prefix: String).void }
-  def initialize(prefix); end
-
-  sig { returns(String) }
-  def prefix; end
-
-  sig { returns(String) }
-  def output; end
-
-  # Appends a line to the output with the current prefix.
-  sig { params(line: String).void }
-  def <<(line); end
-
-  # This generates a string that is used as the header of the inspect output
-  # for any given node.
-  sig { params(node: Prism::Node).returns(String) }
-  def header(node); end
-
-  # Generates a string that represents a list of nodes. It handles properly
-  # using the box drawing characters to make the output look nice.
-  sig { params(prefix: String, nodes: T::Array[Prism::Node]).returns(String) }
-  def list(prefix, nodes); end
-
-  # Generates a string that represents a location field on a node.
-  sig { params(value: Prism::Location).returns(String) }
-  def location(value); end
-
-  # Generates a string that represents a child node.
-  sig { params(node: Prism::Node, append: String).returns(String) }
-  def child_node(node, append); end
-
-  # Returns a new inspector that can be used to inspect a child node.
-  sig { params(append: String).returns(Prism::NodeInspector) }
-  def child_inspector(append); end
-
-  # Returns the output as a string.
-  sig { returns(String) }
-  def to_str; end
-end
-
-class Prism::BasicVisitor
-  sig { params(node: T.nilable(Prism::Node)).void }
-  def visit(node); end
-
-  sig { params(nodes: T::Array[T.nilable(Prism::Node)]).void }
-  def visit_all(nodes); end
-
-  sig { params(node: Prism::Node).void }
-  def visit_child_nodes(node); end
-end