prism 0.24.0 → 0.25.0

data/src/util/pm_string.c

@@ -58,7 +58,7 @@ pm_string_constant_init(pm_string_t *string, const char *source, size_t length)
  * `MapViewOfFile`, on POSIX systems that have access to `mmap` we'll use
  * `mmap`, and on other POSIX systems we'll use `read`.
  */
-bool
+PRISM_EXPORTED_FUNCTION bool
 pm_string_mapped_init(pm_string_t *string, const char *filepath) {
 #ifdef _WIN32
     // Open the file for reading.
@@ -102,7 +102,7 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
 
     *string = (pm_string_t) { .type = PM_STRING_MAPPED, .source = source, .length = (size_t) file_size };
     return true;
-#else
+#elif defined(_POSIX_MAPPED_FILES)
     // Open the file for reading
     int fd = open(filepath, O_RDONLY);
     if (fd == -1) {
@@ -135,6 +135,113 @@ pm_string_mapped_init(pm_string_t *string, const char *filepath) {
     close(fd);
     *string = (pm_string_t) { .type = PM_STRING_MAPPED, .source = source, .length = size };
     return true;
+#else
+    (void) string;
+    (void) filepath;
+    perror("pm_string_mapped_init is not implemented for this platform");
+    return false;
+#endif
+}
+
+/**
+ * Read the file indicated by the filepath parameter into source and load its
+ * contents and size into the given `pm_string_t`. The given `pm_string_t`
+ * should be freed using `pm_string_free` when it is no longer used.
+ */
+PRISM_EXPORTED_FUNCTION bool
+pm_string_file_init(pm_string_t *string, const char *filepath) {
+#ifdef _WIN32
+    // Open the file for reading.
+    HANDLE file = CreateFile(filepath, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+
+    if (file == INVALID_HANDLE_VALUE) {
+        return false;
+    }
+
+    // Get the file size.
+    DWORD file_size = GetFileSize(file, NULL);
+    if (file_size == INVALID_FILE_SIZE) {
+        CloseHandle(file);
+        return false;
+    }
+
+    // If the file is empty, then we don't need to do anything else, we'll set
+    // the source to a constant empty string and return.
+    if (file_size == 0) {
+        CloseHandle(file);
+        const uint8_t source[] = "";
+        *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = source, .length = 0 };
+        return true;
+    }
+
+    // Create a buffer to read the file into.
+    uint8_t *source = xmalloc(file_size);
+    if (source == NULL) {
+        CloseHandle(file);
+        return false;
+    }
+
+    // Read the contents of the file
+    DWORD bytes_read;
+    if (!ReadFile(file, source, file_size, &bytes_read, NULL)) {
+        CloseHandle(file);
+        return false;
+    }
+
+    // Check the number of bytes read
+    if (bytes_read != file_size) {
+        xfree(source);
+        CloseHandle(file);
+        return false;
+    }
+
+    CloseHandle(file);
+    *string = (pm_string_t) { .type = PM_STRING_OWNED, .source = source, .length = (size_t) file_size };
+    return true;
+#elif defined(_POSIX_MAPPED_FILES)
+    FILE *file = fopen(filepath, "rb");
+    if (file == NULL) {
+        return false;
+    }
+
+    fseek(file, 0, SEEK_END);
+    long file_size = ftell(file);
+
+    if (file_size == -1) {
+        fclose(file);
+        return false;
+    }
+
+    if (file_size == 0) {
+        fclose(file);
+        const uint8_t source[] = "";
+        *string = (pm_string_t) { .type = PM_STRING_CONSTANT, .source = source, .length = 0 };
+        return true;
+    }
+
+    size_t length = (size_t) file_size;
+    uint8_t *source = xmalloc(length);
+    if (source == NULL) {
+        fclose(file);
+        return false;
+    }
+
+    fseek(file, 0, SEEK_SET);
+    size_t bytes_read = fread(source, length, 1, file);
+    fclose(file);
+
+    if (bytes_read != 1) {
+        xfree(source);
+        return false;
+    }
+
+    *string = (pm_string_t) { .type = PM_STRING_OWNED, .source = source, .length = length };
+    return true;
+#else
+    (void) string;
+    (void) filepath;
+    perror("pm_string_file_init is not implemented for this platform");
+    return false;
 #endif
 }
 
@@ -161,13 +268,33 @@ pm_string_ensure_owned(pm_string_t *string) {
     size_t length = pm_string_length(string);
     const uint8_t *source = pm_string_source(string);
 
-    uint8_t *memory = malloc(length);
+    uint8_t *memory = xmalloc(length);
     if (!memory) return;
 
     pm_string_owned_init(string, memory, length);
     memcpy((void *) string->source, source, length);
 }
 
+/**
+ * Compare the underlying lengths and bytes of two strings. Returns 0 if the
+ * strings are equal, a negative number if the left string is less than the
+ * right string, and a positive number if the left string is greater than the
+ * right string.
+ */
+int
+pm_string_compare(const pm_string_t *left, const pm_string_t *right) {
+    size_t left_length = pm_string_length(left);
+    size_t right_length = pm_string_length(right);
+
+    if (left_length < right_length) {
+        return -1;
+    } else if (left_length > right_length) {
+        return 1;
+    }
+
+    return memcmp(pm_string_source(left), pm_string_source(right), left_length);
+}
+
 /**
  * Returns the length associated with the string.
  */
@@ -192,12 +319,14 @@ pm_string_free(pm_string_t *string) {
     void *memory = (void *) string->source;
 
     if (string->type == PM_STRING_OWNED) {
-        free(memory);
+        xfree(memory);
+#ifdef PRISM_HAS_MMAP
     } else if (string->type == PM_STRING_MAPPED && string->length) {
 #if defined(_WIN32)
         UnmapViewOfFile(memory);
-#else
+#elif defined(_POSIX_MAPPED_FILES)
         munmap(memory, string->length);
 #endif
+#endif /* PRISM_HAS_MMAP */
     }
 }

data/src/util/pm_string_list.c

@@ -12,7 +12,7 @@ pm_string_list_append(pm_string_list_t *string_list, pm_string_t *string) {
             string_list->capacity *= 2;
         }
 
-        string_list->strings = realloc(string_list->strings, string_list->capacity * sizeof(pm_string_t));
+        string_list->strings = xrealloc(string_list->strings, string_list->capacity * sizeof(pm_string_t));
         if (string_list->strings == NULL) abort();
     }
 
@@ -24,5 +24,5 @@ pm_string_list_append(pm_string_list_t *string_list, pm_string_t *string) {
  */
 void
 pm_string_list_free(pm_string_list_t *string_list) {
-    free(string_list->strings);
+    xfree(string_list->strings);
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: prism
 version: !ruby/object:Gem::Version
-  version: 0.24.0
+  version: 0.25.0
 platform: ruby
 authors:
 - Shopify
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-15 00:00:00.000000000 Z
+date: 2024-04-05 00:00:00.000000000 Z
 dependencies: []
 description:
 email:
@@ -18,6 +18,7 @@ extensions:
 - ext/prism/extconf.rb
 extra_rdoc_files: []
 files:
+- BSDmakefile
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - CONTRIBUTING.md
@@ -38,7 +39,7 @@ files:
 - docs/parser_translation.md
 - docs/parsing_rules.md
 - docs/releasing.md
-- docs/ripper.md
+- docs/ripper_translation.md
 - docs/ruby_api.md
 - docs/ruby_parser_translation.md
 - docs/serialization.md
@@ -59,9 +60,11 @@ files:
 - include/prism/parser.h
 - include/prism/prettyprint.h
 - include/prism/regexp.h
+- include/prism/static_literals.h
 - include/prism/util/pm_buffer.h
 - include/prism/util/pm_char.h
 - include/prism/util/pm_constant_pool.h
+- include/prism/util/pm_integer.h
 - include/prism/util/pm_list.h
 - include/prism/util/pm_memchr.h
 - include/prism/util/pm_newline_list.h
@@ -89,6 +92,7 @@ files:
 - lib/prism/parse_result/comments.rb
 - lib/prism/parse_result/newlines.rb
 - lib/prism/pattern.rb
+- lib/prism/polyfill/string.rb
 - lib/prism/serialize.rb
 - lib/prism/translation.rb
 - lib/prism/translation/parser.rb
@@ -98,13 +102,36 @@ files:
 - lib/prism/translation/parser33.rb
 - lib/prism/translation/parser34.rb
 - lib/prism/translation/ripper.rb
+- lib/prism/translation/ripper/sexp.rb
+- lib/prism/translation/ripper/shim.rb
 - lib/prism/translation/ruby_parser.rb
 - lib/prism/visitor.rb
 - prism.gemspec
 - rbi/prism.rbi
-- rbi/prism_static.rbi
+- rbi/prism/compiler.rbi
+- rbi/prism/desugar_compiler.rbi
+- rbi/prism/mutation_compiler.rbi
+- rbi/prism/node.rbi
+- rbi/prism/node_ext.rbi
+- rbi/prism/parse_result.rbi
+- rbi/prism/translation/parser/compiler.rbi
+- rbi/prism/translation/ripper.rbi
+- rbi/prism/translation/ripper/ripper_compiler.rbi
+- rbi/prism/translation/ruby_parser.rbi
+- rbi/prism/visitor.rbi
 - sig/prism.rbs
-- sig/prism_static.rbs
+- sig/prism/compiler.rbs
+- sig/prism/dispatcher.rbs
+- sig/prism/dot_visitor.rbs
+- sig/prism/dsl.rbs
+- sig/prism/mutation_compiler.rbs
+- sig/prism/node.rbs
+- sig/prism/node_ext.rbs
+- sig/prism/pack.rbs
+- sig/prism/parse_result.rbs
+- sig/prism/pattern.rbs
+- sig/prism/serialize.rbs
+- sig/prism/visitor.rbs
 - src/diagnostic.c
 - src/encoding.c
 - src/node.c
@@ -114,10 +141,12 @@ files:
 - src/prism.c
 - src/regexp.c
 - src/serialize.c
+- src/static_literals.c
 - src/token_type.c
 - src/util/pm_buffer.c
 - src/util/pm_char.c
 - src/util/pm_constant_pool.c
+- src/util/pm_integer.c
 - src/util/pm_list.c
 - src/util/pm_memchr.c
 - src/util/pm_newline_list.c
@@ -148,7 +177,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
+rubygems_version: 3.6.0.dev
 signing_key:
 specification_version: 4
 summary: Prism Ruby parser

data/docs/ripper.md

@@ -1,36 +0,0 @@
-# Ripper
-
-To test the parser, we compare against the output from `Ripper`, both for testing the lexer and testing the parser. The lexer test suite is much more feature complete at the moment.
-
-To lex source code using `prism`, you typically would run `Prism.lex(source)`. If you want to instead get output that `Ripper` would normally produce, you can run `Prism.lex_compat(source)`. This will produce tokens that should be equivalent to `Ripper`.
-
-To parse source code using `prism`, you typically would run `Prism.parse(source)`. If you want to instead using the `Ripper` streaming interface, you can inherit from `Prism::RipperCompat` and override the `on_*` methods. This will produce a syntax tree that should be equivalent to `Ripper`. That would look like:
-
-```ruby
-class ArithmeticRipper < Prism::RipperCompat
-  def on_binary(left, operator, right)
-    left.public_send(operator, right)
-  end
-
-  def on_int(value)
-    value.to_i
-  end
-
-  def on_program(stmts)
-    stmts
-  end
-
-  def on_stmts_new
-    []
-  end
-
-  def on_stmts_add(stmts, stmt)
-    stmts << stmt
-    stmts
-  end
-end
-
-ArithmeticRipper.new("1 + 2 - 3").parse # => [0]
-```
-
-There are also APIs for building trees similar to the s-expression builders in `Ripper`. The method names are the same. These include `Prism::RipperCompat.sexp_raw(source)` and `Prism::RipperCompat.sexp(source)`.

data/rbi/prism_static.rbi

@@ -1,207 +0,0 @@
-class Prism::ParseResult
-  sig { returns(Prism::ProgramNode) }
-  def value; end
-
-  sig { returns(T::Array[Prism::Comment]) }
-  def comments; end
-
-  sig { returns(T::Array[Prism::ParseError]) }
-  def errors; end
-
-  sig { returns(T::Array[Prism::ParseWarning]) }
-  def warnings; end
-
-  sig { returns(Prism::Source) }
-  def source; end
-end
-
-class Prism::ParseError
-  sig { returns(String) }
-  def message; end
-
-  sig { returns(Prism::Location) }
-  def location; end
-end
-
-class Prism::ParseWarning
-  sig { returns(String) }
-  def message; end
-
-  sig { returns(Prism::Location) }
-  def location; end
-end
-
-class Prism::Node
-  sig { returns(Prism::Location) }
-  def location; end
-
-  sig { returns(String) }
-  def slice; end
-
-  sig { returns(String) }
-  def to_dot; end
-
-  sig { params(visitor: Prism::Visitor).void }
-  def accept(visitor); end
-
-  sig { returns(T::Array[T.nilable(Prism::Node)]) }
-  def child_nodes; end
-
-  sig { returns(T::Array[Prism::Node]) }
-  def compact_child_nodes; end
-
-  sig { returns(T::Array[T.nilable(Prism::Node)]) }
-  def deconstruct; end
-
-  sig { returns(Symbol) }
-  def type; end
-end
-
-class Prism::Comment
-  sig { returns(Prism::Location) }
-  def location; end
-
-  sig { returns(T::Boolean) }
-  def trailing?; end
-end
-
-class Prism::InlineComment < Prism::Comment
-  sig { override.returns(T::Boolean) }
-  def trailing?; end
-end
-
-class Prism::EmbDocComment < Prism::Comment
-end
-
-class Prism::DATAComment < Prism::Comment
-end
-
-class Prism::Location
-  sig { params(source: Prism::Source, start_offset: Integer, length: Integer).void }
-  def initialize(source, start_offset, length); end
-
-  sig { returns(String) }
-  def slice; end
-
-  sig { returns(T::Array[Prism::Comment]) }
-  def comments; end
-
-  sig { params(options: T.untyped).returns(Prism::Location) }
-  def copy(**options); end
-
-  sig { returns(Integer) }
-  def start_offset; end
-
-  sig { returns(Integer) }
-  def end_offset; end
-
-  sig { returns(Integer) }
-  def start_line; end
-
-  sig { returns(Integer) }
-  def end_line; end
-
-  sig { returns(Integer) }
-  def start_column; end
-
-  sig { returns(Integer) }
-  def end_column; end
-end
-
-class Prism::Source
-  sig { params(source: String, start_line: Integer, offsets: T::Array[Integer]).void }
-  def initialize(source, start_line, offsets); end
-
-  sig { params(offset: Integer, length: Integer).returns(String) }
-  def slice(offset, length); end
-
-  sig { params(value: Integer).returns(Integer) }
-  def line(value); end
-
-  sig { params(value: Integer).returns(Integer) }
-  def line_offset(value); end
-
-  sig { params(value: Integer).returns(Integer) }
-  def column(value); end
-
-  sig { returns(String) }
-  def source; end
-
-  sig { returns(T::Array[Integer]) }
-  def offsets; end
-end
-
-class Prism::Token
-  sig { params(type: T.untyped, value: String, location: Prism::Location).void }
-  def initialize(type, value, location); end
-
-  sig { params(keys: T.untyped).returns(T.untyped) }
-  def deconstruct_keys(keys); end
-
-  sig { params(q: T.untyped).returns(T.untyped) }
-  def pretty_print(q); end
-
-  sig { params(other: T.untyped).returns(T::Boolean) }
-  def ==(other); end
-
-  sig { returns(T.untyped) }
-  def type; end
-
-  sig { returns(String) }
-  def value; end
-
-  sig { returns(Prism::Location) }
-  def location; end
-end
-
-class Prism::NodeInspector
-  sig { params(prefix: String).void }
-  def initialize(prefix); end
-
-  sig { returns(String) }
-  def prefix; end
-
-  sig { returns(String) }
-  def output; end
-
-  # Appends a line to the output with the current prefix.
-  sig { params(line: String).void }
-  def <<(line); end
-
-  # This generates a string that is used as the header of the inspect output
-  # for any given node.
-  sig { params(node: Prism::Node).returns(String) }
-  def header(node); end
-
-  # Generates a string that represents a list of nodes. It handles properly
-  # using the box drawing characters to make the output look nice.
-  sig { params(prefix: String, nodes: T::Array[Prism::Node]).returns(String) }
-  def list(prefix, nodes); end
-
-  # Generates a string that represents a location field on a node.
-  sig { params(value: Prism::Location).returns(String) }
-  def location(value); end
-
-  # Generates a string that represents a child node.
-  sig { params(node: Prism::Node, append: String).returns(String) }
-  def child_node(node, append); end
-
-  # Returns a new inspector that can be used to inspect a child node.
-  sig { params(append: String).returns(Prism::NodeInspector) }
-  def child_inspector(append); end
-
-  # Returns the output as a string.
-  sig { returns(String) }
-  def to_str; end
-end
-
-class Prism::BasicVisitor
-  sig { params(node: T.nilable(Prism::Node)).void }
-  def visit(node); end
-
-  sig { params(nodes: T::Array[T.nilable(Prism::Node)]).void }
-  def visit_all(nodes); end
-
-  sig { params(node: Prism::Node).void }
-  def visit_child_nodes(node); end
-end