syntax_tree 0.1.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e3977b8a9642c86ca56b63f93e541a7b9bdee08eef9d705f0540e4c1d6e95222
-  data.tar.gz: b550f2a31561c3a06ca456005419a0743f5d62e60360797e70fa42cc075a603e
+  metadata.gz: 4797ffd100da9b050f154cd0ac4a235b342c395e49747ee51574eb0e0f9fcf1a
+  data.tar.gz: 52babf707ae1bf42abaeb51b350644fc50f61fa1b74482abb776e863d2ec48dd
 SHA512:
-  metadata.gz: 1cab71763cb9cb537fb209fb175bc36074e58da9c7535d788911bb4d0e36a8ce28628eeae283d468d957c6c38549b33245dd7e6e07c749c725609fb53bd8ef5c
-  data.tar.gz: b49ee91795fa64acd80d8cb5c244ff30fbbacc711da907c14f81f4f9cb7632c71cc3a6ae9d609224c4adcbdc54bfa89bb5f1b47ec17601653cdf059b8389feae
+  metadata.gz: 62e28b7e6b25e81bdfb70817c02152bd640395149c1ac75fa553154ebb636c9c913dbf2ebda7a65f9687dd1bb69d51e0e3d8f5ffba172d0b1eadb5849e64369a
+  data.tar.gz: 6233b01960317929ce137305a08f45d3d756d741b5f6b6d037f2499377a53922f222543f433912cc853ee31726e2306527fbe29c75ad757e6125207b7476956c

data/.github/workflows/main.yml

@@ -13,7 +13,7 @@ jobs:
     - uses: ruby/setup-ruby@v1
       with:
         bundler-cache: true
-        ruby-version: 3.0
+        ruby-version: '3.1'
     - name: Test
       run: bundle exec rake test
   automerge:

data/CHANGELOG.md

@@ -6,11 +6,108 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 ## [Unreleased]
 
+## [1.2.0] - 2022-01-09
+
+### Added
+
+- Support for Ruby 3.1 syntax, including: blocks without names, hash keys without values, endless methods without parentheses, and new argument forwarding.
+- Support for pinned expressions and variables within pattern matching.
+- Support endless ranges as the final argument to a `when` clause.
+
+## [1.1.1] - 2021-12-09
+
+### Added
+
+- [#7](https://github.com/kddnewton/syntax_tree/issues/7) Better formatting for hashes and arrays that are values in hashes.
+- [#9](https://github.com/kddnewton/syntax_tree/issues/9) Special handling for RSpec matchers when nesting `CommandCall` nodes.
+- [#10](https://github.com/kddnewton/syntax_tree/issues/10) Force the maintaining of the modifier forms of conditionals and loops if the statement includes an assignment. Also, for the maintaining of the block form of conditionals and loops if the predicate includes an assignment.
+
+## [1.1.0] - 2021-12-08
+
+### Added
+
+- Better handling for formatting files with errors.
+- Colorize the output snippet using IRB.
+
+## [1.0.0] - 2021-12-08
+
+### Added
+
+- The ability to "check" formatting by formatting the output of the first format.
+- Comments can now be attached to the `case` keyword.
+- Remove escaped forward slashes from regular expression literals when converting to `%r`.
+- Allow arrays of `CHAR` nodes to be converted to `QWords` under certain conditions.
+- Allow `HashLiteral` opening braces to have trailing comments.
+- Add parentheses if `Yield` breaks onto multiple lines.
+- Ensure all nodes that could have heredocs nested know about their end lines.
+- Ensure comments on assignment after the `=` before the value keep their place.
+- Trailing comments on parameters with no parentheses now do not force a break.
+- Allow `ArrayLiteral` opening brackets to have trailing comments.
+- Allow different line suffix nodes to have different priorities.
+- Better support for encoding by properly reading encoding magic comments.
+- Support singleton single-line method definitions.
+- Support `stree-ignore` comments to ignore formatting nodes.
+- Add special formatting for arrays of `VarRef` nodes whose sum width is greater than 2 * the maximum width.
+- Better output formatting for the CLI.
+
+### Changed
+
+- Force a break if a block is attached to a `Command` or `CommandCall` node.
+- Don't indent `CommandCall` arguments if they don't fit aligned.
+- Force a break in `Call` nodes if there are comments on the receiver.
+- Do not change block bounds if inside of a `Command` or `CommandCall` node.
+- Handle empty parentheses inside method calls.
+- Skip indentation for special array literals on assignment nodes.
+- Ensure a final breakable is inserted when converting an `ArrayLiteral` to a `QSymbols`.
+- Fix up the `doc_width` calculation for `CommandCall` nodes.
+- Ensure parameters inside a lambda literal when there are no parentheses are grouped.
+- Ensure when converting an `ArrayLiteral` to a `QWords` that the strings do not contain `[`.
+- Stop looking for parent `Command` or `CommandCall` nodes in blocks once you hit `Statements`.
+- Ensure nested `Lambda` nodes get their correct bounds.
+- Ensure we do not change block bounds within control flow constructs.
+- Ensure parentheses are added around keywords changing to their modifier forms.
+- Allow conditionals to take modifier form if they are using the `then` keyword with a `VoidStmt`.
+- `UntilMod` and `WhileMod` nodes that wrap a `Begin` should be forced into their modifier forms.
+- Ensure `For` loops keep their trailing commas.
+- Replicate content for `__END__` keyword exactly.
+- Keep block `If`, `Unless`, `While`, and `Until` forms if there is an assignment in the predicate.
+- Force using braces if the block is within the predicate of a conditional or loop.
+- Allow for the possibility that `CommandCall` nodes might not have arguments.
+- Explicitly handle `?"` so that it formats properly.
+- Check that a block is within the predicate in a more relaxed way.
+- Ensure the `Return` breaks with brackets and not parentheses.
+- Ensure trailing comments on parameter declarations are consistent.
+- Make `Command` and `CommandCall` aware that their arguments could exceed their normal expected bounds because of heredocs.
+- Only unescape forward slashes in regular expressions if converting from slash bounds to `%r` bounds.
+- Allow `When` nodes to grab trailing comments away from their statements lists.
+- Allow flip-flop operators to be formatted correctly within `IfMod` and `UnlessMod` nodes.
+- Allow `IfMod` and `UnlessMod` to know about heredocs moving their bounds.
+- Properly handle breaking parameters when there are no parentheses.
+- Properly handle trailing operators in call chains with attached comments.
+- Force using braces if the block is within the predicate of a ternary.
+- Properly handle trailing comments after a `then` operator on a `When` or `In` clause.
+- Ensure nested `HshPtn` nodes use braces.
+- Force using braces if the block is within a `Binary` within the predicate of a loop or conditional.
+- Make sure `StringLiteral` and `StringEmbExpr` know that they can be extended by heredocs.
+- Ensure `Int` nodes with preceding unary `+` get formatted properly.
+- Properly handle byte-order mark column offsets at the beginnings of files.
+- Ensure `Words`, `Symbols`, `QWords`, and `QSymbols` properly format when their contents contain brackets.
+- Ensure ternaries being broken out into `if`...`else`...`end` get wrapped in parentheses if necessary.
+
+### Removed
+
+- The `AccessCtrl` node in favor of just formatting correctly when you hit a `Statements` node.
+- The `MethodAddArg` node is removed in favor of an optional `arguments` field on `Call` and `FCall`.
+
 ## [0.1.0] - 2021-11-16
 
 ### Added
 
 - 🎉 Initial release! 🎉
 
-[unreleased]: https://github.com/kddnewton/syntax_tree/compare/v0.1.0...HEAD
+[unreleased]: https://github.com/kddnewton/syntax_tree/compare/v1.2.0...HEAD
+[1.2.0]: https://github.com/kddnewton/syntax_tree/compare/v1.1.1...v1.2.0
+[1.1.1]: https://github.com/kddnewton/syntax_tree/compare/v1.1.0...v1.1.1
+[1.1.0]: https://github.com/kddnewton/syntax_tree/compare/v1.0.0...v1.1.0
+[1.0.0]: https://github.com/kddnewton/syntax_tree/compare/v0.1.0...v1.0.0
 [0.1.0]: https://github.com/kddnewton/syntax_tree/compare/8aa1f5...v0.1.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    syntax_tree (0.1.0)
+    syntax_tree (1.2.0)
 
 GEM
   remote: https://rubygems.org/
@@ -9,8 +9,8 @@ GEM
     ast (2.4.2)
     benchmark-ips (2.9.2)
     docile (1.4.0)
-    minitest (5.14.4)
-    parser (3.0.3.1)
+    minitest (5.15.0)
+    parser (3.1.0.0)
       ast (~> 2.4.1)
     rake (13.0.6)
     ruby_parser (3.18.1)
@@ -26,6 +26,7 @@ GEM
 
 PLATFORMS
   x86_64-darwin-19
+  x86_64-darwin-21
   x86_64-linux
 
 DEPENDENCIES
@@ -40,4 +41,4 @@ DEPENDENCIES
   syntax_tree!
 
 BUNDLED WITH
-   2.2.15
+   2.2.31

data/README.md

@@ -3,7 +3,7 @@
 [![Build Status](https://github.com/kddnewton/syntax_tree/workflows/Main/badge.svg)](https://github.com/kddnewton/syntax_tree/actions)
 [![Gem Version](https://img.shields.io/gem/v/syntax_tree.svg)](https://rubygems.org/gems/syntax_tree)
 
-A fast ripper subclass used for parsing and formatting Ruby code.
+A fast Ruby parser and formatter with only standard library dependencies.
 
 ## Installation
 
@@ -49,6 +49,13 @@ class MyClass
   ...
 ```
 
+or
+
+```sh
+$ stree write program.rb
+program.rb 1ms
+```
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/exe/stree

@@ -1,84 +1,8 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require_relative File.expand_path("../lib/syntax_tree", __dir__)
+$:.unshift(File.expand_path("../lib", __dir__))
+require "syntax_tree"
+require "syntax_tree/cli"
 
-help = <<~EOF
-  stree MDOE FILE
-
-  MODE: one of "a", "ast", "d", "doc", "f", "format", "w", or "write"
-  FILE: one or more paths to files to parse
-EOF
-
-if ARGV.length < 2
-  warn(help)
-  exit(1)
-end
-
-module SyntaxTree::CLI
-  class AST
-    def run(filepath)
-      pp SyntaxTree.parse(File.read(filepath))
-    end
-  end
-
-  class Doc
-    def run(filepath)
-      formatter = SyntaxTree::Formatter.new([])
-      SyntaxTree.parse(File.read(filepath)).format(formatter)
-      pp formatter.groups.first
-    end
-  end
-
-  class Format
-    def run(filepath)
-      puts SyntaxTree.format(File.read(filepath))
-    end
-  end
-
-  class Write
-    def run(filepath)
-      File.write(filepath, SyntaxTree.format(File.read(filepath)))
-    end
-  end
-end
-
-mode =
-  case ARGV.shift
-  when "a", "ast"
-    SyntaxTree::CLI::AST.new
-  when "d", "doc"
-    SyntaxTree::CLI::Doc.new
-  when "f", "format"
-    SyntaxTree::CLI::Format.new
-  when "w", "write"
-    SyntaxTree::CLI::Write.new
-  else
-    warn(help)
-    exit(1)
-  end
-
-queue = Queue.new
-ARGV.each { |pattern| Dir[pattern].each { |filepath| queue << filepath } }
-
-if queue.size <= 1
-  filepath = queue.shift
-  mode.run(filepath) if File.file?(filepath)
-  return
-end
-
-count = [8, queue.size].min
-threads =
-  count.times.map do
-    Thread.new do
-      loop do
-        filepath = queue.shift
-        break if filepath == :exit
-
-        mode.run(filepath) if File.file?(filepath)
-      end
-    end
-  end
-
-count.times { queue << :exit }
-threads.each(&:join)
+exit(SyntaxTree::CLI.run(ARGV))

data/lib/syntax_tree/cli.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+class SyntaxTree
+  module CLI
+    # A utility wrapper around colored strings in the output.
+    class Color
+      attr_reader :value, :code
+
+      def initialize(value, code)
+        @value = value
+        @code = code
+      end
+
+      def to_s
+        "\033[#{code}m#{value}\033[0m"
+      end
+
+      def self.gray(value)
+        new(value, "38;5;102")
+      end
+
+      def self.red(value)
+        new(value, "1;31")
+      end
+
+      def self.yellow(value)
+        new(value, "33")
+      end
+    end
+
+    # The parent action class for the CLI that implements the basics.
+    class Action
+      def run(filepath, source)
+      end
+
+      def success
+      end
+
+      def failure
+      end
+    end
+
+    # An action of the CLI that prints out the AST for the given source.
+    class AST < Action
+      def run(filepath, source)
+        pp SyntaxTree.parse(source)
+      end
+    end
+
+    # An action of the CLI that ensures that the filepath is formatted as
+    # expected.
+    class Check < Action
+      class UnformattedError < StandardError
+      end
+
+      def run(filepath, source)
+        raise UnformattedError if source != SyntaxTree.format(source)
+      rescue StandardError
+        warn("[#{Color.yellow("warn")}] #{filepath}")
+        raise
+      end
+
+      def success
+        puts("All files matched expected format.")
+      end
+
+      def failure
+        warn("The listed files did not match the expected format.")
+      end
+    end
+
+    # An action of the CLI that formats the source twice to check if the first
+    # format is not idempotent.
+    class Debug < Action
+      class NonIdempotentFormatError < StandardError
+      end
+
+      def run(filepath, source)
+        warning = "[#{Color.yellow("warn")}] #{filepath}"
+        formatted = SyntaxTree.format(source)
+
+        if formatted != SyntaxTree.format(formatted)
+          raise NonIdempotentFormatError
+        end
+      rescue StandardError
+        warn(warning)
+        raise
+      end
+
+      def success
+        puts("All files can be formatted idempotently.")
+      end
+
+      def failure
+        warn("The listed files could not be formatted idempotently.")
+      end
+    end
+
+    # An action of the CLI that prints out the doc tree IR for the given source.
+    class Doc < Action
+      def run(filepath, source)
+        formatter = Formatter.new([])
+        SyntaxTree.parse(source).format(formatter)
+        pp formatter.groups.first
+      end
+    end
+
+    # An action of the CLI that formats the input source and prints it out.
+    class Format < Action
+      def run(filepath, source)
+        puts SyntaxTree.format(source)
+      end
+    end
+
+    # An action of the CLI that formats the input source and writes the
+    # formatted output back to the file.
+    class Write < Action
+      def run(filepath, source)
+        print filepath
+        start = Time.now
+
+        formatted = SyntaxTree.format(source)
+        File.write(filepath, formatted)
+
+        color = source == formatted ? Color.gray(filepath) : filepath
+        delta = ((Time.now - start) * 1000).round
+
+        puts "\r#{color} #{delta}ms"
+      rescue StandardError
+        puts "\r#{filepath}"
+        raise
+      end
+    end
+
+    # The help message displayed if the input arguments are not correctly
+    # ordered or formatted.
+    HELP = <<~HELP
+      stree MODE FILE
+
+      MODE: ast | check | debug | doc | format | write
+      FILE: one or more paths to files to parse
+    HELP
+
+    class << self
+      # Run the CLI over the given array of strings that make up the arguments
+      # passed to the invocation.
+      def run(argv)
+        if argv.length < 2
+          warn(HELP)
+          return 1
+        end
+
+        arg, *patterns = argv
+        action =
+          case arg
+          when "a", "ast"
+            AST.new
+          when "c", "check"
+            Check.new
+          when "debug"
+            Debug.new
+          when "doc"
+            Doc.new
+          when "f", "format"
+            Format.new
+          when "w", "write"
+            Write.new
+          else
+            warn(HELP)
+            return 1
+          end
+
+        errored = false
+        patterns.each do |pattern|
+          Dir.glob(pattern).each do |filepath|
+            next unless File.file?(filepath)
+            source = SyntaxTree.read(filepath)
+
+            begin
+              action.run(filepath, source)
+            rescue ParseError => error
+              warn("Error: #{error.message}")
+
+              if error.lineno
+                highlight_error(error, source)
+              else
+                warn(error.message)
+                warn(error.backtrace)
+              end
+
+              errored = true
+            rescue Check::UnformattedError, Debug::NonIdempotentFormatError
+              errored = true
+            rescue => error
+              warn(error.message)
+              warn(error.backtrace)
+              errored = true
+            end
+          end
+        end
+
+        if errored
+          action.failure
+          1
+        else
+          action.success
+          0
+        end
+      end
+
+      private
+
+      # Highlights a snippet from a source and parse error.
+      def highlight_error(error, source)
+        lines = source.lines
+
+        maximum = [error.lineno + 3, lines.length].min
+        digits = Math.log10(maximum).ceil
+
+        ([error.lineno - 3, 0].max...maximum).each do |line_index|
+          line_number = line_index + 1
+
+          if line_number == error.lineno
+            part1 = Color.red(">")
+            part2 = Color.gray("%#{digits}d |" % line_number)
+            warn("#{part1} #{part2} #{colorize_line(lines[line_index])}")
+
+            part3 = Color.gray("  %#{digits}s |" % " ")
+            warn("#{part3} #{" " * error.column}#{Color.red("^")}")
+          else
+            prefix = Color.gray("  %#{digits}d |" % line_number)
+            warn("#{prefix} #{colorize_line(lines[line_index])}")
+          end
+        end
+      end
+
+      # Take a line of Ruby source and colorize the output.
+      def colorize_line(line)
+        require "irb"
+        IRB::Color.colorize_code(line, complete: false, ignore_error: true)
+      end
+    end
+  end
+end

data/lib/syntax_tree/prettyprint.rb

@@ -113,8 +113,10 @@ class PrettyPrint
     def pretty_print(q)
       q.text("breakable")
 
-      attributes =
-        [("force=true" if force?), ("indent=false" unless indent?)].compact
+      attributes = [
+        ("force=true" if force?),
+        ("indent=false" unless indent?)
+      ].compact
 
       if attributes.any?
         q.text("(")
@@ -213,9 +215,12 @@ class PrettyPrint
   # constantly check where the line ends to avoid accidentally printing some
   # content after a line suffix node.
   class LineSuffix
-    attr_reader :contents
+    DEFAULT_PRIORITY = 1
 
-    def initialize(contents: [])
+    attr_reader :priority, :contents
+
+    def initialize(priority: DEFAULT_PRIORITY, contents: [])
+      @priority = priority
       @contents = contents
     end
 
@@ -741,10 +746,17 @@ class PrettyPrint
 
     # This is a separate command stack that includes the same kind of triplets
     # as the commands variable. It is used to keep track of things that should
-    # go at the end of printed lines once the other doc nodes are
-    # accounted for. Typically this is used to implement comments.
+    # go at the end of printed lines once the other doc nodes are accounted for.
+    # Typically this is used to implement comments.
     line_suffixes = []
 
+    # This is a special sort used to order the line suffixes by both the
+    # priority set on the line suffix and the index it was in the original
+    # array.
+    line_suffix_sort = ->(line_suffix) do
+      [-line_suffix.last, -line_suffixes.index(line_suffix)]
+    end
+
     # This is a linear stack instead of a mutually recursive call defined on
     # the individual doc nodes for efficiency.
     while commands.any?
@@ -783,7 +795,7 @@ class PrettyPrint
           commands << [indent, mode, doc.flat_contents] if doc.flat_contents
         end
       when LineSuffix
-        line_suffixes << [indent, mode, doc.contents]
+        line_suffixes << [indent, mode, doc.contents, doc.priority]
       when Breakable
         if mode == MODE_FLAT
           if doc.force?
@@ -804,7 +816,7 @@ class PrettyPrint
         # to flush them now, as we are about to add a newline.
         if line_suffixes.any?
           commands << [indent, mode, doc]
-          commands += line_suffixes.reverse
+          commands += line_suffixes.sort_by(&line_suffix_sort)
           line_suffixes = []
           next
         end
@@ -838,7 +850,7 @@ class PrettyPrint
       end
 
       if commands.empty? && line_suffixes.any?
-        commands += line_suffixes.reverse
+        commands += line_suffixes.sort_by(&line_suffix_sort)
         line_suffixes = []
       end
     end
@@ -1012,8 +1024,8 @@ class PrettyPrint
 
   # Inserts a LineSuffix node into the print tree. The contents of the node are
   # determined by the block.
-  def line_suffix
-    doc = LineSuffix.new
+  def line_suffix(priority: LineSuffix::DEFAULT_PRIORITY)
+    doc = LineSuffix.new(priority: priority)
     target << doc
 
     with_target(doc.contents) { yield }

data/lib/syntax_tree/version.rb

@@ -3,5 +3,5 @@
 require "ripper"
 
 class SyntaxTree < Ripper
-  VERSION = "0.1.0"
+  VERSION = "1.2.0"
 end