repper 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0889e10cd309dbdd42315e1049c2f7e10a9ab46b949b08c390614965aa2c1483'
-  data.tar.gz: 8ef2f61c38c6e23abcd933c1e3e462602c19d153824c7632114673b57b291ad5
+  metadata.gz: 6493f978abbb5042a52adf386b6222ed571ba0de09b0926e0fb16104a3b60ad4
+  data.tar.gz: 52604ae6ebccc88a5608de1b57c7e8e0de85ba55aaf1704a2e066c6ae4fe93b5
 SHA512:
-  metadata.gz: a78030f011bb7d74c0263b2e6353114bdd94c9bdd5ee05f36ec7712dd5ad719f23050cea5627fb51cfc8ebe8bdf8b7813ec2094ea163c0253c58996e1fa5997c
-  data.tar.gz: 28b9f8a9243497e1be1ae2243c81817e2dc566532b314b03b9aa3410a947a9eb50f16b79b9b33a8e76e189c63efbc98b173acf160f814ca9bff2e91deb1d839f
+  metadata.gz: 0306346ad34c63bec6558c6ef9516e675e6a8c6b5caa59d92c3ded9fee74687c74cb69e293df474552e8c9204f0c6dfb78d9fd35d03a7b4c9aa8b86edfd2c28c
+  data.tar.gz: 2c3ebb13aa9795f4992057da6cb318a2bc16b571ad15c52a4117f27c88e4803526a15a06dd0ce0d62eb1a5cd4133dc6a7ebee6ac03e2cd0e4f611571681b8309

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 ## [Unreleased]
 
-## [1.0.0] - 2022-04-30
+...
+
+## [1.1.0] - 2022-05-29
+
+### Added
+
+- codemod capability and corresponding `exe/repper` executable
+- new format `Repper::Format::Extended`
+- `Regexp#original_inspect` when using `Regexp#inspect` override
+
+### Fixed
+
+- use the more appropriate `:inline` format when passing `format: nil | false`
+
+## [1.0.0] - 2022-05-09
 
 - Initial release

data/README.md

@@ -1,9 +1,8 @@
-<h1>
-  Repper
-  <img alt='Rapper necklace with dollar pendant' src='https://user-images.githubusercontent.com/10758879/167485870-5c49284d-a783-453e-8be0-a3597c2ef97c.png' height='46' align='right' />
-</h1>
+<img alt='Rapper necklace with dollar pendant' src='https://user-images.githubusercontent.com/10758879/167485870-5c49284d-a783-453e-8be0-a3597c2ef97c.png' height='64' align='right' />
 
-Repper is a regular expression pretty printer for Ruby.
+# Repper
+
+Repper is a regular expression pretty printer and formatter for Ruby.
 
 ## Installation
 
@@ -11,20 +10,28 @@ Repper is a regular expression pretty printer for Ruby.
 
 ## Usage
 
-`repper` can be integrated into the REPL (e.g. IRB) through core extensions or used manually.
+`repper` can be integrated into the REPL (e.g. IRB) through core extensions for Regexp pretty-printing, integrated into editors to format Regexps, or called manually.
 
 There are also a few customization options.
 
-### Full REPL integration (recommended)
+### REPL integration
+
+#### Via Regexp#inspect (recommended)
 
 `require 'repper/core_ext/regexp'` in your `~/.irbrc` or `~/.pryrc` to override `Regexp#inspect` and automatically use `repper` to display Regexps:
 
-<img width="313" alt="screenshot1" src="https://user-images.githubusercontent.com/10758879/167497359-e5bb94db-1382-465b-903a-3e114721b7a6.png">
+<img width="475" alt="screenshot1" src="https://user-images.githubusercontent.com/10758879/167719748-60f4013a-c8d4-4a62-843a-d9f27057bcd3.png">
 
-### Extending Kernel#pp
+#### Via Kernel#pp
 
 Alternatively, `require 'repper/core_ext/kernel'` to make the `pp` command give nicer output for Regexps (which will look like above by default).
 
+### Editor integration
+
+Use [vscode-repper](https://github.com/jaynetics/vscode-repper) to format Regexps in VSCode.
+
+![vscode-repper](https://user-images.githubusercontent.com/10758879/170892739-e2f408f2-e239-4b13-8d28-c14fb7a9dbb9.gif)
+
 ### Using Repper manually
 
 ```ruby
@@ -36,11 +43,13 @@ Repper.render(/foo/) # returns the pretty print String
 
 #### Customizing the format
 
-The default format is the annotated, indented format shown above.
-
-If you want to see the indented structure without annotations, use the `:structured` format.
+Multiple formats are available out of the box:
 
-If you only want colorization you can use the `:inline` format.
+- `:annotated` is the default, verbose format, shown above
+- `:inline` adds only colorization and does not restructure the Regexp
+- `:structured` is like `:annotated`, just without annotations
+- `:x` (or `:extended`) returns a lightly formatted but equivalent Regexp
+  - this format is used for the repper executable and [vscode-repper](https://github.com/jaynetics/vscode-repper)
 
 You can change the format globally:
 
@@ -50,14 +59,14 @@ Repper.format = :structured
 
 Or pick a format on a case-by-case basis:
 
-<img width="445" alt="screenshot2" src="https://user-images.githubusercontent.com/10758879/167497599-105f39c7-91e0-4954-bce3-d04ad7266695.png">
+<img width="711" alt="screenshot2" src="https://user-images.githubusercontent.com/10758879/167719567-ae8ee42f-839e-4ce4-af56-a139044d3436.png">
 
 Or create your own format:
 
 ```ruby
 require 'csv'
 
-csv_format = ->(elements, _theme) { elements.map(&:text).to_csv }
+csv_format = ->(tokens, _theme) { tokens.map(&:text).to_csv }
 Repper.render(/re[\p{pe}\r]$/, format: csv_format)
 => "/,re,[,\\p{pe},\\r,],$,/\n"
 ```
@@ -69,7 +78,8 @@ The color theme can also be set globally or passed on call:
 ```ruby
 Repper.theme = :monokai # a nicer theme, if the terminal supports it
 ```
-<img width="316" alt="screenshot3" src="https://user-images.githubusercontent.com/10758879/167497895-0cdc017f-5c77-4b15-afaa-207f7eb887cc.png">
+
+<img width="478" alt="screenshot3" src="https://user-images.githubusercontent.com/10758879/167719807-9170ba92-48d1-4669-a05d-a72f962b961d.png">
 
 ```ruby
 Repper.call(/foo/, theme: nil) # render without colors

data/exe/repper

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require_relative "../lib/repper"
+
+Repper::Command.call(ARGV)

data/lib/repper/codemod.rb

@@ -0,0 +1,68 @@
+module Repper
+  # Formatter for Ruby code containing Regexp literals
+  module Codemod
+    module_function
+
+    def call(code)
+      formatted_code = code.dup
+
+      regexp_locations(code).reverse.each do |loc|
+        beg_idx = code[/\A(.*\n){#{loc.beg_line}}/].size + loc.beg_char
+        end_idx = code[/\A(.*\n){#{loc.end_line}}/].size + loc.end_char
+        range = beg_idx..end_idx
+
+        /\A(?<start>\/|%r.)(?<source>.*)(?<stop>[^a-z])(?<flags>[a-z]*)\z/m =~
+          code[range]
+
+        tokens = Tokenizer.call(source, delimiters: [start, stop], flags: flags)
+        formatted_regexp = Format::Extended.call(tokens, Theme::Plain)
+
+        # indent consistently by applying leading line indentation to all lines
+        lead_indentation = code.lines[loc.beg_line][/^ */]
+        formatted_regexp.gsub!("\n", "\n#{lead_indentation}")
+
+        formatted_code[range] = formatted_regexp
+      end
+
+      formatted_code
+    end
+
+    require 'ripper'
+
+    def regexp_locations(code)
+      embed_level = 0
+      beg_tokens = {}
+
+      Ripper.lex(code).each.with_object([]) do |token, acc|
+        case token[1]
+        when :on_regexp_beg
+          beg_tokens[embed_level] = token
+          # nested regexp literals are not supported a.t.m., so if we're
+          # in an embed, discard the location of the surrounding regexp.
+          beg_tokens[embed_level - 1] = nil
+        when :on_regexp_end
+          next unless beg_token = beg_tokens[embed_level]
+
+          acc << Location.new(
+            beg_line: beg_token[0][0] - 1, # note: using 0-indexed line values
+            beg_char: beg_token[0][1],
+            end_line: token[0][0] - 1,
+            end_char: token[0][1] + (token[2].length - 1), # lex includes flags
+          )
+        when :on_embexpr_beg # embedded expression a.k.a. interpolation (#{...})
+          embed_level += 1
+        when :on_embexpr_end
+          embed_level -= 1
+        when :on_const, :on_ident
+          # OK when embedded - Regexp::Parser will treat them as literals
+        else
+          # other embedded expressions are not supported
+          beg_tokens[embed_level - 1] = nil
+        end
+      end
+    end
+
+    Location = Struct.new(:beg_line, :beg_char, :end_line, :end_char,
+                          keyword_init: true)
+  end
+end

data/lib/repper/command.rb

@@ -0,0 +1,39 @@
+module Repper
+  # Service object for exe/repper
+  module Command
+    module_function
+
+    PARSE_ERROR_EXIT_STATUS = 1
+
+    def call(argv)
+      if argv.count == 0
+        format_stdio
+      else
+        format_files(argv)
+      end
+    end
+
+    def format_stdio
+      input = STDIN.read
+      output = format(input)
+      print output
+    end
+
+    def format(string)
+      Codemod.call(string)
+    rescue Repper::Error => e
+      warn "Parsing failed: #{e.class} - #{e.message}"
+      exit PARSE_ERROR_EXIT_STATUS
+    end
+
+    def format_files(paths)
+      paths.grep(/\.rb\z/).each { |path| format_file(path) }
+    end
+
+    def format_file(path)
+      code = File.read(path)
+      formatted_code = format(code)
+      File.write(path, formatted_code)
+    end
+  end
+end

data/lib/repper/core_ext/regexp.rb

@@ -1,4 +1,6 @@
 require 'repper'
 require_relative 'regexp_ext'
 
+Regexp.alias_method :original_inspect, :inspect
+
 ::Regexp.prepend(Repper::RegexpExt)

data/lib/repper/format/annotated.rb

@@ -1,7 +1,8 @@
 module Repper
   module Format
-    Annotated = ->(elements, theme) do
-      table = Tabulo::Table.new(elements.reject(&:whitespace?), **TABULO_STYLE)
+    # A structured and colorized format with annotations about token types.
+    Annotated = ->(tokens, theme) do
+      table = Tabulo::Table.new(tokens.reject(&:whitespace?), **TABULO_STYLE)
       table.add_column(
         :indented_text,
         styler: ->(_, string, cell) { theme.colorize(string, cell.source.type) }

data/lib/repper/format/extended.rb

@@ -0,0 +1,41 @@
+module Repper
+  module Format
+    # A lightly structured format that retains parsability
+    # and functional equivalence, for use in code.
+    Extended = ->(tokens, theme) do
+      run_types = %i[escape literal nonposixclass nonproperty
+                     posixclass property set type]
+      forms_run = ->(el){ run_types.include?(el.type) || el.subtype == :dot }
+      prev = nil
+
+      tokens.each.with_object(''.dup) do |tok, acc|
+        # drop existing x-mode whitespace, if any
+        if tok.whitespace?
+          next
+        # keep some tokens in line:
+        # - option switches and conditions for syntactic correctness
+        # - quantifiers and codepoint runs for conciseness
+        elsif tok.type == :quantifier ||
+              tok.subtype == :options_switch && tok.text == ')' ||
+              tok.subtype == :condition ||
+              prev && forms_run.call(prev) && forms_run.call(tok)
+          acc << theme.colorize(tok.inlined_text, tok.type)
+        # keep comments in line, too, but with padding
+        elsif tok.comment?
+          acc << " #{theme.colorize(tok.inlined_text, tok.type)}"
+        # render root start as wtokl as empty root end in same line
+        elsif tok.subtype == :root && (prev.nil? || prev.subtype == :root)
+          acc << theme.colorize(tok.text, tok.type)
+        # tokse, if root is not empty, ensure x-flag is present at end
+        elsif tok.subtype == :root
+          acc << "\n#{theme.colorize(tok.text.sub(/x?\z/, 'x'), tok.type)}"
+        # render other tokens on their own lines for an indented structure,
+        # e.g. groups, alternations, anchors, assertions, ...
+        else
+          acc << "\n#{theme.colorize(tok.indented_text, tok.type)}"
+        end
+        prev = tok
+      end
+    end
+  end
+end

data/lib/repper/format/inline.rb

@@ -1,7 +1,8 @@
 module Repper
   module Format
-    Inline = ->(elements, theme) do
-      elements.map { |el| theme.colorize(el.text, el.type) }.join
+    # A format that only adds color but does not change structure.
+    Inline = ->(tokens, theme) do
+      tokens.map { |tok| theme.colorize(tok.text, tok.type) }.join
     end
   end
 end

data/lib/repper/format/plain.rb

@@ -1,7 +1,8 @@
 module Repper
   module Format
-    Plain = ->(elements, *) do
-      elements.map(&:text).join
+    # A no-op format, equivalent to the original Regexp#inspect.
+    Plain = ->(tokens, *) do
+      tokens.map(&:text).join
     end
   end
 end

data/lib/repper/format/structured.rb

@@ -1,7 +1,8 @@
 module Repper
   module Format
-    Structured = ->(elements, theme) do
-      table = Tabulo::Table.new(elements.reject(&:whitespace?), **TABULO_STYLE)
+    # A structured format with colorization.
+    Structured = ->(tokens, theme) do
+      table = Tabulo::Table.new(tokens.reject(&:whitespace?), **TABULO_STYLE)
       table.add_column(
         :indented_text,
         styler: ->(_, string, cell) { theme.colorize(string, cell.source.type) }

data/lib/repper/format/tabulo_style.rb

@@ -4,7 +4,7 @@ module Repper
   module Format
     TABULO_STYLE = {
       border:               :blank,
-      header_frequency:     nil,
+      header_frequency:     nil, # i.e. omit column headers
       truncation_indicator: '…',
       wrap_body_cells_to:   1,
     }

data/lib/repper/format.rb

@@ -3,8 +3,9 @@ module Repper
     def self.cast(arg)
       case arg
       when ::Proc             then arg
+      when :x                 then Format::Extended
       when ::Symbol, ::String then Format.const_get(arg.capitalize) rescue nil
-      when false, nil         then Format::Plain
+      when false, nil         then Format::Inline
       end || raise(Repper::ArgumentError, "unknown format #{arg.inspect}")
     end
   end

data/lib/repper/{element.rb → token.rb}

@@ -1,14 +1,27 @@
 module Repper
-  Element = Struct.new(:type, :subtype, :level, :text, :id, keyword_init: true) do
+  Token = Struct.new(:type, :subtype, :level, :text, :id, keyword_init: true) do
     def indented_text
-      inlined_text = text.gsub(/[\n\r\t\v]/) { |ws| ws.inspect.delete(?") }
       "#{'  ' * level}#{inlined_text}"
     end
 
+    def inlined_text
+      if comment?
+        text.strip
+      else
+        text
+          .gsub(/(?<!\\) /, '\\ ')
+          .gsub(/[\n\r\t\v]/) { |ws| ws.inspect.delete(?") }
+      end
+    end
+
     def whitespace?
       subtype == :whitespace
     end
 
+    def comment?
+      subtype == :comment
+    end
+
     def annotation
       case [type, subtype]
       in [_, :root]

data/lib/repper/{parser.rb → tokenizer.rb}

@@ -1,28 +1,28 @@
 require 'regexp_parser'
 
 module Repper
-  module Parser
+  module Tokenizer
     module_function
 
-    def call(regexp)
-      tree = ::Regexp::Parser.parse(regexp)
-      flatten(tree)
+    def call(regexp, delimiters: ['/', '/'], flags: nil)
+      tree = Regexp::Parser.parse(regexp, options: flags =~ /x/ && Regexp::EXTENDED)
+      flatten(tree, delimiters: delimiters, flags: flags)
     rescue ::Regexp::Parser::Error => e
       raise e.extend(Repper::Error)
     end
 
     # Turn Regexp::Parser AST back into a flat Array of visual elements
     # that match the Regexp notation.
-    def flatten(exp, acc = [])
+    def flatten(exp, acc = [], delimiters: nil, flags: nil)
       # Add opening entry.
-      exp.is?(:root) && acc << make_element(exp, '/')
+      exp.is?(:root) && acc << make_token(exp, delimiters[0])
 
       # Ignore nesting of invisible intermediate branches for better visuals.
       exp.is?(:sequence) && exp.nesting_level -= 1
 
       exp.parts.each do |part|
         if part.instance_of?(::String)
-          acc << make_element(exp, part)
+          acc << make_token(exp, part)
         else # part.is_a?(Regexp::Expression::Base)
           flatten(part, acc)
         end
@@ -31,13 +31,16 @@ module Repper
       exp.quantified? && flatten(exp.quantifier, acc)
 
       # Add closing entry.
-      exp.is?(:root) && acc << make_element(exp, "/#{exp.options.keys.join}")
+      exp.is?(:root) && begin
+        flags ||= exp.options.keys.join
+        acc << make_token(exp, "#{delimiters[1]}#{flags.chars.uniq.sort.join}")
+      end
 
       acc
     end
 
-    def make_element(exp, text)
-      Element.new(
+    def make_token(exp, text)
+      Token.new(
         type:    exp.type,
         subtype: exp.token,
         level:   exp.nesting_level,

data/lib/repper/version.rb

@@ -1,3 +1,3 @@
 module Repper
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/lib/repper.rb

@@ -1,7 +1,9 @@
-require_relative "repper/element"
+require_relative "repper/codemod"
+require_relative "repper/command"
 require_relative "repper/errors"
 require_relative "repper/format"
-require_relative "repper/parser"
+require_relative "repper/token"
+require_relative "repper/tokenizer"
 require_relative "repper/theme"
 require_relative "repper/version"
 
@@ -14,18 +16,18 @@ module Repper
     end
 
     def render(regexp, format: self.format, theme: self.theme)
-      elements = Parser.call(regexp)
-      format   = Format.cast(format)
-      theme    = Theme.cast(theme)
-      format.call(elements, theme)
+      tokens = Tokenizer.call(regexp)
+      format = Format.cast(format)
+      theme  = Theme.cast(theme)
+      format.call(tokens, theme)
     end
 
-    def theme=(theme)
-      @theme = Theme.cast(theme)
+    def format=(arg)
+      @format = Format.cast(arg)
     end
 
-    def format=(format)
-      @format = Format.cast(format)
+    def theme=(arg)
+      @theme = Theme.cast(arg)
     end
   end
 

data/repper.gemspec

@@ -6,7 +6,7 @@ Gem::Specification.new do |spec|
   spec.authors = ["Janosch Müller"]
   spec.email = ["janosch84@gmail.com"]
 
-  spec.summary = "Regexp pretty printer for Ruby"
+  spec.summary = "Regexp pretty printer and formatter for Ruby"
   spec.homepage = "https://github.com/jaynetics/repper"
   spec.license = "MIT"
   spec.required_ruby_version = ">= 2.7.0"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: repper
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Janosch Müller
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-05-09 00:00:00.000000000 Z
+date: 2022-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rainbow
@@ -55,7 +55,8 @@ dependencies:
 description:
 email:
 - janosch84@gmail.com
-executables: []
+executables:
+- repper
 extensions: []
 extra_rdoc_files: []
 files:
@@ -65,24 +66,28 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- exe/repper
 - lib/repper.rb
+- lib/repper/codemod.rb
+- lib/repper/command.rb
 - lib/repper/core_ext/kernel.rb
 - lib/repper/core_ext/kernel_ext.rb
 - lib/repper/core_ext/regexp.rb
 - lib/repper/core_ext/regexp_ext.rb
-- lib/repper/element.rb
 - lib/repper/errors.rb
 - lib/repper/format.rb
 - lib/repper/format/annotated.rb
+- lib/repper/format/extended.rb
 - lib/repper/format/inline.rb
 - lib/repper/format/plain.rb
 - lib/repper/format/structured.rb
 - lib/repper/format/tabulo_style.rb
-- lib/repper/parser.rb
 - lib/repper/theme.rb
 - lib/repper/theme/default.rb
 - lib/repper/theme/monokai.rb
 - lib/repper/theme/plain.rb
+- lib/repper/token.rb
+- lib/repper/tokenizer.rb
 - lib/repper/version.rb
 - repper.gemspec
 - repper.svg
@@ -112,5 +117,5 @@ requirements: []
 rubygems_version: 3.4.0.dev
 signing_key:
 specification_version: 4
-summary: Regexp pretty printer for Ruby
+summary: Regexp pretty printer and formatter for Ruby
 test_files: []