prism 1.8.1 → 2.0.0.rc1

data/lib/prism/translation/ripper/lexer.rb

@@ -6,10 +6,8 @@ require_relative "../ripper"
 module Prism
   module Translation
     class Ripper
-      class Lexer # :nodoc:
-        # :stopdoc:
-        class State
-
+      class Lexer < Ripper # :nodoc:
+        class State # :nodoc:
           attr_reader :to_int, :to_s
 
           def initialize(i)
@@ -38,8 +36,97 @@ module Prism
           def allbits?(i) to_int.allbits?(i) end
           def anybits?(i) to_int.anybits?(i) end
           def nobits?(i) to_int.nobits?(i) end
+
+          # Instances are frozen and there are only a handful of them so we
+          # cache them here.
+          STATES = Hash.new { |hash, key| hash[key] = State.new(key) }
+          private_constant :STATES
+
+          def self.[](i)
+            STATES[i]
+          end
+        end
+
+        class Elem # :nodoc:
+          attr_accessor :pos, :event, :tok, :state, :message
+
+          def initialize(pos, event, tok, state, message = nil)
+            @pos = pos
+            @event = event
+            @tok = tok
+            @state = State[state]
+            @message = message
+          end
+
+          def [](index)
+            case index
+            when 0, :pos
+              @pos
+            when 1, :event
+              @event
+            when 2, :tok
+              @tok
+            when 3, :state
+              @state
+            when 4, :message
+              @message
+            else
+              nil
+            end
+          end
+
+          def inspect
+            "#<#{self.class}: #{event}@#{pos[0]}:#{pos[1]}:#{state}: #{tok.inspect}#{": " if message}#{message}>"
+          end
+
+          alias to_s inspect
+
+          def pretty_print(q)
+            q.group(2, "#<#{self.class}:", ">") {
+              q.breakable
+              q.text("#{event}@#{pos[0]}:#{pos[1]}")
+              q.breakable
+              state.pretty_print(q)
+              q.breakable
+              q.text("token: ")
+              tok.pretty_print(q)
+              if message
+                q.breakable
+                q.text("message: ")
+                q.text(message)
+              end
+            }
+          end
+
+          def to_a
+            if @message
+              [@pos, @event, @tok, @state, @message]
+            else
+              [@pos, @event, @tok, @state]
+            end
+          end
+        end
+
+        # Pretty much just the same as Prism.lex_compat.
+        def lex(raise_errors: false)
+          Ripper.lex(@source, filename, lineno, raise_errors: raise_errors)
+        end
+
+        # Returns the lex_compat result wrapped in `Elem`. Errors are omitted.
+        # Since ripper is a streaming parser, tokens are expected to be emitted in the order
+        # that the parser encounters them. This is not implemented.
+        def parse(...)
+          lex(...).map do |position, event, token, state|
+            Elem.new(position, event, token, state.to_int)
+          end
+        end
+
+        # Similar to parse but ripper sorts the elements by position in the source. Also
+        # includes errors. Since prism does error recovery, in cases of syntax errors
+        # the result may differ greatly compared to ripper.
+        def scan(...)
+          parse(...)
         end
-        # :startdoc:
       end
     end
   end

data/lib/prism/translation/ripper/sexp.rb

@@ -8,9 +8,7 @@ module Prism
     class Ripper
       # This class mirrors the ::Ripper::SexpBuilder subclass of ::Ripper that
       # returns the arrays of [type, *children].
-      class SexpBuilder < Ripper
-        # :stopdoc:
-
+      class SexpBuilder < Ripper # :nodoc:
         attr_reader :error
 
         private
@@ -65,16 +63,12 @@ module Prism
         remove_method :on_parse_error
         alias on_parse_error on_error
         alias compile_error on_error
-
-        # :startdoc:
       end
 
       # This class mirrors the ::Ripper::SexpBuilderPP subclass of ::Ripper that
       # returns the same values as ::Ripper::SexpBuilder except with a couple of
       # niceties that flatten linked lists into arrays.
-      class SexpBuilderPP < SexpBuilder
-        # :stopdoc:
-
+      class SexpBuilderPP < SexpBuilder # :nodoc:
         private
 
         def on_heredoc_dedent(val, width)
@@ -118,8 +112,6 @@ module Prism
             alias_method "on_#{event}", :_dispatch_event_push
           end
         end
-
-        # :startdoc:
       end
     end
   end

data/lib/prism/translation/ripper/shim.rb

@@ -2,4 +2,6 @@
 
 # This writes the prism ripper translation into the Ripper constant so that
 # users can transparently use Ripper without any changes.
+# :stopdoc:
 Ripper = Prism::Translation::Ripper
+# :startdoc: