ripper 1.0.0

data/lib/ripper/lexer.rb

@@ -0,0 +1,179 @@
+#
+# $Id: lexer.rb 25189 2009-10-02 12:04:37Z akr $
+#
+# Copyright (c) 2004,2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper/core'
+
+class Ripper
+
+  # Tokenizes Ruby program and returns an Array of String.
+  def Ripper.tokenize(src, filename = '-', lineno = 1)
+    Lexer.new(src, filename, lineno).tokenize
+  end
+
+  # Tokenizes Ruby program and returns an Array of Array,
+  # which is formatted like [[lineno, column], type, token].
+  #
+  #   require 'ripper'
+  #   require 'pp'
+  #
+  #   p Ripper.lex("def m(a) nil end")
+  #     #=> [[[1,  0], :on_kw,     "def"],
+  #          [[1,  3], :on_sp,     " "  ],
+  #          [[1,  4], :on_ident,  "m"  ],
+  #          [[1,  5], :on_lparen, "("  ],
+  #          [[1,  6], :on_ident,  "a"  ],
+  #          [[1,  7], :on_rparen, ")"  ],
+  #          [[1,  8], :on_sp,     " "  ],
+  #          [[1,  9], :on_kw,     "nil"],
+  #          [[1, 12], :on_sp,     " "  ],
+  #          [[1, 13], :on_kw,     "end"]]
+  #
+  def Ripper.lex(src, filename = '-', lineno = 1)
+    Lexer.new(src, filename, lineno).lex
+  end
+
+  class Lexer < ::Ripper   #:nodoc: internal use only
+    def tokenize
+      lex().map {|pos, event, tok| tok }
+    end
+
+    def lex
+      parse().sort_by {|pos, event, tok| pos }
+    end
+
+    def parse
+      @buf = []
+      super
+      @buf
+    end
+
+    private
+
+    SCANNER_EVENTS.each do |event|
+      module_eval(<<-End, __FILE__+'/module_eval', __LINE__ + 1)
+        def on_#{event}(tok)
+          @buf.push [[lineno(), column()], :on_#{event}, tok]
+        end
+      End
+    end
+  end
+
+  # [EXPERIMENTAL]
+  # Parses +src+ and return a string which was matched to +pattern+.
+  # +pattern+ should be described as Regexp.
+  #
+  #   require 'ripper'
+  #
+  #   p Ripper.slice('def m(a) nil end', 'ident')                   #=> "m"
+  #   p Ripper.slice('def m(a) nil end', '[ident lparen rparen]+')  #=> "m(a)"
+  #   p Ripper.slice("<<EOS\nstring\nEOS",
+  #                  'heredoc_beg nl $(tstring_content*) heredoc_end', 1)
+  #       #=> "string\n"
+  #
+  def Ripper.slice(src, pattern, n = 0)
+    if m = token_match(src, pattern)
+    then m.string(n)
+    else nil
+    end
+  end
+
+  def Ripper.token_match(src, pattern)   #:nodoc:
+    TokenPattern.compile(pattern).match(src)
+  end
+
+  class TokenPattern   #:nodoc:
+
+    class Error < ::StandardError; end
+    class CompileError < Error; end
+    class MatchError < Error; end
+
+    class << self
+      alias compile new
+    end
+
+    def initialize(pattern)
+      @source = pattern
+      @re = compile(pattern)
+    end
+
+    def match(str)
+      match_list(::Ripper.lex(str))
+    end
+
+    def match_list(tokens)
+      if m = @re.match(map_tokens(tokens))
+      then MatchData.new(tokens, m)
+      else nil
+      end
+    end
+
+    private
+
+    def compile(pattern)
+      if m = /[^\w\s$()\[\]{}?*+\.]/.match(pattern)
+        raise CompileError, "invalid char in pattern: #{m[0].inspect}"
+      end
+      buf = ''
+      pattern.scan(/(?:\w+|\$\(|[()\[\]\{\}?*+\.]+)/) do |tok|
+        case tok
+        when /\w/
+          buf.concat map_token(tok)
+        when '$('
+          buf.concat '('
+        when '('
+          buf.concat '(?:'
+        when /[?*\[\])\.]/
+          buf.concat tok
+        else
+          raise 'must not happen'
+        end
+      end
+      Regexp.compile(buf)
+    rescue RegexpError => err
+      raise CompileError, err.message
+    end
+
+    def map_tokens(tokens)
+      tokens.map {|pos,type,str| map_token(type.to_s.sub(/\Aon_/,'')) }.join
+    end
+
+    MAP = {}
+    seed = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a
+    SCANNER_EVENT_TABLE.each do |ev, |
+      raise CompileError, "[RIPPER FATAL] too many system token" if seed.empty?
+      MAP[ev.to_s.sub(/\Aon_/,'')] = seed.shift
+    end
+
+    def map_token(tok)
+      MAP[tok]  or raise CompileError, "unknown token: #{tok}"
+    end
+
+    class MatchData
+      def initialize(tokens, match)
+        @tokens = tokens
+        @match = match
+      end
+
+      def string(n = 0)
+        return nil unless @match
+        match(n).join
+      end
+
+      private
+
+      def match(n = 0)
+        return [] unless @match
+        @tokens[@match.begin(n)...@match.end(n)].map {|pos,type,str| str }
+      end
+    end
+
+  end
+
+end

data/lib/ripper/sexp.rb

@@ -0,0 +1,99 @@
+#
+# $Id: sexp.rb 25189 2009-10-02 12:04:37Z akr $
+#
+# Copyright (c) 2004,2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper/core'
+
+class Ripper
+
+  # [EXPERIMENTAL]
+  # Parses +src+ and create S-exp tree.
+  # This method is for mainly developper use.
+  #
+  #   require 'ripper'
+  #   require 'pp
+  #
+  #   pp Ripper.sexp("def m(a) nil end")
+  #     #=> [:program,
+  #          [:stmts_add,
+  #           [:stmts_new],
+  #           [:def,
+  #            [:@ident, "m", [1, 4]],
+  #            [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil]],
+  #            [:bodystmt,
+  #             [:stmts_add, [:stmts_new], [:var_ref, [:@kw, "nil", [1, 9]]]],
+  #             nil,
+  #             nil,
+  #             nil]]]]
+  #
+  def Ripper.sexp(src, filename = '-', lineno = 1)
+    SexpBuilderPP.new(src, filename, lineno).parse
+  end
+
+  def Ripper.sexp_raw(src, filename = '-', lineno = 1)
+    SexpBuilder.new(src, filename, lineno).parse
+  end
+
+  class SexpBuilderPP < ::Ripper   #:nodoc:
+    private
+
+    PARSER_EVENT_TABLE.each do |event, arity|
+      if /_new\z/ =~ event.to_s and arity == 0
+        module_eval(<<-End, __FILE__, __LINE__ + 1)
+          def on_#{event}
+            []
+          end
+        End
+      elsif /_add\z/ =~ event.to_s
+        module_eval(<<-End, __FILE__, __LINE__ + 1)
+          def on_#{event}(list, item)
+            list.push item
+            list
+          end
+        End
+      else
+        module_eval(<<-End, __FILE__, __LINE__ + 1)
+          def on_#{event}(*args)
+            [:#{event}, *args]
+          end
+        End
+      end
+    end
+
+    SCANNER_EVENTS.each do |event|
+      module_eval(<<-End, __FILE__, __LINE__ + 1)
+        def on_#{event}(tok)
+          [:@#{event}, tok, [lineno(), column()]]
+        end
+      End
+    end
+  end
+
+  class SexpBuilder < ::Ripper   #:nodoc:
+    private
+
+    PARSER_EVENTS.each do |event|
+      module_eval(<<-End, __FILE__, __LINE__ + 1)
+        def on_#{event}(*args)
+          args.unshift :#{event}
+          args
+        end
+      End
+    end
+
+    SCANNER_EVENTS.each do |event|
+      module_eval(<<-End, __FILE__, __LINE__ + 1)
+        def on_#{event}(tok)
+          [:@#{event}, tok, [lineno(), column()]]
+        end
+      End
+    end
+  end
+
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification 
+name: ripper
+version: !ruby/object:Gem::Version 
+  hash: 23
+  prerelease: 
+  segments: 
+  - 1
+  - 0
+  - 0
+  version: 1.0.0
+platform: ruby
+authors: 
+- Loren Segal
+- Minero Aoki
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-01-11 00:00:00 -05:00
+default_executable: 
+dependencies: []
+
+description: "    This gem is meant for Ruby 1.8.x builds. Ruby 1.9.x comes with a functioning\n    ripper implementation out of the box. This gem is a port of the Ripper 1.9\n    for Ruby 1.8.x. Ripper was written by Minero Aoki and ported by Loren Segal.\n"
+email: lsegal@soen.ca
+executables: []
+
+extensions: 
+- ext/extconf.rb
+extra_rdoc_files: []
+
+files: 
+- lib/ripper/core.rb
+- lib/ripper/filter.rb
+- lib/ripper/lexer.rb
+- lib/ripper/sexp.rb
+- lib/ripper.rb
+- ext/backports/backports.h
+- ext/backports/regenc.h
+- ext/backports/ruby/encoding.h
+- ext/depend
+- ext/eventids2.c
+- ext/extconf.rb
+- ext/extra/node.h
+- ext/extra/regenc.h
+- ext/id.c
+- ext/id.h
+- ext/lex.c
+- ext/Makefile
+- ext/parse.h
+- ext/parse.y
+- ext/tools/generate-param-macros.rb
+- ext/tools/generate.rb
+- ext/tools/preproc.rb
+- ext/tools/strip.rb
+- ext/tools/ytab.sed
+- Rakefile
+- README
+has_rdoc: true
+homepage: http://github.com/lsegal/ripper
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+- ext
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.4.0
+signing_key: 
+specification_version: 3
+summary: Ripper parses Ruby source and tokenizes or builds an AST
+test_files: []
+