langscan 1.2

data/lib/langscan/ruby/compat/ripper/filter.rb

@@ -0,0 +1,70 @@
+#
+# ripper/filter.rb
+#
+# Copyright (C) 2004 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'langscan/ruby/compat/ripper/lexer'
+
+class Ripper
+
+  # This class handles only scanner events,
+  # and they are dispatched in the `right' order (same with input).
+  class Filter
+
+    def initialize(src, filename = '-', lineno = 1)
+      @__lexer = Lexer.new(src, filename, lineno)
+      @__line = nil
+      @__col = nil
+    end
+
+    # The file name of the input.
+    def filename
+      @__lexer.filename
+    end
+
+    # The line number of the current token.
+    # This value starts from 1.
+    # This method is valid only in event handlers.
+    def lineno
+      @__line
+    end
+
+    # The column number of the current token.
+    # This value starts from 0.
+    # This method is valid only in event handlers.
+    def column
+      @__col
+    end
+
+    # Starts parsing.  _init_ is a data accumulator.
+    # It is passed to the next event handler (as of Enumerable#inject).
+    def parse(init = nil)
+      data = init
+      @__lexer.lex.each do |pos, event, tok|
+        @__line, @__col = *pos
+        data = if respond_to?(event, true)
+               then __send__(event, tok, data)
+               else on_default(event, tok, data)
+               end
+      end
+      data
+    end
+
+    private
+
+    # This method is called when some event handler have not defined.
+    # _event_ is :on_XXX, _token_ is scanned token, _data_ is a data
+    # accumulator.  The return value of this method is passed to the
+    # next event handler (as of Enumerable#inject).
+    def on_default(event, token, data)
+      data
+    end
+
+  end
+
+end

data/lib/langscan/ruby/compat/ripper/lexer.rb

@@ -0,0 +1,179 @@
+#
+# ripper/lexer.rb
+#
+# Copyright (C) 2004,2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'langscan/ruby/compat/ripper/core'
+
+class Ripper
+
+  # Tokenizes Ruby program and returns an Array of String.
+  def Ripper.tokenize(src, filename = '-', lineno = 1)
+    Lexer.new(src, filename, lineno).tokenize
+  end
+
+  # Tokenizes Ruby program and returns an Array of Array,
+  # which is formatted like [[lineno, column], type, token].
+  #
+  #   require 'ripper'
+  #   require 'pp'
+  #
+  #   p Ripper.scan("def m(a) nil end")
+  #     #=> [[[1,  0], :on_kw,     "def"],
+  #          [[1,  3], :on_sp,     " "  ],
+  #          [[1,  4], :on_ident,  "m"  ],
+  #          [[1,  5], :on_lparen, "("  ],
+  #          [[1,  6], :on_ident,  "a"  ],
+  #          [[1,  7], :on_rparen, ")"  ],
+  #          [[1,  8], :on_sp,     " "  ],
+  #          [[1,  9], :on_kw,     "nil"],
+  #          [[1, 12], :on_sp,     " "  ],
+  #          [[1, 13], :on_kw,     "end"]]
+  #
+  def Ripper.lex(src, filename = '-', lineno = 1)
+    Lexer.new(src, filename, lineno).lex
+  end
+
+  class Lexer < ::Ripper   #:nodoc: internal use only
+    def tokenize
+      lex().map {|pos, event, tok| tok }
+    end
+
+    def lex
+      parse().sort_by {|pos, event, tok| pos }
+    end
+
+    def parse
+      @buf = []
+      super
+      @buf
+    end
+
+    private
+
+    SCANNER_EVENTS.each do |event|
+      module_eval(<<-End, __FILE__+'/module_eval', __LINE__ + 1)
+        def on_#{event}(tok)
+          @buf.push [[lineno(), column()], :on_#{event}, tok]
+        end
+      End
+    end
+  end
+
+  # [EXPERIMENTAL]
+  # Parses +src+ and return a string which was matched to +pattern+.
+  # +pattern+ should be described as Regexp.
+  #
+  #   require 'ripper'
+  #
+  #   p Ripper.slice('def m(a) nil end', 'ident')                   #=> "m"
+  #   p Ripper.slice('def m(a) nil end', '[ident lparen rparen]+')  #=> "m(a)"
+  #   p Ripper.slice("<<EOS\nstring\nEOS",
+  #                  'heredoc_beg nl $(tstring_content*) heredoc_end', 1)
+  #       #=> "string\n"
+  #
+  def Ripper.slice(src, pattern, n = 0)
+    if m = token_match(src, pattern)
+    then m.string(n)
+    else nil
+    end
+  end
+
+  def Ripper.token_match(src, pattern)   #:nodoc:
+    TokenPattern.compile(pattern).match(src)
+  end
+
+  class TokenPattern   #:nodoc:
+
+    class Error < ::StandardError; end
+    class CompileError < Error; end
+    class MatchError < Error; end
+
+    class << self
+      alias compile new
+    end
+
+    def initialize(pattern)
+      @source = pattern
+      @re = compile(pattern)
+    end
+
+    def match(str)
+      match_list(::Ripper.lex(str))
+    end
+
+    def match_list(tokens)
+      if m = @re.match(map_tokens(tokens))
+      then MatchData.new(tokens, m)
+      else nil
+      end
+    end
+
+    private
+
+    def compile(pattern)
+      if m = /[^\w\s$()\[\]{}?*+\.]/.match(pattern)
+        raise CompileError, "invalid char in pattern: #{m[0].inspect}"
+      end
+      buf = ''
+      pattern.scan(/(?:\w+|\$\(|[()\[\]\{\}?*+\.]+)/) do |tok|
+        case tok
+        when /\w/
+          buf.concat map_token(tok)
+        when '$('
+          buf.concat '('
+        when '('
+          buf.concat '(?:'
+        when /[?*\[\])\.]/
+          buf.concat tok
+        else
+          raise 'must not happen'
+        end
+      end
+      Regexp.compile(buf)
+    rescue RegexpError => err
+      raise CompileError, err.message
+    end
+
+    def map_tokens(tokens)
+      tokens.map {|pos,type,str| map_token(type.to_s.sub(/\Aon_/,'')) }.join
+    end
+
+    MAP = {}
+    seed = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a
+    SCANNER_EVENT_TABLE.each do |ev, |
+      raise CompileError, "[RIPPER FATAL] too many system token" if seed.empty?
+      MAP[ev.to_s.sub(/\Aon_/,'')] = seed.shift
+    end
+
+    def map_token(tok)
+      MAP[tok]  or raise CompileError, "unknown token: #{tok}"
+    end
+
+    class MatchData
+      def initialize(tokens, match)
+        @tokens = tokens
+        @match = match
+      end
+
+      def string(n = 0)
+        return nil unless @match
+        match(n).join
+      end
+
+      private
+
+      def match(n = 0)
+        return [] unless @match
+        @tokens[@match.begin(n)...@match.end(n)].map {|pos,type,str| str }
+      end
+    end
+  
+  end
+
+end

data/lib/langscan/ruby/compat/ripper/sexp.rb

@@ -0,0 +1,100 @@
+#
+# ripper/sexp.rb
+#
+# Copyright (C) 2004,2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'langscan/ruby/compat/ripper/core'
+
+class Ripper
+
+  # [EXPERIMENTAL]
+  # Parses +src+ and create S-exp tree.
+  # This method is for mainly developper use.
+  #
+  #   require 'ripper'
+  #   require 'pp
+  #
+  #   pp Ripper.sexp("def m(a) nil end")
+  #     #=> [:program,
+  #          [:stmts_add,
+  #           [:stmts_new],
+  #           [:def,
+  #            [:@ident, "m", [1, 4]],
+  #            [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil]],
+  #            [:bodystmt,
+  #             [:stmts_add, [:stmts_new], [:var_ref, [:@kw, "nil", [1, 9]]]],
+  #             nil,
+  #             nil,
+  #             nil]]]]
+  #
+  def Ripper.sexp(src, filename = '-', lineno = 1)
+    SexpBuilderPP.new(src, filename, lineno).parse
+  end
+
+  def Ripper.sexp_raw(src, filename = '-', lineno = 1)
+    SexpBuilder.new(src, filename, lineno).parse
+  end
+
+  class SexpBuilderPP < ::Ripper   #:nodoc:
+    private
+
+    PARSER_EVENTS.each do |event|
+      case event.to_s
+      when /_new\z/
+        module_eval(<<-End, __FILE__, __LINE__ + 1)
+          def on_#{event}(*args)
+            []
+          end
+        End
+      when /_add\z/
+        module_eval(<<-End, __FILE__, __LINE__ + 1)
+          def on_#{event}(list, item)
+            list.push item
+            list
+          end
+        End
+      else
+        module_eval(<<-End, __FILE__, __LINE__ + 1)
+          def on_#{event}(*args)
+            [:#{event}, *args]
+          end
+        End
+      end
+    end
+
+    SCANNER_EVENTS.each do |event|
+      module_eval(<<-End, __FILE__, __LINE__ + 1)
+        def on_#{event}(tok)
+          [:@#{event}, tok, [lineno(), column()]]
+        end
+      End
+    end
+  end
+
+  class SexpBuilder < ::Ripper   #:nodoc:
+    private
+
+    PARSER_EVENTS.each do |event|
+      module_eval(<<-End, __FILE__, __LINE__ + 1)
+        def on_#{event}(*args)
+          args.unshift :#{event}
+          args
+        end
+      End
+    end
+
+    SCANNER_EVENTS.each do |event|
+      module_eval(<<-End, __FILE__, __LINE__ + 1)
+        def on_#{event}(tok)
+          [:@#{event}, tok, [lineno(), column()]]
+        end
+      End
+    end
+  end
+
+end

data/lib/langscan/scheme.rb

@@ -0,0 +1,160 @@
+#
+# scheme.rb - a Scheme module of LangScan
+#
+# Copyright (C) 2005 Kenichi Ishibashi <bashi at dream.ie.ariake-nct.ac.jp>
+#     All rights reserved.
+#     This is free software with ABSOLUTELY NO WARRANTY.
+#
+# You can redistribute it and/or modify it under the terms of 
+# the GNU General Public License version 2.
+#
+require 'langscan/scheme/scheme'
+require 'langscan/_common'
+require 'langscan/_pairmatcher'
+
+class Struct::LangScanPair
+  def each_outer
+    ret = self
+    while o = ret.outer
+      yield o
+    end
+  end
+end
+
+module LangScan
+  module Scheme
+    module_function
+    def name
+      "Scheme"
+    end
+
+
+    def abbrev
+      "scheme"
+    end
+
+    def extnames
+      [".scm"]
+    end
+
+    # LangScan::Scheme.scan iterates over Scheme program.
+    # It yields for each Fragment.
+    def scan(input, &block)
+      sorter = PairMatcher.fragmentsorter(block)
+      scan_unsorted(input, &sorter)
+    end
+
+    def scan_unsorted(input, &block)
+      pm = LangScan::PairMatcher.new(2,2,2,2)
+      pm.define_intertoken_fragment :space, nil
+      pm.define_intertoken_fragment :comment, nil
+      pm.define_pair :paren, :punct, "(", :punct, ")"
+      pm.define_pair :vector, :punct, "#(", :punct, ")"
+      reporter = lambda {|f|
+        if (f.type == :ident || f.type == :funcall) && KeywordsHash[f.text]
+          f.type = :keyword
+        end
+        if f.type == :number
+          f.type = if f.text.include?("i")
+                     :imaginary
+                   elsif f.text =~ /[.\/]|[0-9]#*[esfdl][0-9\-+]/
+                     :floating
+                   else
+                     :integer
+                   end
+        end
+        if f.type == :quote_chars
+          f.type = :punct
+        end
+        yield f
+      }
+      pm.parse(LangScan::Scheme::Tokenizer.new(input), reporter) {|list|
+        if list.around_open(1).type == :ident
+          list.around_open(1).type = case
+                                     when fundef_list?(list)
+                                       :fundef
+                                     when funcall_list?(list)
+                                       :funcall
+                                     else
+                                       :ident
+                                     end
+        end
+      }
+    end
+
+    def fundef_list?(list)
+      if list.before_open_length >= 1 && list.around_open(-1).text == "define"
+        return true
+      end
+      return false
+    end
+
+    def funcall_list?(list)
+      if list.before_open_length == 0
+        return true
+      end
+      if NotFuncallWordsHash[list.around_open(-1).text]
+        return false
+      end
+      if quote_list?(list)
+        return false
+      end
+      outer = list.outer
+      second_outer = outer.outer unless outer == nil
+      if second_outer and NotFuncall2ndOuterWordsHash[second_outer.around_open(1).text]
+        if NotFuncall2ndOuterWordsHash[outer.around_open(-1).text]
+          return false
+        end
+      end
+      return true
+    end
+
+    def quote_nestlevel(str)
+      str.count("`") - str.count(",")
+    end
+
+    def quote_list?(list)
+      l = list
+      nest = 0
+      while l
+        if l.before_open_length >= 1
+          before = l.around_open(-1)
+          if before.type == :quote_chars
+            return true if before.text.include?("'")
+            nest = nest + quote_nestlevel(before.text)
+          end
+        end
+        if l.after_open_length >= 1
+          after = l.around_open(1)
+          if after.text == 'quote'
+            return true
+          elsif after.text == 'quasiquote'
+            nest = nest + 1
+          elsif after.text == 'unquote'
+            nest = nest - 1
+          end
+        end
+        l = l.outer
+      end
+      return nest > 0
+    end
+
+    Keywords = %w(
+      else => define unquote unquote-splicing quote lambda if
+      set! begin cond and or case let let* letrec do delay quasiquote
+      syntax-rules define-syntax
+    )
+    KeywordsHash = {}
+    Keywords.each {|k| KeywordsHash[k] = k }
+
+    NotFuncallWordsHash = {"lambda" => "lambda"}
+
+    NotFuncall2ndOuterWords = %w(
+      let let* letrec let-syntax letrec-syntax do
+    )
+    NotFuncall2ndOuterWordsHash = {}
+    NotFuncall2ndOuterWords.each {|k| NotFuncall2ndOuterWordsHash[k] = k }
+
+    LangScan.register(self)
+  end
+end