ripper 1.0.0

data/ext/tools/generate-param-macros.rb

@@ -0,0 +1,14 @@
+off = true
+ARGF.each do |line|
+  case line
+  when /RIPPER_PARAMS_DECL_BEGIN/
+    off = false
+  when /RIPPER_PARAMS_DECL_END/
+    exit
+  when /ripper/
+    next if off
+    var = line.scan(/\w+/).last or next
+    base = var.sub(/ripper_/, '')
+    puts %"\#define #{base}\t\t(parser->ripper_#{base})"
+  end
+end

data/ext/tools/generate.rb

@@ -0,0 +1,152 @@
+# $Id: generate.rb 25189 2009-10-02 12:04:37Z akr $
+
+require 'optparse'
+
+def main
+  mode = nil
+  ids1src = nil
+  ids2src = nil
+  template = nil
+  output = nil
+
+  parser = @parser = OptionParser.new
+  parser.banner = "Usage: #{File.basename($0)} --mode=MODE [--ids1src=PATH] [--ids2src=PATH] [--output=PATH]"
+  parser.on('--mode=MODE', 'check, eventids1, or eventids2table.') {|m|
+    mode = m
+  }
+  parser.on('--ids1src=PATH', 'A source file of event-IDs 1 (parse.y).') {|path|
+    ids1src = path
+  }
+  parser.on('--ids2src=PATH', 'A source file of event-IDs 2 (eventids2.c).') {|path|
+    ids2src = path
+  }
+  parser.on('--output=PATH', 'An output file.') {|path|
+    output = path
+  }
+  parser.on('--help', 'Prints this message and quit.') {
+    puts parser.help
+    exit true
+  }
+  begin
+    parser.parse!
+  rescue OptionParser::ParseError => err
+    usage err.message
+  end
+  usage 'no mode given' unless mode
+  case mode
+  when 'check'
+    usage 'no --ids1src' unless ids1src
+    usage 'no --ids2src' unless ids2src
+    h = read_ids1_with_locations(ids1src)
+    check_arity h
+    ids2 = read_ids2(ids2src)
+    common = h.keys & ids2
+    unless common.empty?
+      abort "event crash: #{common.join(' ')}"
+    end
+    exit 0
+  when 'eventids1'
+    usage 'no --ids1src' unless ids1src
+    result = generate_eventids1(read_ids1(ids1src))
+  when 'eventids2table'
+    usage 'no --ids2src' unless ids2src
+    result = generate_eventids2_table(read_ids2(ids2src))
+  end
+  if output
+    File.open(output, 'w') {|f|
+      f.write result
+    }
+  else
+    puts result
+  end
+end
+
+def usage(msg)
+  $stderr.puts msg
+  $stderr.puts @parser.help
+  exit false
+end
+
+def generate_eventids1(ids)
+  buf = ""
+  ids.each do |id, arity|
+    buf << %Q[static ID ripper_id_#{id};\n]
+  end
+  buf << %Q[\n]
+  buf << %Q[static void\n]
+  buf << %Q[ripper_init_eventids1(VALUE self)\n]
+  buf << %Q[{\n]
+  buf << %Q[    VALUE h;\n]
+  buf << %Q[    ID id;\n]
+  ids.each do |id, arity|
+    buf << %Q[    ripper_id_#{id} = rb_intern_const("on_#{id}");\n]
+  end
+  buf << %Q[\n]
+  buf << %Q[    h = rb_hash_new();\n]
+  buf << %Q[    rb_define_const(self, "PARSER_EVENT_TABLE", h);\n]
+  ids.each do |id, arity|
+    buf << %Q[    id = rb_intern_const("#{id}");\n]
+    buf << %Q[    rb_hash_aset(h, ID2SYM(id), INT2NUM(#{arity}));\n]
+  end
+  buf << %Q[}\n]
+  buf
+end
+
+def generate_eventids2_table(ids)
+  buf = ""
+  buf << %Q[static void\n]
+  buf << %Q[ripper_init_eventids2_table(VALUE self)\n]
+  buf << %Q[{\n]
+  buf << %Q[    VALUE h = rb_hash_new();\n]
+  buf << %Q[    ID id;\n]
+  buf << %Q[    rb_define_const(self, "SCANNER_EVENT_TABLE", h);\n]
+  ids.each do |id|
+    buf << %Q[    id = rb_intern_const("#{id}");\n]
+    buf << %Q[    rb_hash_aset(h, ID2SYM(id), INT2NUM(1));\n]
+  end
+  buf << %Q[}\n]
+  buf
+end
+
+def read_ids1(path)
+  strip_locations(read_ids1_with_locations(path))
+end
+
+def strip_locations(h)
+  h.map {|event, list| [event, list.first[1]] }\
+      .sort_by {|event, arity| event.to_s }
+end
+
+def check_arity(h)
+  invalid = false
+  h.each do |event, list|
+    unless list.map {|line, arity| arity }.uniq.size == 1
+      invalid = true
+      locations = list.map {|line, a| "#{line}:#{a}" }.join(', ')
+      $stderr.puts "arity crash [event=#{event}]: #{locations}"
+    end
+  end
+  abort if invalid
+end
+
+def read_ids1_with_locations(path)
+  h = {}
+  File.open(path) {|f|
+    f.each do |line|
+      next if /\A\#\s*define\s+s?dispatch/ =~ line
+      next if /ripper_dispatch/ =~ line
+      line.scan(/dispatch(\d)\((\w+)/) do |arity, event|
+        (h[event] ||= []).push [f.lineno, arity.to_i]
+      end
+    end
+  }
+  h
+end
+
+def read_ids2(path)
+  File.open(path) {|f|
+    return f.read.scan(/ripper_id_(\w+)/).flatten.uniq.sort
+  }
+end
+
+main

data/ext/tools/preproc.rb

@@ -0,0 +1,91 @@
+# $Id: preproc.rb 25189 2009-10-02 12:04:37Z akr $
+
+require 'optparse'
+
+def main
+  output = nil
+  parser = OptionParser.new
+  parser.banner = "Usage: #{File.basename($0)} [--output=PATH] <parse.y>"
+  parser.on('--output=PATH', 'An output file.') {|path|
+    output = path
+  }
+  parser.on('--help', 'Prints this message and quit.') {
+    puts parser.help
+    exit true
+  }
+  begin
+    parser.parse!
+  rescue OptionParser::ParseError => err
+    $stderr.puts err.message
+    $stderr.puts parser.help
+    exit false
+  end
+  unless ARGV.size == 1
+    abort "wrong number of arguments (#{ARGV.size} for 1)"
+  end
+  out = ""
+  File.open(ARGV[0]) {|f|
+    prelude f, out
+    grammar f, out
+    usercode f, out
+  }
+  if output
+    File.open(output, 'w') {|f|
+      f.write out
+    }
+  else
+    print out
+  end
+end
+
+def prelude(f, out)
+  while line = f.gets
+    case line
+    when %r</\*%%%\*/>
+      out << '/*' << $/
+    when %r</\*%>
+      out << '*/' << $/
+    when %r<%\*/>
+      out << $/
+    when /\A%%/
+      out << '%%' << $/
+      return
+    when /\A%token/
+      out << line.sub(/<\w+>/, '<val>')
+    when /\A%type/
+      out << line.sub(/<\w+>/, '<val>')
+    else
+      out << line
+    end
+  end
+end
+
+def grammar(f, out)
+  while line = f.gets
+    case line
+    when %r</\*%%%\*/>
+      out << '#if 0' << $/
+    when %r</\*%c%\*/>
+      out << '/*' << $/
+    when %r</\*%c>
+      out << '*/' << $/
+    when %r</\*%>
+      out << '#endif' << $/
+    when %r<%\*/>
+      out << $/
+    when /\A%%/
+      out << '%%' << $/
+      return
+    else
+      out << line
+    end
+  end
+end
+
+def usercode(f, out)
+  while line = f.gets
+    out << line
+  end
+end
+
+main

data/ext/tools/strip.rb

@@ -0,0 +1,12 @@
+last_is_void = false
+ARGF.each do |line|
+  if line.strip.empty?
+    #puts() unless last_is_void
+    last_is_void = true
+  elsif /\A\#/ === line
+    ;
+  else
+    print line
+    last_is_void = false
+  end
+end

data/ext/tools/ytab.sed

@@ -0,0 +1,31 @@
+#!/bin/sed -f
+/^int yydebug;/{
+i\
+#ifndef yydebug
+a\
+#endif
+}
+/^yydestruct.*yymsg/,/#endif/{
+  /^yydestruct/{
+    /parser/!{
+      h
+      s/^/ruby_parser_&/
+      s/)$/, parser)/
+      /\*/s/parser)$/struct parser_params *&/
+    }
+  }
+  /^#endif/{
+    x
+    /^./{
+      i\
+    struct parser_params *parser;
+      a\
+#define yydestruct(m, t, v) ruby_parser_yydestruct(m, t, v, parser)
+    }
+    x
+  }
+}
+s/^\([ 	]*\)\(yyerror[ 	]*([ 	]*parser,\)/\1parser_\2/
+s!^ *extern char \*getenv();!/* & */!
+s/^\(#.*\)".*\.tab\.c"/\1"ripper.c"/
+/^\(#.*\)".*\.y"/s:\\\\:/:g

data/lib/ripper.rb

@@ -0,0 +1,4 @@
+require 'ripper/core'
+require 'ripper/lexer'
+require 'ripper/filter'
+require 'ripper/sexp'

data/lib/ripper/core.rb

@@ -0,0 +1,70 @@
+#
+# $Id: core.rb 25189 2009-10-02 12:04:37Z akr $
+#
+# Copyright (c) 2003-2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require File.dirname(__FILE__) + '/../../ext/ripper.so'
+
+class Ripper
+
+  # Parses Ruby program read from _src_.
+  # _src_ must be a String or a IO or a object which has #gets method.
+  def Ripper.parse(src, filename = '(ripper)', lineno = 1)
+    new(src, filename, lineno).parse
+  end
+
+  # This array contains name of parser events.
+  PARSER_EVENTS = PARSER_EVENT_TABLE.keys
+
+  # This array contains name of scanner events.
+  SCANNER_EVENTS = SCANNER_EVENT_TABLE.keys
+
+  # This array contains name of all ripper events.
+  EVENTS = PARSER_EVENTS + SCANNER_EVENTS
+
+  private
+
+  #
+  # Parser Events
+  #
+
+  PARSER_EVENT_TABLE.each do |id, arity|
+    module_eval(<<-End, __FILE__, __LINE__ + 1)
+      def on_#{id}(#{ ('a'..'z').to_a[0, arity].join(', ') })
+        #{arity == 0 ? 'nil' : 'a'}
+      end
+    End
+  end
+
+  # This method is called when weak warning is produced by the parser.
+  # _fmt_ and _args_ is printf style.
+  def warn(fmt, *args)
+  end
+
+  # This method is called when strong warning is produced by the parser.
+  # _fmt_ and _args_ is printf style.
+  def warning(fmt, *args)
+  end
+
+  # This method is called when the parser found syntax error.
+  def compile_error(msg)
+  end
+
+  #
+  # Scanner Events
+  #
+
+  SCANNER_EVENTS.each do |id|
+    module_eval(<<-End, __FILE__, __LINE__ + 1)
+      def on_#{id}(token)
+        token
+      end
+    End
+  end
+
+end

data/lib/ripper/filter.rb

@@ -0,0 +1,70 @@
+#
+# $Id: filter.rb 25189 2009-10-02 12:04:37Z akr $
+#
+# Copyright (c) 2004,2005 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper/lexer'
+
+class Ripper
+
+  # This class handles only scanner events,
+  # and they are dispatched in the `right' order (same with input).
+  class Filter
+
+    def initialize(src, filename = '-', lineno = 1)
+      @__lexer = Lexer.new(src, filename, lineno)
+      @__line = nil
+      @__col = nil
+    end
+
+    # The file name of the input.
+    def filename
+      @__lexer.filename
+    end
+
+    # The line number of the current token.
+    # This value starts from 1.
+    # This method is valid only in event handlers.
+    def lineno
+      @__line
+    end
+
+    # The column number of the current token.
+    # This value starts from 0.
+    # This method is valid only in event handlers.
+    def column
+      @__col
+    end
+
+    # Starts parsing.  _init_ is a data accumulator.
+    # It is passed to the next event handler (as of Enumerable#inject).
+    def parse(init = nil)
+      data = init
+      @__lexer.lex.each do |pos, event, tok|
+        @__line, @__col = *pos
+        data = if respond_to?(event, true)
+               then __send__(event, tok, data)
+               else on_default(event, tok, data)
+               end
+      end
+      data
+    end
+
+    private
+
+    # This method is called when some event handler have not defined.
+    # _event_ is :on_XXX, _token_ is scanned token, _data_ is a data
+    # accumulator.  The return value of this method is passed to the
+    # next event handler (as of Enumerable#inject).
+    def on_default(event, token, data)
+      data
+    end
+
+  end
+
+end