mkbison 0.0.1

data/ext/bison_parser/bison_parser.y

@@ -0,0 +1,295 @@
+
+%token	IDENTIFIER
+%token	NUMBER
+%token	STRING
+%token	COLON
+%token	SEMICOLON
+%token	LBRACK
+%token	RBRACK
+%token	PIPE
+%token	HASH
+%token	DOUBLE_HASH
+%token	KW_TOKEN
+%token	KW_LEFT
+%token	KW_RIGHT
+%token	ACTIONS
+
+
+%define api.pure true
+%define parse.error verbose
+%parse-param { VALUE __actions }
+%lex-param { VALUE __actions }
+%locations
+
+%{
+#include <ruby.h>
+#define YYSTYPE VALUE
+%}
+
+%code provides {
+static int yylex(YYSTYPE *, YYLTYPE *, VALUE);
+static void yyerror(YYLTYPE *, VALUE, const char *);
+}
+
+%%
+
+grammar_file:
+  token_list DOUBLE_HASH grammar_rules optional_code 
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    rb_ivar_set(__actions, rb_intern("@tokens"), rb_ary_new3(2, INT2FIX(@1.first_line), INT2FIX(@1.first_column)));
+    rb_ivar_set(__actions, rb_intern("@rules"), rb_ary_new3(2, INT2FIX(@3.first_line), INT2FIX(@3.first_column)));
+    rb_ivar_set(__actions, rb_intern("@code"), rb_ary_new3(2, INT2FIX(@4.first_line), INT2FIX(@4.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_0_grammar_file"), 3, $1, $3, $4);
+  }
+
+;
+
+optional_code:
+  
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_0_optional_code"), 0);
+  }
+
+|
+  DOUBLE_HASH ACTIONS 
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    rb_ivar_set(__actions, rb_intern("@actions"), rb_ary_new3(2, INT2FIX(@2.first_line), INT2FIX(@2.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_1_optional_code"), 1, $2);
+  }
+
+;
+
+token_list:
+  
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_0_token_list"), 0);
+  }
+
+|
+  token_list token 
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    rb_ivar_set(__actions, rb_intern("@list"), rb_ary_new3(2, INT2FIX(@1.first_line), INT2FIX(@1.first_column)));
+    rb_ivar_set(__actions, rb_intern("@token"), rb_ary_new3(2, INT2FIX(@2.first_line), INT2FIX(@2.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_1_token_list"), 2, $1, $2);
+  }
+
+;
+
+token:
+  HASH KW_TOKEN IDENTIFIER 
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    rb_ivar_set(__actions, rb_intern("@name"), rb_ary_new3(2, INT2FIX(@3.first_line), INT2FIX(@3.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_0_token"), 1, $3);
+  }
+
+|
+  HASH KW_LEFT IDENTIFIER 
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    rb_ivar_set(__actions, rb_intern("@name"), rb_ary_new3(2, INT2FIX(@3.first_line), INT2FIX(@3.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_1_token"), 1, $3);
+  }
+
+|
+  HASH KW_RIGHT IDENTIFIER 
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    rb_ivar_set(__actions, rb_intern("@name"), rb_ary_new3(2, INT2FIX(@3.first_line), INT2FIX(@3.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_2_token"), 1, $3);
+  }
+
+|
+  token NUMBER 
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    rb_ivar_set(__actions, rb_intern("@token"), rb_ary_new3(2, INT2FIX(@1.first_line), INT2FIX(@1.first_column)));
+    rb_ivar_set(__actions, rb_intern("@num"), rb_ary_new3(2, INT2FIX(@2.first_line), INT2FIX(@2.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_3_token"), 2, $1, $2);
+  }
+
+;
+
+grammar_rules:
+  
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_0_grammar_rules"), 0);
+  }
+
+|
+  grammar_rules grammar_rule 
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    rb_ivar_set(__actions, rb_intern("@list"), rb_ary_new3(2, INT2FIX(@1.first_line), INT2FIX(@1.first_column)));
+    rb_ivar_set(__actions, rb_intern("@rule"), rb_ary_new3(2, INT2FIX(@2.first_line), INT2FIX(@2.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_1_grammar_rules"), 2, $1, $2);
+  }
+
+;
+
+grammar_rule:
+  IDENTIFIER COLON components SEMICOLON 
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    rb_ivar_set(__actions, rb_intern("@name"), rb_ary_new3(2, INT2FIX(@1.first_line), INT2FIX(@1.first_column)));
+    rb_ivar_set(__actions, rb_intern("@components"), rb_ary_new3(2, INT2FIX(@3.first_line), INT2FIX(@3.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_0_grammar_rule"), 2, $1, $3);
+  }
+
+;
+
+components:
+  sequence 
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    rb_ivar_set(__actions, rb_intern("@sequence"), rb_ary_new3(2, INT2FIX(@1.first_line), INT2FIX(@1.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_0_components"), 1, $1);
+  }
+
+|
+  components PIPE sequence 
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    rb_ivar_set(__actions, rb_intern("@sequences"), rb_ary_new3(2, INT2FIX(@1.first_line), INT2FIX(@1.first_column)));
+    rb_ivar_set(__actions, rb_intern("@sequence"), rb_ary_new3(2, INT2FIX(@3.first_line), INT2FIX(@3.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_1_components"), 2, $1, $3);
+  }
+
+;
+
+sequence:
+  
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_0_sequence"), 0);
+  }
+
+|
+  sequence ACTIONS 
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    rb_ivar_set(__actions, rb_intern("@sequence"), rb_ary_new3(2, INT2FIX(@1.first_line), INT2FIX(@1.first_column)));
+    rb_ivar_set(__actions, rb_intern("@code"), rb_ary_new3(2, INT2FIX(@2.first_line), INT2FIX(@2.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_1_sequence"), 2, $1, $2);
+  }
+
+|
+  sequence IDENTIFIER 
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    rb_ivar_set(__actions, rb_intern("@sequence"), rb_ary_new3(2, INT2FIX(@1.first_line), INT2FIX(@1.first_column)));
+    rb_ivar_set(__actions, rb_intern("@follower"), rb_ary_new3(2, INT2FIX(@2.first_line), INT2FIX(@2.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_2_sequence"), 2, $1, $2);
+  }
+
+|
+  sequence IDENTIFIER LBRACK IDENTIFIER RBRACK 
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    rb_ivar_set(__actions, rb_intern("@sequence"), rb_ary_new3(2, INT2FIX(@1.first_line), INT2FIX(@1.first_column)));
+    rb_ivar_set(__actions, rb_intern("@follower"), rb_ary_new3(2, INT2FIX(@2.first_line), INT2FIX(@2.first_column)));
+    rb_ivar_set(__actions, rb_intern("@tag"), rb_ary_new3(2, INT2FIX(@4.first_line), INT2FIX(@4.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_3_sequence"), 3, $1, $2, $4);
+  }
+
+|
+  sequence STRING 
+  {
+    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));
+    rb_ivar_set(__actions, rb_intern("@sequence"), rb_ary_new3(2, INT2FIX(@1.first_line), INT2FIX(@1.first_column)));
+    rb_ivar_set(__actions, rb_intern("@follower"), rb_ary_new3(2, INT2FIX(@2.first_line), INT2FIX(@2.first_column)));
+    $$ = rb_funcall(__actions, rb_intern("_4_sequence"), 2, $1, $2);
+  }
+
+;
+
+
+%%
+
+static VALUE cBisonParser;
+static VALUE cBisonParserTokens;
+static VALUE cBisonParserActions;
+
+static VALUE bison_parser_parse(VALUE);
+
+void Init_bison_parser(void) {
+  cBisonParser = rb_define_class("BisonParser", rb_cObject);
+  cBisonParserTokens = rb_define_module_under(cBisonParser, "Tokens");
+  cBisonParserActions = rb_define_class_under(cBisonParser, "Actions", rb_cObject);
+
+  rb_define_const(cBisonParserTokens, "IDENTIFIER", INT2FIX(IDENTIFIER));
+  rb_define_const(cBisonParserTokens, "NUMBER", INT2FIX(NUMBER));
+  rb_define_const(cBisonParserTokens, "STRING", INT2FIX(STRING));
+  rb_define_const(cBisonParserTokens, "COLON", INT2FIX(COLON));
+  rb_define_const(cBisonParserTokens, "SEMICOLON", INT2FIX(SEMICOLON));
+  rb_define_const(cBisonParserTokens, "LBRACK", INT2FIX(LBRACK));
+  rb_define_const(cBisonParserTokens, "RBRACK", INT2FIX(RBRACK));
+  rb_define_const(cBisonParserTokens, "PIPE", INT2FIX(PIPE));
+  rb_define_const(cBisonParserTokens, "HASH", INT2FIX(HASH));
+  rb_define_const(cBisonParserTokens, "DOUBLE_HASH", INT2FIX(DOUBLE_HASH));
+  rb_define_const(cBisonParserTokens, "KW_TOKEN", INT2FIX(KW_TOKEN));
+  rb_define_const(cBisonParserTokens, "KW_LEFT", INT2FIX(KW_LEFT));
+  rb_define_const(cBisonParserTokens, "KW_RIGHT", INT2FIX(KW_RIGHT));
+  rb_define_const(cBisonParserTokens, "ACTIONS", INT2FIX(ACTIONS));
+
+  rb_define_method(cBisonParser, "parse", bison_parser_parse, 0);
+}
+
+VALUE bison_parser_parse(VALUE self) {
+  VALUE actions = rb_funcall(cBisonParserActions, rb_intern("new"), 0);
+  rb_funcall(actions, rb_intern("parser="), 1, self);
+  if (yyparse(actions))
+    return Qnil;
+  return rb_funcall(actions, rb_intern("result"), 0);
+}
+
+static void yyerror(YYLTYPE *loc, VALUE actions, const char *msg) {
+  VALUE parser = rb_funcall(actions, rb_intern("parser"), 0);
+  rb_funcall(parser, rb_intern("error"), 3,
+             rb_str_new_cstr(msg), 
+             INT2FIX(loc->first_line), 
+             INT2FIX(loc->first_column));
+}
+
+static int yylex(YYSTYPE *lval, YYLTYPE *lloc, VALUE actions) {
+  int c;
+  VALUE parser, value, vtok;
+
+  parser = rb_funcall(actions, rb_intern("parser"), 0);
+
+  rb_funcall(parser, rb_intern("lex_value="), 1, Qnil);
+  rb_funcall(parser, rb_intern("token_row="), 1, INT2FIX(lloc->last_line));
+  rb_funcall(parser, rb_intern("token_col="), 1, INT2FIX(lloc->last_column));
+
+  vtok = rb_funcall(parser, rb_intern("lex"), 0);
+  value = rb_funcall(parser, rb_intern("lex_value"), 0);
+
+  lloc->first_line = FIX2INT(rb_funcall(parser, rb_intern("token_row"), 0));
+  lloc->first_column = FIX2INT(rb_funcall(parser, rb_intern("token_col"), 0));
+  lloc->last_line = FIX2INT(rb_funcall(parser, rb_intern("row"), 0));
+  lloc->last_column = FIX2INT(rb_funcall(parser, rb_intern("col"), 0));
+
+  if (vtok == Qnil) {
+    *lval = Qnil;
+    return 0;
+  }
+
+  if (vtok & 1) {
+    *lval = value;
+    return FIX2INT(vtok);
+  }
+
+  if (RBASIC(vtok)->klass == rb_cString) {
+    c = StringValueCStr(vtok)[0];
+    *lval = rb_sprintf("%c", c);
+    return c;
+  }
+
+  return 0;
+}

data/ext/bison_parser/extconf.rb

@@ -0,0 +1,9 @@
+require 'mkmf'
+
+output = "#{File.dirname(__FILE__)}/bison_parser.c"
+bison_file = "#{File.dirname(__FILE__)}/bison_parser.y"
+
+bison = ENV['BISON_PATH'] || 'bison'
+system(bison, '-o', output, bison_file)
+
+create_makefile 'bison_parser/bison_parser'

data/lib/bison.rb

@@ -0,0 +1,14 @@
+
+module Bison
+end
+
+require 'bison/version'
+require 'bison/grammar_file'
+require 'bison/token'
+require 'bison/rule'
+require 'bison/sequence'
+require 'bison/action'
+require 'bison/nonterminal'
+require 'bison/string'
+
+require 'bison_parser'

data/lib/bison/action.rb

@@ -0,0 +1,63 @@
+
+require 'digest'
+require 'tempfile'
+
+module Bison
+  class Action
+    attr_accessor :code
+    attr_accessor :location
+    attr_accessor :predecessors
+    attr_accessor :sequence
+
+    def initialize(code)
+      self.code = code
+    end
+
+    def to_bison
+      code = "\n  {\n"
+      code << %(    rb_ivar_set(__actions, rb_intern("@_"), rb_ary_new3(2, INT2FIX(@$.first_line), INT2FIX(@$.first_column)));\n)
+      predecessor_tags.each do |i, name|
+        code << %(    rb_ivar_set(__actions, rb_intern("@#{name}"), rb_ary_new3(2, INT2FIX(@#{i}.first_line), INT2FIX(@#{i}.first_column)));\n)
+      end
+      code << %(    $$ = #{funcall('__actions')};\n)
+      code << "  }\n"
+    end
+
+    def predecessor_tags
+      tags = predecessors.each_with_index.map do |e, i|
+        [i+1, e.tag] if (Bison::Nonterminal === e) && e.tag
+      end.compact
+      
+      Hash[tags]
+    end
+
+    def errors
+      tmp = Tempfile.new('action-src.rb').tap do |tmp|
+        location[0].times{ tmp.puts }
+        tmp.puts(code)
+        tmp.close
+      end
+
+      errors = `ruby -c "#{tmp.path}" 2>&1`
+      if $?.success?
+        return nil
+      else
+        return errors.gsub(tmp.path, '-')
+      end
+    end
+
+    def name
+      base = "_#{sequence.index}_#{sequence.rule.name}"
+#      base << "_#{Digest::MD5.hexdigest(predecessor_tags.inspect)}"
+    end
+
+    # What to do about default $1/Qnil?
+    def funcall(receiver)
+      method = "rb_intern(#{name.inspect})"
+      args = predecessor_tags.keys.map{ |i| "$#{i}" }.join(', ')
+      args = args.empty? ? '0' : "#{predecessor_tags.size}, #{args}"
+      "rb_funcall(#{receiver}, #{method}, #{args})"
+    end
+
+  end
+end

data/lib/bison/grammar_file.rb

@@ -0,0 +1,59 @@
+
+require 'erb'
+
+module Bison
+  class GrammarFile
+    attr_accessor :name
+    attr_reader :tokens, :rules, :code
+
+    def initialize(tokens, rules, code)
+      @tokens, @rules, @code = tokens, rules, code
+    end
+
+    def validate
+      errors = []
+      symbols = tokens.map(&:name) + rules.map(&:name)
+      rules.map(&:components).flatten.map(&:elements).flatten.each do |el|
+        if el.is_a?(Bison::Nonterminal) && !symbols.include?(el.name)
+          errors << "#{el.location.join('.')}: #{el.name} is not defined"
+        end
+      end
+      rules.map(&:components).flatten.each do |seq|
+        seq.elements.grep(Bison::Action).each do |action|
+          err = action.errors
+          errors << err unless err.nil?
+        end
+      end
+      abort(errors.join("\n")) unless errors.empty?
+    end
+
+    def print_class(out=$stdout)
+      template = File.expand_path('../../../templates/class.rb.erb', __FILE__)
+      out.puts(ERB.new(File.read(template), nil, '-').result(binding))
+    end
+
+    def print_base_module(out=$stdout)
+      template = File.expand_path('../../../templates/base.rb.erb', __FILE__)
+      out.puts(ERB.new(File.read(template), nil, '-').result(binding))
+    end
+
+    def print_actions_module(out=$stdout)
+      template = File.expand_path('../../../templates/actions.rb.erb', __FILE__)
+      out.puts(ERB.new(File.read(template), nil, '-').result(binding))
+    end
+
+    def print_bison(out=$stdout)
+      template = File.expand_path('../../../templates/parser.y.erb', __FILE__)
+      out.puts(ERB.new(File.read(template), nil, '-').result(binding))
+    end
+
+    def print_extconf(out=$stdout)
+      template = File.expand_path('../../../templates/extconf.rb.erb', __FILE__)
+      out.puts(ERB.new(File.read(template), nil, '-').result(binding))
+    end
+    
+    def uname
+      name.gsub(/([a-z])([A-Z])/, '\1_\2').downcase
+    end
+  end
+end

data/lib/bison/nonterminal.rb

@@ -0,0 +1,17 @@
+module Bison
+  class Nonterminal
+    attr_reader :name
+    attr_reader :tag
+    attr_accessor :location
+    attr_accessor :sequence
+
+    def initialize(name, tag=nil)
+      @name = name
+      @tag = tag
+    end
+
+    def to_bison
+      name
+    end
+  end
+end