astrapi 0.0.5

data/lib/lexer.rb

@@ -0,0 +1,152 @@
+require 'pp'
+require 'strscan'
+#require 'benchmark'
+
+class Token
+  attr_accessor :kind,:val,:pos
+  def initialize tab
+    @kind,@val,@pos=*tab
+  end
+
+  def is_a? kind
+    case kind
+    when Symbol
+      return @kind==kind
+    when Array
+      for sym in kind
+        return true if @kind==sym
+      end
+      return false
+    else
+      raise "wrong type during lookahead"
+    end
+  end
+end
+
+module Astrapi
+
+  class Lexer
+
+    # I like explicit things :
+
+    #................................................
+    IDENT             = /\AIDENT/
+    INT               = /\AINT/
+    FLOAT             = /\AFLOAT/
+    STRING            = /\ASTRING/
+    #................................................
+    MODULE            = /\Amodule$/
+    CLASS             = /\Aclass$/
+    END_              = /\Aend$/
+    ATTR              = /\Aattr$/
+    #................................................
+    NEWLINE            = /[\n]/
+    SPACE              = /[ \t]+/
+    #.............punctuation........................
+    LPAREN             = /\A\(/
+    RPAREN             = /\A\)/
+    COMMENT            = /\A\#(.*)/
+    #.............operators..........................
+
+    ARROW              = /\A\=\>/
+    LBRACK             = /\A\[/
+    RBRACK             = /\A\]/
+    LT                 = /\A</
+    # .............literals.........................
+    IDENTIFIER         = /\A[a-zA-Z]+[a-zA-Z_0-9]*/i
+
+
+    attr_accessor :suppress_comment
+
+    def initialize str=''
+      init(str)
+      @suppress_space=true
+      @suppress_comment=false
+    end
+
+    def init str
+      @ss=StringScanner.new(str)
+      @line=0
+    end
+
+    def tokenize str
+      @tokens=[]
+      init(str)
+      until @ss.eos?
+        @tokens << next_token()
+      end
+      return @tokens[0..-2]
+    end
+
+    #next token can detect spaces length
+    def next_token
+
+      if @ss.bol?
+        @line+=1
+        @old_pos=@ss.pos
+      end
+
+      position=[@line,@ss.pos-@old_pos+1]
+
+      return :eos if @ss.eos?
+
+      case
+      when text = @ss.scan(NEWLINE)
+        next_token()
+      when text = @ss.scan(SPACE)
+        next_token()
+      when text = @ss.scan(COMMENT)
+        next_token()
+      when text = @ss.scan(ARROW)
+        return Token.new [:arrow,text,position]
+      when text = @ss.scan(LT)
+        return Token.new [:lt,text,position]
+      when text = @ss.scan(LBRACK)
+        return Token.new [:lbrack,text,position]
+      when text = @ss.scan(RBRACK)
+        return Token.new [:rbrack,text,position]
+      when text = @ss.scan(IDENTIFIER)
+        case
+        when value = text.match(IDENT)
+          return Token.new [:IDENT,text,position]
+        when value = text.match(INT)
+          return Token.new [:INT,text,position]
+        when value = text.match(FLOAT)
+          return Token.new [:FLOAT,text,position]
+        when value = text.match(STRING)
+          return Token.new [:STRING,text,position]
+        when value = text.match(MODULE)
+          return Token.new [:module,text,position]
+        when value = text.match(CLASS)
+          return Token.new [:class,text,position]
+        when value = text.match(END_)
+          return Token.new [:end,text,position]
+        when value = text.match(ATTR)
+          return Token.new [:attr,text,position]
+        when value = text.match(LPAREN)
+          return Token.new [:lparen,text,position]
+        when value = text.match(RPAREN)
+          return Token.new [:rparen,text,position]
+        else
+          return Token.new [:identifier,text,position]
+        end
+      else
+        x = @ss.getch
+        return Token.new [x, x,position]
+      end
+    end
+  end
+end
+
+if $PROGRAM_NAME == __FILE__
+  str=IO.read(ARGV[0])
+  #str.downcase!
+  puts str
+  t1 = Time.now
+  lexer=Astrapi::Lexer.new
+  tokens=lexer.tokenize(str)
+  t2 = Time.now
+  pp tokens
+  puts "number of tokens : #{tokens.size}"
+  puts "tokenized in     : #{t2-t1} s"
+end

data/lib/parser.rb

@@ -0,0 +1,110 @@
+require 'pp'
+require_relative 'lexer'
+require_relative 'indent'
+require_relative 'ast'
+
+module Astrapi
+
+  class Parser
+    include Indent #helper methods mixed in
+
+    attr_accessor :lexer,:tokens
+
+    def initialize
+      @verbose=false
+      @lexer=Lexer.new
+    end
+
+    def parse filename
+      str=IO.read(filename)
+      @tokens=lexer.tokenize(str)
+      @tokens=@tokens.select{|tok| tok.kind!=:comment}
+      pp @tokens if @verbose
+      parseModule
+    end
+
+    def acceptIt
+      tok=tokens.shift
+      say "consuming #{tok.val} (#{tok.kind})"
+      tok
+    end
+
+    def showNext
+      tokens.first
+    end
+
+    def expect kind
+      if (actual=showNext.kind)!=kind
+        abort "ERROR at #{showNext.pos}. Expecting #{kind}. Got #{actual}"
+      else
+        return acceptIt()
+      end
+    end
+
+    def parseModule
+      indent "parseModule"
+      expect :module
+      name=Identifier.new(expect :identifier)
+      classes=[]
+      while showNext.is_a? :class
+        classes << parseClass
+      end
+      expect :end
+      dedent
+      return Astrapi::Module.new(name,classes)
+    end
+
+    def parseClass
+      indent "parseClass"
+      expect :class
+      name=Identifier.new(expect :identifier)
+      if showNext.is_a? :lt
+        acceptIt
+        inheritance= Identifier.new(expect :identifier)
+      end
+      attrs=[]
+      while showNext.is_a? :attr
+        attrs << parseAttr
+      end
+      expect :end
+      dedent
+      return Astrapi::Klass.new(name,inheritance,attrs)
+    end
+
+    def parseAttr
+      indent "parseAttr"
+      expect :attr
+      name=Identifier.new(expect :identifier)
+      expect :arrow
+      type=parseAttrType
+      dedent
+      Attr.new(name,type)
+    end
+
+    def parseAttrType
+      indent "parseAttrType"
+      if showNext.is_a? [:IDENT,:FLOAT,:INT]
+        type=Type.new(Identifier.new(acceptIt))
+      else
+        type=Type.new(Identifier.new(expect :identifier))
+      end
+      if showNext.is_a? :lbrack
+        acceptIt
+        expect :rbrack
+        type=ArrayOf.new(type)
+      end
+      dedent
+      return type
+    end
+
+  end
+end
+
+if $PROGRAM_NAME == __FILE__
+  filename=ARGV[0]
+  raise "need a file !" if filename.nil?
+  t1 = Time.now
+  Astrapi::Parser.new.parse(filename)
+  t2 = Time.now
+  puts "parsed in     : #{t2-t1} s"
+end

data/lib/pretty_printer.rb

@@ -0,0 +1,68 @@
+require_relative 'ast'
+require_relative 'indent'
+require_relative 'code'
+
+module Astrapi
+
+  class PrettyPrinter
+
+    include Indent
+
+    def print ast
+      ast.accept(self)
+    end
+
+    def visitModule modul,args=nil
+      indent "visitModule"
+      code=Code.new
+      code << "module #{name=modul.name}"
+      code.newline
+      code.indent=2
+      modul.classes.each{|k| code << k.accept(self) ; code.newline}
+      code.indent=0
+      code << "end"
+      dedent
+      return code
+    end
+
+    def visitKlass klass,args=nil
+      indent "visitKlass"
+      code = Code.new
+      inherit="< #{klass.inheritance}" if klass.inheritance
+      code << "class #{klass.name} #{inherit}"
+      code.indent=2
+      klass.attrs.each{|attr| code << attr.accept(self)}
+      code.indent=0
+      code << "end"
+      dedent
+      code
+    end
+
+    def visitAttr attr,args=nil
+      indent "visitAttr"
+      type=attr.type.accept(self)
+      str="attr #{attr.name} => #{type}"
+      dedent
+      str
+    end
+
+    def visitType type,args=nil
+      indent "visitType"
+      dedent
+      return type.name.to_s
+    end
+
+    def visitArrayOf arrayOf,args=nil
+      indent "visitArrayOf"
+      str="#{arrayOf.type.name}[]"
+      dedent
+      return str
+    end
+
+    def visitIdentifier id,args=nil
+      indent "visitIdentifier"
+      dedent
+    end
+
+  end
+end

data/lib/ruby_generator.rb

@@ -0,0 +1,329 @@
+require 'erb'
+require_relative 'ast'
+
+module Astrapi
+
+  class RubyGenerator
+
+    attr_accessor :mm
+
+    #Astrapi basic types :
+    BASIC_TYPES = [:integer,:float,:range,:ident,:string]
+
+    def generate mm
+      @mm=mm
+      @path=__dir__ #ruby > 2.0
+      @path+="/"
+      generate_misc
+      generate_ast
+      generate_dot_ast_printer
+      generate_sexp_lexer
+      generate_sexp_parser
+      generate_sexp_pretty_printer
+      generate_visitor
+      generate_dsl_compiler
+    end
+
+    def generate_ast
+      puts "----> generating #{mm.name} DSL AST classes"
+      mm_name=mm.name
+      code = Code.new
+      code << "# code generated automatically by M3E"
+      code << "module #{mm_name}"
+      code.newline
+      code.indent=2
+      code << "class Ast"
+      code.indent=4
+      code << "attr_accessor :comments_"
+      code << "def accept(visitor, arg=nil)"
+      code << "  name = self.class.name.split(/::/)[1]"
+      code << "  visitor.send(\"visit\#{name}\".to_sym, self ,arg) # Metaprograming !"
+      code << "end"
+      code.newline
+      code.indent=2
+      code << "end"
+      code.newline
+      code.indent=2
+      code << "class Comment_ < Ast"
+      code.indent=4
+      code << "attr_accessor :list"
+      code.indent=2
+      code << "end"
+      code.newline
+      code.indent=2
+
+      mm.classes.each{|k|
+        code << code_for(k)
+        code.newline
+      }
+      code.indent=0
+      code << "end"
+      code.save_as("#{mm_name.to_s.downcase}_ast.rb",false)
+    end
+
+    def code_for klass
+      code=Code.new
+      mother = klass.inheritance
+      inheritance="< #{klass.inheritance}" if mother
+      inheritance ||= "< Ast"
+      code << "class #{klass.name} #{inheritance}"
+      code.indent=2
+      code << "attr_accessor #{klass.attrs.collect{|atr| ':'+atr.name.to_s}.join(',')}"
+      code << "def initialize"
+      code.indent=4
+      klass.attrs.each do |atr|
+        init=(atr.type.is_a? ArrayOf) ? "[]" : "nil"
+        code << "@#{atr.name} = #{init}"
+      end
+      code.indent=2
+      code << "end"
+      code.indent=0
+      code << "end"
+      code
+    end
+
+    def generate_sexp_lexer
+      puts "----> generating #{mm.name} DSL lexer"
+      keywords=mm.classes.collect{|k| k.name.to_s}.map{|k| k.downcase.to_sym}
+      regexp_code=Code.new #to be inserted in ERB template
+      regexp_code.indent=4
+      keywords.each do |kw|
+        regexp_code << "#{kw.upcase.to_s.ljust(30)} = /\\A#{kw}/"
+      end
+      keywords_regexp=regexp_code.finalize
+      #.......
+      regexp_code=Code.new
+      regexp_code.indent=8
+      keywords.each do |kw|
+        regexp_code << "when value = text.match(#{kw.upcase})"
+        regexp_code << "  return Token.new [:#{kw},text,position]"
+      end
+      apply_regexp=regexp_code.finalize
+      #...... call templating engine
+      engine = ERB.new(IO.read(@path+"template_lexer.rb"))
+      lexer_rb_code=engine.result(binding)
+      File.open("#{mm.name.to_s.downcase}_lexer.rb",'w'){|f| f.puts lexer_rb_code}
+    end
+
+    def generate_sexp_parser
+      puts "----> generating #{mm.name} DSL parser"
+      code=Code.new
+      code.indent=4
+      mm.classes.each do |klass|
+        kname=klass.name.to_s
+        v=kname.downcase
+        code.newline
+        code << "def parse#{kname}"
+        code.indent=6
+        code << "indent \"#{klass.name}\""
+        code << "if n=nil_maybe?"
+        code << "  return n"
+        code << "end"
+        code << "#{v}_ = #{mm.name}::#{kname}.new"
+        code << "#{v}_.comments_=parse_comments()"
+        code << "expect :lparen"
+        code << "expect :#{v}"
+        code << core_parsing_for(klass)
+        code << "expect :rparen"
+        code << "dedent"
+        code << "return #{v}_"
+        code.indent=4
+        code << "end"
+      end
+      parsing_methods=code.finalize
+      #...... call templating engine
+      engine = ERB.new(IO.read(@path+"template_parser.rb"))
+      parser_rb_code=engine.result(binding)
+      File.open("#{mm.name.to_s.downcase}_parser.rb",'w'){|f| f.puts parser_rb_code}
+    end
+
+    def generate_sexp_pretty_printer
+      puts "----> generating #{mm.name} DSL pretty printer"
+      code=Code.new
+      code.indent=4
+      mm.classes.each do |klass|
+        kname=klass.name.to_s
+        v=kname.downcase
+        code.newline
+        code << "def visit#{kname}(#{v}_,args=nil)"
+        code.indent=6
+        code << "indent \"#{klass.name}\""
+        code << "ary=[]"
+        code << "ary << #{v}_.comments_.val if #{v}_.comments_"
+        code << "ary << #{v}_.class.to_s.downcase.split('::')[1].to_sym"
+        attrs = hierarchical_attrs(klass) + klass.attrs #note the order!
+        code_for_attrs=attrs.collect{|atr|
+          if atr.type.is_a? ArrayOf
+            e=((name=atr.name.to_s).end_with?('s')) ? name[0..-2] : "e"
+            code << "#{v}_.#{atr.name}.each do |#{e}|"
+            code.indent=8
+            code << "ary << #{e}.accept(self,args)"
+            code.indent=6
+            code << "end"
+          else
+            code << "ary << #{v}_.#{atr.name}.accept(self)"
+          end
+        }
+        code << "dedent"
+        code << "return ary"
+        code.indent=4
+        code << "end"
+      end
+      visiting_methods=code.finalize
+      #...... call templating engine
+      engine = ERB.new(IO.read(@path+"template_pretty_printer.rb"))
+      parser_rb_code=engine.result(binding)
+      File.open("#{mm.name.to_s.downcase}_pp.rb",'w'){|f|
+        f.puts parser_rb_code
+      }
+    end
+
+    def generate_visitor
+      puts "----> generating #{mm.name} DSL visitor"
+      code=Code.new
+      code.indent=4
+      mm.classes.each do |klass|
+        kname=klass.name.to_s
+        v=kname.downcase
+        code.newline
+        code << "def visit#{kname}(#{v}_,args=nil)"
+        code.indent=6
+        code << "indent \"#{klass.name}\""
+        attrs = hierarchical_attrs(klass) + klass.attrs #note the order!
+        code_for_attrs=attrs.collect{|atr|
+          if atr.type.is_a? ArrayOf
+            e=((name=atr.name.to_s).end_with?('s')) ? name[0..-2] : "e"
+            code << "#{v}_.#{atr.name}.each do |#{e}|"
+            code.indent=8
+            code << "#{e}.accept(self,args)"
+            code.indent=6
+            code << "end"
+          else
+            code << "#{v}_.#{atr.name}.accept(self)"
+          end
+        }
+        code << "dedent"
+        code.indent=4
+        code << "end"
+      end
+      visiting_methods=code.finalize
+      #...... call templating engine
+      engine = ERB.new(IO.read(@path+"template_visitor.rb"))
+      parser_rb_code=engine.result(binding)
+      File.open("#{mm.name.to_s.downcase}_visitor.rb",'w'){|f|
+        f.puts parser_rb_code
+      }
+    end
+
+    def generate_dot_ast_printer
+      puts "----> generating #{mm.name} DSL AST printer"
+      #...... call templating engine
+      engine = ERB.new(IO.read(@path+"template_ast_printer.rb"))
+      printer_rb_code=engine.result(binding)
+      File.open("#{mm.name.to_s.downcase}_ast_printer.rb",'w'){|f| f.puts printer_rb_code}
+    end
+
+    def generate_dsl_compiler
+      puts "----> generating #{mm.name} DSL compiler"
+      #...... call templating engine
+      engine = ERB.new(IO.read(@path+"template_compiler.rb"))
+      code=engine.result(binding)
+      File.open("#{mm.name.to_s.downcase}_compiler.rb",'w'){|f| f.puts code}
+    end
+
+    def generate_misc
+      engine = ERB.new(IO.read(@path+"template_indent.rb"))
+      code=engine.result(binding)
+      File.open("indent.rb",'w'){|f| f.puts code}
+      engine = ERB.new(IO.read(@path+"template_code.rb"))
+      code=engine.result(binding)
+      File.open("code.rb",'w'){|f| f.puts code}
+    end
+
+    def core_parsing_for klass
+      kname=klass.name.to_s
+      #puts "#{kname}".center(80,'=')
+      v=kname.downcase
+      attrs = hierarchical_attrs(klass) + klass.attrs #note the order!
+      code=Code.new
+      attrs.each{|attr| code << parsing_code_for(attr,v)}
+      return code
+    end
+
+    def hierarchical_attrs klass
+      inherited_attrs=[]
+      if mother=find_mother_of(klass)
+        inherited_attrs << mother.attrs
+        inherited_attrs << hierarchical_attrs(mother)
+      end
+      return inherited_attrs.flatten
+    end
+
+    def find_mother_of klass
+      if id=klass.inheritance #identifier
+        return mother=mm.classes.find{|k| k.name==id}
+      end
+      nil
+    end
+
+    def find_sons_of klass
+      ret=mm.classes.select{|k| k.inheritance==klass.name}
+      ret
+    end
+
+    def parsing_code_for attr,ast_node_var
+      code=Code.new
+      case type=attr.type
+      when ArrayOf #note the position, before 'when Type' !
+        possible_starters=compute_possible_starters(type.type)
+        code << "comments=parse_comments()"
+        lookahead=possible_starters.first.to_s.end_with?("_lit") ? 0 : 1
+        starters=possible_starters.collect{|sym| ":#{sym}"}.join(',')
+        code << "while showNext(#{lookahead}) && showNext(#{lookahead}).is_a?([#{starters}])"
+        code.indent=2
+        code << "case showNext(#{lookahead}).kind"
+        possible_starters.each do |starter|
+          code << "when :#{starter}"
+          code.indent=4
+          if starter.to_s.end_with?("_lit") #base types
+            code << "#{ast_node_var}_.#{attr.name} << acceptIt"
+          else
+            code << "#{ast_node_var}_.#{attr.name} << parse#{starter.capitalize}()"
+          end
+          #code << "#{ast_node_var}_.#{attr.name} << parse#{starter.capitalize}()"
+          code.indent=2
+        end
+        code << "else "
+        code << "  abort \"ERROR when parsing attribute #{attr.name} : expecting one of [#{starters}].\n Got \#{showNext(1).kind}\""
+        code << "end"
+        code << "if #{ast_node_var}_.#{attr.name}.last.respond_to? :comments_"
+        code.indent=4
+        code << "#{ast_node_var}_.#{attr.name}.last.comments_=comments"
+        code.indent=2
+        code << "end"
+        code << "comments=parse_comments()"
+        code.indent=0
+        code << "end"
+      when Type
+        if is_basic_type(type) #base types
+          code << "#{ast_node_var}_.#{attr.name} = acceptIt"
+        else
+          code << "#{ast_node_var}_.#{attr.name} = parse#{type.name}()"
+        end
+      end
+      return code
+    end
+
+    def is_basic_type type
+      type.name.to_s.upcase==type.name.to_s
+    end
+
+    def compute_possible_starters type
+      ret=[]
+      sons=find_sons_of(type)
+      starters= (sons << type).flatten
+      ret=starters.collect{|starter| starter.name.to_s.downcase.to_sym}
+      ret=ret.collect{|sym| (BASIC_TYPES.include? sym) ? (sym.to_s+"_lit").to_sym : sym}
+    end
+  end
+end