iv-phonic 0.0.3 → 0.0.5

data/ext/include/iv/location.h

@@ -15,6 +15,12 @@ struct Location {
   inline std::size_t end_position() const {
     return end_position_;
   }
+  inline void set_begin_position(std::size_t begin) {
+    begin_position_ = begin;
+  }
+  inline void set_end_position(std::size_t end) {
+    end_position_ = end;
+  }
 };
 
 IV_STATIC_ASSERT(std::tr1::is_pod<Location>::value);

data/ext/include/iv/parser.h

@@ -9,10 +9,10 @@
 #include "static_assert.h"
 #include "ast.h"
 #include "ast-factory.h"
-#include "source.h"
 #include "lexer.h"
 #include "noncopyable.h"
 #include "utils.h"
+#include "ustring.h"
 #include "none.h"
 
 #define IS(token)\
@@ -42,6 +42,15 @@
   } while (0)
 
 #define RAISE(str)\
+  do {\
+    *res = false;\
+    error_state_ |= kNotRecoverable;\
+    SetErrorHeader(lexer_.line_number());\
+    error_.append(str);\
+    return NULL;\
+  } while (0)
+
+#define RAISE_RECOVERVABLE(str)\
   do {\
     *res = false;\
     SetErrorHeader(lexer_.line_number());\
@@ -52,6 +61,7 @@
 #define RAISE_WITH_NUMBER(str, line)\
   do {\
     *res = false;\
+    error_state_ |= kNotRecoverable;\
     SetErrorHeader(line);\
     error_.append(str);\
     return NULL;\
@@ -124,11 +134,12 @@ const UString ParserData<T>::kSet(
 
 typedef detail::ParserData<None> ParserData;
 
-template<typename Factory>
-class Parser : private Noncopyable<Parser<Factory> >::type {
+template<typename Factory, typename Source>
+class Parser : private Noncopyable<Parser<Factory, Source> >::type {
  public:
-  typedef Parser<Factory> this_type;
-  typedef Parser<Factory> parser_type;
+  typedef Parser<Factory, Source> this_type;
+  typedef Parser<Factory, Source> parser_type;
+  typedef Lexer<Source> lexer_type;
 #define V(AST) typedef typename ast::AST<Factory> AST;
   AST_NODE_LIST(V)
 #undef V
@@ -138,6 +149,11 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
 #define V(S) typedef typename SpaceUString<Factory>::type S;
   AST_STRING(V)
 #undef V
+
+  enum ErrorState {
+    kNotRecoverable = 1
+  };
+
   class Target : private Noncopyable<Target>::type {
    public:
     typedef typename SpaceVector<Factory, Identifier*>::type Identifiers;
@@ -174,7 +190,9 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
       return !IsAnonymous();
     }
     inline BreakableStatement** node() {
-      node_ = new(parser_->factory())BreakableStatement*();
+      if (!node_) {
+        node_ = parser_->factory()->template NewPtr<BreakableStatement>();
+      }
       return node_;
     }
     inline Identifiers* labels() const {
@@ -193,10 +211,30 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
     int type_;
   };
 
-  Parser(BasicSource* source, Factory* space)
+  class TargetScope : private Noncopyable<Target>::type {
+   public:
+    TargetScope(parser_type* parser)
+      : parser_(parser),
+        target_(parser->target()),
+        labels_(parser->labels()) {
+      parser_->set_target(NULL);
+      parser_->set_labels(NULL);
+    }
+    ~TargetScope() {
+      parser_->set_target(target_);
+      parser_->set_labels(labels_);
+    }
+   private:
+    parser_type* parser_;
+    Target* target_;
+    Identifiers* labels_;
+  };
+
+  Parser(Factory* space, const Source* source)
     : lexer_(source),
       error_(),
       strict_(false),
+      error_state_(0),
       factory_(space),
       scope_(NULL),
       target_(NULL),
@@ -206,8 +244,9 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
 // Program
 //   : SourceElements
   FunctionLiteral* ParseProgram() {
-    FunctionLiteral* global = factory_->NewFunctionLiteral(
+    FunctionLiteral* const global = factory_->NewFunctionLiteral(
         FunctionLiteral::GLOBAL);
+    global->set_strict(strict_);
     assert(target_ == NULL);
     bool error_flag = true;
     bool *res = &error_flag;
@@ -402,7 +441,7 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
         FunctionLiteral::GENERAL, CHECK);
     // define named function as FunctionDeclaration
     scope_->AddFunctionDeclaration(expr);
-    return factory_->NewFunctionStatement(expr);
+    return factory_->NewFunctionDeclaration(expr);
   }
 
 //  Block
@@ -552,7 +591,14 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
 
     EXPECT(Token::RPAREN);
 
-    ExpectSemicolon(CHECK);
+    // ex:
+    //   do {
+    //     print("valid syntax");
+    //   } while (0) return true;
+    // is syntax valid
+    if (token_ == Token::SEMICOLON) {
+      Next();
+    }
     DoWhileStatement* const dowhile = factory_->NewDoWhileStatement(stmt, expr);
     target.set_node(dowhile);
     return dowhile;
@@ -845,6 +891,7 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
     Next();
     // Throw requires Expression
     if (lexer_.has_line_terminator_before_next()) {
+      // TODO(Constellation) more refined parse error system
       RAISE("missing expression between throw and newline");
     }
     Expression* const expr = ParseExpression(true, CHECK);
@@ -1410,7 +1457,7 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
 //    | MemberExpression '[' Expression ']'
 //    | NEW MemberExpression Arguments
   Expression* ParseMemberExpression(bool allow_call, bool *res) {
-    Expression *expr;
+    Expression* expr;
     if (token_ != Token::NEW) {
       if (token_ == Token::FUNCTION) {
         // FunctionExpression
@@ -1421,8 +1468,8 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
       }
     } else {
       Next();
-      Expression *target = ParseMemberExpression(false, CHECK);
-      ConstructorCall *con = factory_->NewConstructorCall(target);
+      Expression* const target = ParseMemberExpression(false, CHECK);
+      ConstructorCall* const con = factory_->NewConstructorCall(target);
       if (token_ == Token::LPAREN) {
         ParseArguments(con, CHECK);
       }
@@ -1432,14 +1479,14 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
       switch (token_) {
         case Token::LBRACK: {
           Next();
-          Expression* index = ParseExpression(true, CHECK);
+          Expression* const index = ParseExpression(true, CHECK);
           expr = factory_->NewIndexAccess(expr, index);
           EXPECT(Token::RBRACK);
           break;
         }
 
         case Token::PERIOD: {
-          Next(Lexer::kIgnoreReservedWords);  // IDENTIFIERNAME
+          Next<IgnoreReservedWords>();  // IDENTIFIERNAME
           IS(Token::IDENTIFIER);
           Identifier* const ident = ParseIdentifier(lexer_.Buffer());
           expr = factory_->NewIdentifierAccess(expr, ident);
@@ -1507,7 +1554,7 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
       case Token::NUMBER:
         // section 7.8.3
         // strict mode forbids Octal Digits Literal
-        if (strict_ && lexer_.NumericType() == Lexer::OCTAL) {
+        if (strict_ && lexer_.NumericType() == lexer_type::OCTAL) {
           RAISE("octal integer literal not allowed in strict code");
         }
         result = factory_->NewNumberLiteral(lexer_.Numeric());
@@ -1515,11 +1562,11 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
         break;
 
       case Token::STRING: {
-        const Lexer::State state = lexer_.StringEscapeType();
-        if (strict_ && state == Lexer::OCTAL) {
+        const typename lexer_type::State state = lexer_.StringEscapeType();
+        if (strict_ && state == lexer_type::OCTAL) {
           RAISE("octal excape sequence not allowed in strict code");
         }
-        if (state == Lexer::NONE) {
+        if (state == lexer_type::NONE) {
           result = factory_->NewDirectivable(lexer_.Buffer());
         } else {
           result = factory_->NewStringLiteral(lexer_.Buffer());
@@ -1551,7 +1598,7 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
         break;
 
       default:
-        RAISE("invalid primary expression token");
+        UNEXPECT(token_);
         break;
     }
     return result;
@@ -1564,11 +1611,11 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
 //  ArgumentList
 //    : AssignmentExpression
 //    | ArgumentList ',' AssignmentExpression
-  Call* ParseArguments(Call* func, bool *res) {
-    Expression* expr;
+  template<typename Callable>
+  Callable* ParseArguments(Callable* func, bool *res) {
     Next();
     while (token_ != Token::RPAREN) {
-      expr = ParseAssignmentExpression(true, CHECK);
+      Expression* const expr = ParseAssignmentExpression(true, CHECK);
       func->AddArgument(expr);
       if (token_ != Token::RPAREN) {
         EXPECT(Token::COMMA);
@@ -1580,15 +1627,14 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
 
   Expression* ParseRegExpLiteral(bool contains_eq, bool *res) {
     if (lexer_.ScanRegExpLiteral(contains_eq)) {
-      RegExpLiteral* expr;
       const std::vector<uc16> content(lexer_.Buffer());
       if (!lexer_.ScanRegExpFlags()) {
         RAISE("invalid regular expression flag");
-      } else {
-        expr = factory_->NewRegExpLiteral(content, lexer_.Buffer());
-        if (!expr) {
-          RAISE("invalid regular expression");
-        }
+      }
+      RegExpLiteral* const expr =
+          factory_->NewRegExpLiteral(content, lexer_.Buffer());
+      if (!expr) {
+        RAISE("invalid regular expression");
       }
       Next();
       return expr;
@@ -1662,12 +1708,12 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
     Identifier* ident;
 
     // IDENTIFIERNAME
-    Next(Lexer::kIgnoreReservedWordsAndIdentifyGetterOrSetter);
+    Next<IgnoreReservedWordsAndIdentifyGetterOrSetter>();
     while (token_ != Token::RBRACE) {
       if (token_ == Token::GET || token_ == Token::SET) {
         const bool is_get = token_ == Token::GET;
         // this is getter or setter or usual prop
-        Next(Lexer::kIgnoreReservedWords);  // IDENTIFIERNAME
+        Next<IgnoreReservedWords>();  // IDENTIFIERNAME
         if (token_ == Token::COLON) {
           // property
           ident = ParseIdentifier(
@@ -1720,13 +1766,17 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
               it->second |= type;
             }
           } else {
-            RAISE("invalid property name");
+            RAISE_RECOVERVABLE("invalid property name");
           }
         }
       } else if (token_ == Token::IDENTIFIER ||
                  token_ == Token::STRING ||
                  token_ == Token::NUMBER) {
-        ident = ParseIdentifier(lexer_.Buffer());
+        if (token_ == Token::NUMBER) {
+          ident = ParseIdentifier(lexer_.Buffer8());
+        } else {
+          ident = ParseIdentifier(lexer_.Buffer());
+        }
         EXPECT(Token::COLON);
         expr = ParseAssignmentExpression(true, CHECK);
         object->AddDataProperty(ident, expr);
@@ -1745,13 +1795,13 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
           }
         }
       } else {
-        RAISE("invalid property name");
+        RAISE_RECOVERVABLE("invalid property name");
       }
 
       if (token_ != Token::RBRACE) {
         IS(Token::COMMA);
         // IDENTIFIERNAME
-        Next(Lexer::kIgnoreReservedWordsAndIdentifyGetterOrSetter);
+        Next<IgnoreReservedWordsAndIdentifyGetterOrSetter>();
       }
     }
     Next();
@@ -1797,7 +1847,8 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
     }
 
     const ScopeSwitcher switcher(this, literal->scope());
-    literal->set_start_position(lexer_.pos() - 2);
+    const TargetScope scope(this);
+    literal->set_start_position(lexer_.begin_position());
 
     //  '(' FormalParameterList_opt ')'
     EXPECT(Token::LPAREN);
@@ -1886,8 +1937,7 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
           break;
       }
     }
-    literal->set_end_position(lexer_.pos() - 2);
-    literal->SubStringSource(lexer_.source());
+    literal->set_end_position(lexer_.end_position());
     Next();
     return literal;
   }
@@ -2028,12 +2078,15 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
     UNEXPECT(token_);
   }
 
-  inline Lexer& lexer() {
+  inline lexer_type& lexer() {
     return lexer_;
   }
-  inline Token::Type Next(Lexer::LexType type = Lexer::kIdentifyReservedWords) {
-    return token_ = lexer_.Next(
-        type | (strict_ ? Lexer::kStrict : Lexer::kClear));
+  template<typename LexType>
+  inline Token::Type Next() {
+    return token_ = lexer_.Next<LexType>(strict_);
+  }
+  inline Token::Type Next() {
+    return token_ = lexer_.Next<IdentifyReservedWords>(strict_);
   }
   inline Token::Type Peek() const {
     return token_;
@@ -2068,6 +2121,9 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
   inline void set_strict(bool strict) {
     strict_ = strict;
   }
+  inline bool RecoverableError() const {
+    return (!(error_state_ & kNotRecoverable)) && token_ == Token::EOS;
+  }
 
  protected:
   class ScopeSwitcher : private Noncopyable<ScopeSwitcher>::type {
@@ -2137,10 +2193,11 @@ class Parser : private Noncopyable<Parser<Factory> >::type {
     }
   }
 
-  Lexer lexer_;
+  lexer_type lexer_;
   Token::Type token_;
   std::string error_;
   bool strict_;
+  int error_state_;
   Factory* factory_;
   Scope* scope_;
   Target* target_;

data/ext/include/iv/token.h

@@ -83,6 +83,7 @@ class Token {
     ASSIGN_SHL,      // <<=
     ASSIGN_BIT_AND,  // &=
     ASSIGN_BIT_OR,   // |=
+    ASSIGN_BIT_XOR,  // ^=
     ASSIGN_LAST,     // ASSIGN OP LAST
 
     DELETE,          // delete
@@ -237,6 +238,7 @@ const char* TokenContents<T>::kContents[Token::NUM_TOKENS] = {
   "<<=",
   "&=",
   "|=",
+  "~=",
   NULL,            // ASSIGN OP LAST
   "delete",
   "typeof",

data/ext/iv/phonic/ast-fwd.h

@@ -4,8 +4,24 @@
 #include <iv/ast.h>
 namespace iv {
 namespace phonic {
-
 class AstFactory;
+}  // namespace iv::phonic
+namespace core {
+namespace ast {
+template<>
+class AstNodeBase<iv::phonic::AstFactory>
+  : public Inherit<iv::phonic::AstFactory, kAstNode> {
+ public:
+  void Location(std::size_t begin, std::size_t end) {
+    begin_ = begin;
+    end_ = end;
+  }
+ private:
+  std::size_t begin_;
+  std::size_t end_;
+};
+} }  // namespace iv::core::ast
+namespace phonic {
 #define V(AST) typedef core::ast::AST<AstFactory> AST;
   AST_NODE_LIST(V)
 #undef V

data/ext/iv/phonic/creator.h

@@ -45,6 +45,14 @@ class Creator : public iv::core::ast::AstVisitor<AstFactory>::const_type {
     ret_ = hash;
   }
 
+  void Visit(const FunctionDeclaration* decl) {
+    VALUE hash = rb_hash_new();
+    rb_hash_aset(hash, SYM("type"), rb_str_new_cstr("FunctionDeclaration"));
+    Visit(decl->function());
+    rb_hash_aset(hash, SYM("body"), ret_);
+    ret_ = hash;
+  }
+
   void Visit(const VariableStatement* stmt) {
     VALUE hash = rb_hash_new();
     rb_hash_aset(hash, SYM("type"), rb_str_new_cstr("VariableStatement"));

data/ext/iv/phonic/factory.h

@@ -6,7 +6,7 @@
 #include <iv/alloc.h>
 #include <iv/ustringpiece.h>
 #include "encoding.h"
-#include "ast.h"
+#include "ast-fwd.h"
 namespace iv {
 namespace phonic {
 
@@ -77,6 +77,11 @@ class AstFactory : public core::Space<2> {
         Identifiers(Identifiers::allocator_type(this));
   }
 
+  template<typename T>
+  T** NewPtr() {
+    return new (New(sizeof(T*))) T*;
+  }
+
   NullLiteral* NewNullLiteral() {
     return null_instance_;
   }
@@ -109,6 +114,10 @@ class AstFactory : public core::Space<2> {
     return new (this) FunctionStatement(func);
   }
 
+  FunctionDeclaration* NewFunctionDeclaration(FunctionLiteral* func) {
+    return new (this) FunctionDeclaration(func);
+  }
+
   Block* NewBlock() {
     return new (this) Block(this);
   }

data/ext/iv/phonic/parser.h

@@ -3,10 +3,11 @@
 #include <ruby.h>
 #include <iv/parser.h>
 #include "factory.h"
+#include "source.h"
 namespace iv {
 namespace phonic {
 
-typedef core::Parser<AstFactory> Parser;
+typedef core::Parser<AstFactory, UTF16Source> Parser;
 
 } }  // namespace iv::phonic
 #endif  // _IV_PHONIC_PARSER_H_

data/ext/iv/phonic/rparser.h

@@ -2,7 +2,6 @@
 #define _IV_PHONIC_RPARSER_H_
 #include <cstdlib>
 #include <cstring>
-#include <tr1/type_traits>
 #include <ruby.h>
 #include <ruby/encoding.h>
 #include <ruby/intern.h>
@@ -27,10 +26,10 @@ class RParser {
                                          0,
                                          Qnil);
     const char* str = StringValuePtr(encoded_rb_str);
-    std::size_t len = RSTRING_LEN(encoded_rb_str);
+    const std::size_t len = RSTRING_LEN(encoded_rb_str);
     UTF16Source source(str, len);
-    Parser parser(&source, &factory);
-    FunctionLiteral* global = parser.ParseProgram();
+    Parser parser(&factory, &source);
+    const FunctionLiteral* global = parser.ParseProgram();
     if (!global) {
       rb_raise(cParseError, "%s", parser.error().c_str());
     } else {

data/ext/iv/phonic/source.h

@@ -5,7 +5,6 @@
 #include <string>
 #include <tr1/cstdint>
 #include <ruby.h>
-#include <iv/source.h>
 #include <iv/stringpiece.h>
 #include <iv/ustring.h>
 #include <iv/none.h>
@@ -25,7 +24,7 @@ const std::string FilenameData<T>::kFilename = "<anonymous>";
 
 typedef detail::FilenameData<core::None> FilenameData;
 
-class AsciiSource : public core::BasicSource {
+class AsciiSource {
  public:
   static const int kEOS = -1;
   explicit AsciiSource(const char* str)
@@ -52,7 +51,7 @@ class AsciiSource : public core::BasicSource {
   std::string source_;
 };
 
-class UTF16Source : public core::BasicSource {
+class UTF16Source {
  public:
   static const int kEOS = -1;
   explicit UTF16Source(const char* str, std::size_t len)
@@ -82,7 +81,7 @@ class UTF16Source : public core::BasicSource {
   std::size_t size_;
 };
 
-class UTF16LESource : public core::BasicSource {
+class UTF16LESource {
  public:
   static const int kEOS = -1;
   explicit UTF16LESource(const char* str, std::size_t len)
@@ -112,7 +111,7 @@ class UTF16LESource : public core::BasicSource {
   std::size_t size_;
 };
 
-class UTF16BESource : public core::BasicSource {
+class UTF16BESource {
  public:
   static const int kEOS = -1;
   explicit UTF16BESource(const char* str, std::size_t len)

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 0
-  - 3
-  version: 0.0.3
+  - 5
+  version: 0.0.5
 platform: ruby
 authors: 
 - Constellation
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-10 00:00:00 +09:00
+date: 2010-11-28 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -93,8 +93,8 @@ files:
 - ext/include/iv/ast-visitor.h
 - ext/include/iv/token.h
 - ext/include/iv/enable_if.h
-- ext/include/iv/source.h
 - ext/include/iv/any.h
+- ext/include/iv/keyword.h
 - ext/include/iv/cmdline.h
 - ext/include/iv/xorshift.h
 - ext/include/iv/location.h
@@ -103,14 +103,13 @@ files:
 - ext/include/iv/ustring.h
 - ext/include/iv/ast-fwd.h
 - ext/include/iv/utils.h
+- ext/include/iv/ast-info.h
 - ext/include/iv/alloc.h
 - ext/include/iv/chars.h
 - ext/include/iv/fixedcontainer.h
 - ext/include/iv/ustringpiece.h
 - ext/include/iv/ast-serializer.h
 - ext/include/iv/ucdata.h
-- ext/iv/phonic/rnode.h
-- ext/iv/phonic/ast.h
 - ext/iv/phonic/factory.h
 - ext/iv/phonic/parser.h
 - ext/iv/phonic/encoding.h

data/ext/include/iv/source.h

@@ -1,27 +0,0 @@
-#ifndef _IV_SOURCE_H_
-#define _IV_SOURCE_H_
-#include <cstddef>
-#include <cassert>
-#include <string>
-#include "ustring.h"
-#include "ustringpiece.h"
-
-namespace iv {
-namespace core {
-
-class BasicSource {
- public:
-  static const int kEOS = -1;
-  virtual ~BasicSource() = 0;
-
-  virtual uc16 Get(std::size_t pos) const = 0;
-  virtual std::size_t size() const = 0;
-  virtual const std::string& filename() const = 0;
-  virtual UStringPiece SubString(std::size_t n,
-                                 std::size_t len = std::string::npos) const = 0;
-};
-
-inline BasicSource::~BasicSource() { }
-
-} }  // namespace iv::core
-#endif  // _IV_SOURCE_H_

data/ext/iv/phonic/ast.h

@@ -1,10 +0,0 @@
-#ifndef _IV_PHONIC_AST_H_
-#define _IV_PHONIC_AST_H_
-#include <ruby.h>
-#include <iv/ast.h>
-#include "ast-fwd.h"
-namespace iv {
-namespace phonic {
-
-} }  // namespace iv::phonic
-#endif  // _IV_PHONIC_AST_H_

data/ext/iv/phonic/rnode.h

@@ -1,15 +0,0 @@
-#ifndef _IV_PHONIC_RNODE_H_
-#define _IV_PHONIC_RNODE_H_
-#include <cstdlib>
-#include <cstring>
-#include <tr1/type_traits>
-#include <ruby.h>
-#include <ruby/encoding.h>
-#include <ruby/intern.h>
-#include <iv/ast.h>
-#include <iv/static_assert.h>
-namespace iv {
-namespace phonic {
-
-} }  // namespace iv::phonic
-#endif  // _IV_PHONIC_RNODE_H_