iv-phonic 0.0.5 → 0.0.6

data/Manifest.txt

@@ -3,46 +3,45 @@ Manifest.txt
 README.rdoc
 Rakefile
 test/test_iv_phonic.rb
-ext/include/iv/uchar.h
-ext/include/iv/space.h
-ext/include/iv/lexer.h
-ext/include/iv/conversions.h
-ext/include/iv/dtoa.h
 ext/include/iv/mt19937.h
-ext/include/iv/ast.h
-ext/include/iv/noncopyable.h
-ext/include/iv/ast-factory.h
-ext/include/iv/stringpiece.h
-ext/include/iv/static_assert.h
-ext/include/iv/functor.h
-ext/include/iv/parser.h
-ext/include/iv/errors.h
-ext/include/iv/ast-visitor.h
 ext/include/iv/token.h
-ext/include/iv/enable_if.h
 ext/include/iv/any.h
+ext/include/iv/ast-fwd.h
+ext/include/iv/static_assert.h
+ext/include/iv/ast.h
+ext/include/iv/dtoa.h
+ext/include/iv/ast-visitor.h
+ext/include/iv/ustringpiece.h
+ext/include/iv/ucdata.h
+ext/include/iv/ustring.h
+ext/include/iv/noncopyable.h
 ext/include/iv/keyword.h
-ext/include/iv/cmdline.h
-ext/include/iv/xorshift.h
+ext/include/iv/chars.h
+ext/include/iv/enable_if.h
+ext/include/iv/utils.h
+ext/include/iv/conversions.h
 ext/include/iv/location.h
-ext/include/iv/none.h
+ext/include/iv/lexer.h
+ext/include/iv/xorshift.h
+ext/include/iv/space.h
 ext/include/iv/algorithm.h
-ext/include/iv/ustring.h
-ext/include/iv/ast-fwd.h
-ext/include/iv/utils.h
-ext/include/iv/ast-info.h
-ext/include/iv/alloc.h
-ext/include/iv/chars.h
 ext/include/iv/fixedcontainer.h
-ext/include/iv/ustringpiece.h
+ext/include/iv/functor.h
+ext/include/iv/cmdline.h
 ext/include/iv/ast-serializer.h
-ext/include/iv/ucdata.h
-ext/iv/phonic/factory.h
-ext/iv/phonic/parser.h
-ext/iv/phonic/encoding.h
+ext/include/iv/stringpiece.h
+ext/include/iv/uchar.h
+ext/include/iv/ast-factory.h
+ext/include/iv/alloc.h
+ext/include/iv/ast-info.h
+ext/include/iv/parser.h
+ext/include/iv/none.h
+ext/include/iv/errors.h
+ext/iv/phonic/ast-fwd.h
 ext/iv/phonic/source.h
-ext/iv/phonic/phonic.cc
-ext/iv/phonic/rparser.h
 ext/iv/phonic/creator.h
-ext/iv/phonic/ast-fwd.h
-ext/iv/phonic/extconf.rb
+ext/iv/phonic/phonic.cc
+ext/iv/phonic/extconf.rb
+ext/iv/phonic/encoding.h
+ext/iv/phonic/parser.h
+ext/iv/phonic/factory.h

data/README.rdoc

@@ -8,8 +8,8 @@ phonic is ruby binding for iv AST API
 
 == MILESTONE
 
-+ parser - done
-+ specialized AST - working
+* parser - done
+* specialized AST - working
 
 == TARGET
 

data/Rakefile

@@ -10,7 +10,7 @@ directory "ext/include/iv"
 
 HOE = Hoe.spec 'iv-phonic' do
   developer('Constellation', 'utatane.tea@gmail.com')
-  self.version = '0.0.5'
+  self.version = '0.0.6'
   self.readme_file = 'README.rdoc'
   self.history_file = 'ChangeLog'
   self.extra_rdoc_files = FileList['*.rdoc']

data/ext/include/iv/conversions.h

@@ -48,6 +48,7 @@ inline double StringToDouble(Iter it, Iter last, bool parse_float) {
   bool is_decimal = true;
   bool is_signed = false;
   bool is_sign_found = false;
+  bool is_found_zero = false;
   std::size_t pos = 0;
   int significant_digits = 0;
   int insignificant_digits = 0;
@@ -84,9 +85,10 @@ inline double StringToDouble(Iter it, Iter last, bool parse_float) {
 
   if (Chars::IsDecimalDigit(*it)) {
     if (*it == '0') {
+      is_found_zero = true;
       ++it;
       if (it == last) {
-        return 0;
+        return (is_signed) ? -0.0 : 0.0;
       }
       if (!parse_float && (*it == 'x' || *it == 'X')) {
         if (is_sign_found) {
@@ -148,6 +150,7 @@ inline double StringToDouble(Iter it, Iter last, bool parse_float) {
     if (*it == '.') {
       buffer[pos++] = '.';
       ++it;
+      const Iter start = it;
       while (it != last &&
              Chars::IsDecimalDigit(*it)) {
         if (significant_digits < Conversions::kMaxSignificantDigits) {
@@ -156,6 +159,9 @@ inline double StringToDouble(Iter it, Iter last, bool parse_float) {
         }
         ++it;
       }
+      if (start == it) {
+        return Conversions::kNaN;
+      }
     } else {
       for (std::string::const_iterator inf_it = Conversions::kInfinity.begin(),
            inf_last = Conversions::kInfinity.end();
@@ -190,17 +196,29 @@ inline double StringToDouble(Iter it, Iter last, bool parse_float) {
     buffer[pos++] = *it;
     ++it;
     if (it == last) {
+      if (parse_float) {
+        --it;
+        --pos;
+        goto exponent_pasing_done;
+      }
       return Conversions::kNaN;
     }
+    bool is_signed_exp = false;
     if (*it == '+' || *it == '-') {
       buffer[pos++] = *it;
       ++it;
+      is_signed_exp = true;
     }
-    if (it == last) {
-      return Conversions::kNaN;
-    }
-    // more than 1 decimal digit required
-    if (!Chars::IsDecimalDigit(*it)) {
+    if (it == last || !Chars::IsDecimalDigit(*it)) {
+      if (parse_float) {
+        --it;
+        --pos;
+        if (is_signed_exp) {
+          --it;
+          --pos;
+        }
+        goto exponent_pasing_done;
+      }
       return Conversions::kNaN;
     }
     int exponent = 0;
@@ -220,6 +238,9 @@ inline double StringToDouble(Iter it, Iter last, bool parse_float) {
     pos+=4;
   }
 
+  // exponent_pasing_done label
+  exponent_pasing_done:
+
   while (it != last &&
          (Chars::IsWhiteSpace(*it) || Chars::IsLineTerminator(*it))) {
     ++it;
@@ -228,7 +249,7 @@ inline double StringToDouble(Iter it, Iter last, bool parse_float) {
   if (it == last || parse_float) {
     if (pos == 0) {
       // empty
-      return (parse_float) ? Conversions::kNaN : 0;
+      return (parse_float && !is_found_zero) ? Conversions::kNaN : 0;
     } else {
       buffer[pos++] = '\0';
       return std::strtod(buffer.data(), NULL);
@@ -246,6 +267,108 @@ inline double StringToDouble(const UStringPiece& str, bool parse_float) {
   return StringToDouble(str.begin(), str.end(), parse_float);
 }
 
+inline int OctalValue(const int c) {
+  if ('0' <= c && c <= '8') {
+    return c - '0';
+  }
+  return -1;
+}
+
+inline int HexValue(const int c) {
+  if ('0' <= c && c <= '9') {
+    return c - '0';
+  }
+  if ('a' <= c && c <= 'f') {
+    return c - 'a' + 10;
+  }
+  if ('A' <= c && c <= 'F') {
+    return c - 'A' + 10;
+  }
+  return -1;
+}
+
+inline int Radix36Value(const int c) {
+  if ('0' <= c && c <= '9') {
+    return c - '0';
+  }
+  if ('a' <= c && c <= 'z') {
+    return c - 'a' + 10;
+  }
+  if ('A' <= c && c <= 'Z') {
+    return c - 'A' + 10;
+  }
+  return -1;
+}
+
+template<typename Iter>
+inline double StringToIntegerWithRadix(Iter it, Iter last,
+                                       int radix, bool strip_prefix) {
+  // remove leading white space
+  while (it != last &&
+         (Chars::IsWhiteSpace(*it) || Chars::IsLineTerminator(*it))) {
+    ++it;
+  }
+
+  // empty string ""
+  if (it == last) {
+    return Conversions::kNaN;
+  }
+
+  int sign = 1;
+  if (*it == '-') {
+    sign = -1;
+    ++it;
+  } else if (*it == '+') {
+    ++it;
+  }
+
+  if (it == last) {
+    return Conversions::kNaN;
+  }
+
+  if (strip_prefix) {
+    if (*it == '0') {
+      ++it;
+      if (it != last &&
+          (*it == 'x' || *it == 'X')) {
+        // strip_prefix
+        ++it;
+        radix = 16;
+      } else {
+        --it;
+      }
+    }
+  }
+
+  if (it == last) {
+    return Conversions::kNaN;
+  }
+
+  double result = 0;
+  const Iter start = it;
+  for (; it != last; ++it) {
+    const int val = Radix36Value(*it);
+    if (val != -1 && val < radix) {
+      result = result * radix + val;
+    } else {
+      return (start == it) ? Conversions::kNaN : sign * result;
+    }
+  }
+  return sign * result;
+}
+
+inline double StringToIntegerWithRadix(const StringPiece& range,
+                                       int radix, bool strip_prefix) {
+  return StringToIntegerWithRadix(range.begin(), range.end(),
+                                  radix, strip_prefix);
+}
+
+inline double StringToIntegerWithRadix(const UStringPiece& range,
+                                       int radix, bool strip_prefix) {
+  return StringToIntegerWithRadix(range.begin(), range.end(),
+                                  radix, strip_prefix);
+}
+
 inline std::size_t StringToHash(const UStringPiece& x) {
   std::size_t len = x.size();
   std::size_t step = (len >> 5) + 1;
@@ -284,8 +407,8 @@ inline int32_t DoubleToInt32(double d) {
                               d - Conversions::DoubleToInt32_Two32 : d);
 }
 
-inline int32_t DoubleToUInt32(double d) {
-  return static_cast<uint32_t>(d);
+inline uint32_t DoubleToUInt32(double d) {
+  return static_cast<uint32_t>(DoubleToInt32(d));
 }
 
 inline double DoubleToInteger(double d) {
@@ -299,6 +422,7 @@ inline double DoubleToInteger(double d) {
 }
 
 inline bool ConvertToUInt32(const UStringPiece& str, uint32_t* value) {
+  static const uint32_t uint32_t_max = std::numeric_limits<uint32_t>::max();
   uint16_t ch;
   *value = 0;
   UStringPiece::const_iterator it = str.begin();
@@ -320,9 +444,9 @@ inline bool ConvertToUInt32(const UStringPiece& str, uint32_t* value) {
       return false;
     }
   }
-  return (prev < (std::numeric_limits<uint32_t>::max() / 10) ||
-          ((prev == (std::numeric_limits<uint32_t>::max() / 10)) &&
-           (ch < (std::numeric_limits<uint32_t>::max() % 10))));
+  return (prev < (uint32_t_max / 10) ||
+          ((prev == (uint32_t_max / 10)) &&
+           (ch < (uint32_t_max % 10))));
 }
 
 template<typename T>

data/ext/include/iv/lexer.h

@@ -12,6 +12,7 @@
 #include "location.h"
 #include "noncopyable.h"
 #include "keyword.h"
+#include "conversions.h"
 
 namespace iv {
 namespace core {
@@ -840,26 +841,6 @@ class Lexer: private Noncopyable<Lexer<Source> >::type {
     return res;
   }
 
-  inline int OctalValue(const int c) const {
-    if ('0' <= c && c <= '8') {
-      return c - '0';
-    }
-    return -1;
-  }
-
-  inline int HexValue(const int c) const {
-    if ('0' <= c && c <= '9') {
-      return c - '0';
-    }
-    if ('a' <= c && c <= 'f') {
-      return c - 'a' + 10;
-    }
-    if ('A' <= c && c <= 'F') {
-      return c - 'A' + 10;
-    }
-    return -1;
-  }
-
   void ScanDecimalDigits() {
     while (Chars::IsDecimalDigit(c_)) {
       Record8Advance();

data/ext/include/iv/space.h

@@ -53,8 +53,6 @@ class SpaceAllocator {
     : space_(alloc.space()) {
   }
 
-  ~SpaceAllocator() throw() {}
-
   inline pointer address(reference x) const {
     return &x;
   }

data/ext/iv/phonic/ast-fwd.h

@@ -1,13 +1,17 @@
 #ifndef _IV_PHONIC_AST_FWD_H_
 #define _IV_PHONIC_AST_FWD_H_
-#include <ruby.h>
 #include <iv/ast.h>
+
 namespace iv {
 namespace phonic {
+
 class AstFactory;
+
 }  // namespace iv::phonic
+
 namespace core {
 namespace ast {
+
 template<>
 class AstNodeBase<iv::phonic::AstFactory>
   : public Inherit<iv::phonic::AstFactory, kAstNode> {
@@ -20,8 +24,11 @@ class AstNodeBase<iv::phonic::AstFactory>
   std::size_t begin_;
   std::size_t end_;
 };
+
 } }  // namespace iv::core::ast
+
 namespace phonic {
+
 #define V(AST) typedef core::ast::AST<AstFactory> AST;
   AST_NODE_LIST(V)
 #undef V

data/ext/iv/phonic/creator.h

@@ -1,6 +1,8 @@
 #ifndef _IV_PHONIC_CREATOR_H_
 #define _IV_PHONIC_CREATOR_H_
+extern "C" {
 #include <ruby.h>
+}
 #include <iv/ast-visitor.h>
 #include <iv/utils.h>
 #include "factory.h"

data/ext/iv/phonic/encoding.h

@@ -1,8 +1,10 @@
 #ifndef _IV_PHONIC_ENCODING_H_
 #define _IV_PHONIC_ENCODING_H_
+extern "C" {
 #include <ruby.h>
 #include <ruby/encoding.h>
 #include <ruby/intern.h>
+}
 namespace iv {
 namespace phonic {
 

data/ext/iv/phonic/factory.h

@@ -1,8 +1,10 @@
 #ifndef _IV_PHONIC_FACTORY_H_
 #define _IV_PHONIC_FACTORY_H_
 #include <tr1/array>
+extern "C" {
 #include <ruby.h>
 #include <ruby/oniguruma.h>
+}
 #include <iv/alloc.h>
 #include <iv/ustringpiece.h>
 #include "encoding.h"

data/ext/iv/phonic/parser.h

@@ -1,6 +1,5 @@
 #ifndef _IV_PHONIC_PARSER_H_
 #define _IV_PHONIC_PARSER_H_
-#include <ruby.h>
 #include <iv/parser.h>
 #include "factory.h"
 #include "source.h"

data/ext/iv/phonic/phonic.cc

@@ -1,10 +1,45 @@
+extern "C" {
 #include <ruby.h>
-#include "rparser.h"
+#include <ruby/encoding.h>
+#include <ruby/intern.h>
+}
+#include <iv/parser.h>
+#include "encoding.h"
+#include "factory.h"
+#include "parser.h"
+#include "creator.h"
+#include "source.h"
 
 #define RBFUNC(func) (reinterpret_cast<VALUE(*)(...)>(func))
 
-extern "C" {
+namespace iv {
+namespace phonic {
 
+static VALUE cParseError;
+
+static VALUE Parse(VALUE self, VALUE rb_str) {
+  AstFactory factory;
+  Check_Type(rb_str, T_STRING);
+  VALUE encoded_rb_str = rb_str_encode(rb_str,
+                                       Encoding::UTF16Encoding(),
+                                       0,
+                                       Qnil);
+  const char* str = StringValuePtr(encoded_rb_str);
+  const std::size_t len = RSTRING_LEN(encoded_rb_str);
+  UTF16Source source(str, len);
+  Parser parser(&factory, &source);
+  const FunctionLiteral* global = parser.ParseProgram();
+  if (!global) {
+    rb_raise(cParseError, "%s", parser.error().c_str());
+  } else {
+    Creator creator;
+    return creator.Result(global);
+  }
+}
+
+} }  // namespace iv::phonic
+
+extern "C" {
 void Init_phonic() {
   iv::phonic::Encoding::Init();
   // IV
@@ -13,10 +48,10 @@ void Init_phonic() {
   // IV::Phonic
   VALUE mIVPhonic = rb_define_module_under(mIV, "Phonic");
   rb_define_module_function(mIVPhonic, "parse",
-                            RBFUNC(&iv::phonic::RParser::Parse), 1);
-  iv::phonic::RParser::Init(mIVPhonic);
+                            RBFUNC(&iv::phonic::Parse), 1);
+  iv::phonic::cParseError = rb_define_class_under(
+      mIVPhonic, "ParseError", rb_eStandardError);
 }
-
 }
 
 #undef RBFUNC

data/ext/iv/phonic/source.h

@@ -4,7 +4,6 @@
 #include <cassert>
 #include <string>
 #include <tr1/cstdint>
-#include <ruby.h>
 #include <iv/stringpiece.h>
 #include <iv/ustring.h>
 #include <iv/none.h>

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: iv-phonic
 version: !ruby/object:Gem::Version 
+  hash: 19
   prerelease: false
   segments: 
   - 0
   - 0
-  - 5
-  version: 0.0.5
+  - 6
+  version: 0.0.6
 platform: ruby
 authors: 
 - Constellation
@@ -14,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-28 00:00:00 +09:00
+date: 2010-12-02 00:00:00 +09:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -25,6 +26,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 7
         segments: 
         - 2
         - 0
@@ -40,6 +42,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -53,11 +56,12 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 19
         segments: 
         - 2
-        - 6
-        - 2
-        version: 2.6.2
+        - 7
+        - 0
+        version: 2.7.0
   type: :development
   version_requirements: *id003
 description: phonic is ruby binding for iv AST API
@@ -76,49 +80,48 @@ files:
 - README.rdoc
 - Rakefile
 - test/test_iv_phonic.rb
-- ext/include/iv/uchar.h
-- ext/include/iv/space.h
-- ext/include/iv/lexer.h
-- ext/include/iv/conversions.h
-- ext/include/iv/dtoa.h
 - ext/include/iv/mt19937.h
-- ext/include/iv/ast.h
-- ext/include/iv/noncopyable.h
-- ext/include/iv/ast-factory.h
-- ext/include/iv/stringpiece.h
-- ext/include/iv/static_assert.h
-- ext/include/iv/functor.h
-- ext/include/iv/parser.h
-- ext/include/iv/errors.h
-- ext/include/iv/ast-visitor.h
 - ext/include/iv/token.h
-- ext/include/iv/enable_if.h
 - ext/include/iv/any.h
+- ext/include/iv/ast-fwd.h
+- ext/include/iv/static_assert.h
+- ext/include/iv/ast.h
+- ext/include/iv/dtoa.h
+- ext/include/iv/ast-visitor.h
+- ext/include/iv/ustringpiece.h
+- ext/include/iv/ucdata.h
+- ext/include/iv/ustring.h
+- ext/include/iv/noncopyable.h
 - ext/include/iv/keyword.h
-- ext/include/iv/cmdline.h
-- ext/include/iv/xorshift.h
+- ext/include/iv/chars.h
+- ext/include/iv/enable_if.h
+- ext/include/iv/utils.h
+- ext/include/iv/conversions.h
 - ext/include/iv/location.h
-- ext/include/iv/none.h
+- ext/include/iv/lexer.h
+- ext/include/iv/xorshift.h
+- ext/include/iv/space.h
 - ext/include/iv/algorithm.h
-- ext/include/iv/ustring.h
-- ext/include/iv/ast-fwd.h
-- ext/include/iv/utils.h
-- ext/include/iv/ast-info.h
-- ext/include/iv/alloc.h
-- ext/include/iv/chars.h
 - ext/include/iv/fixedcontainer.h
-- ext/include/iv/ustringpiece.h
+- ext/include/iv/functor.h
+- ext/include/iv/cmdline.h
 - ext/include/iv/ast-serializer.h
-- ext/include/iv/ucdata.h
-- ext/iv/phonic/factory.h
-- ext/iv/phonic/parser.h
-- ext/iv/phonic/encoding.h
+- ext/include/iv/stringpiece.h
+- ext/include/iv/uchar.h
+- ext/include/iv/ast-factory.h
+- ext/include/iv/alloc.h
+- ext/include/iv/ast-info.h
+- ext/include/iv/parser.h
+- ext/include/iv/none.h
+- ext/include/iv/errors.h
+- ext/iv/phonic/ast-fwd.h
 - ext/iv/phonic/source.h
-- ext/iv/phonic/phonic.cc
-- ext/iv/phonic/rparser.h
 - ext/iv/phonic/creator.h
-- ext/iv/phonic/ast-fwd.h
+- ext/iv/phonic/phonic.cc
 - ext/iv/phonic/extconf.rb
+- ext/iv/phonic/encoding.h
+- ext/iv/phonic/parser.h
+- ext/iv/phonic/factory.h
 has_rdoc: true
 homepage: https://github.com/Constellation/iv/tree/master/src/phonic/
 licenses: []
@@ -134,6 +137,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 49
       segments: 
       - 1
       - 9
@@ -144,6 +148,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"

data/ext/iv/phonic/rparser.h

@@ -1,47 +0,0 @@
-#ifndef _IV_PHONIC_RPARSER_H_
-#define _IV_PHONIC_RPARSER_H_
-#include <cstdlib>
-#include <cstring>
-#include <ruby.h>
-#include <ruby/encoding.h>
-#include <ruby/intern.h>
-#include <iv/parser.h>
-#include "encoding.h"
-#include "factory.h"
-#include "parser.h"
-#include "creator.h"
-#include "source.h"
-namespace iv {
-namespace phonic {
-
-static VALUE cParseError;
-
-class RParser {
- public:
-  static VALUE Parse(VALUE self, VALUE rb_str) {
-    AstFactory factory;
-    Check_Type(rb_str, T_STRING);
-    VALUE encoded_rb_str = rb_str_encode(rb_str,
-                                         Encoding::UTF16Encoding(),
-                                         0,
-                                         Qnil);
-    const char* str = StringValuePtr(encoded_rb_str);
-    const std::size_t len = RSTRING_LEN(encoded_rb_str);
-    UTF16Source source(str, len);
-    Parser parser(&factory, &source);
-    const FunctionLiteral* global = parser.ParseProgram();
-    if (!global) {
-      rb_raise(cParseError, "%s", parser.error().c_str());
-    } else {
-      Creator creator;
-      return creator.Result(global);
-    }
-  }
-
-  static void Init(VALUE mIVPhonic) {
-    cParseError = rb_define_class_under(mIVPhonic, "ParseError", rb_eStandardError);
-  }
-};
-
-} }  // namespace iv::phonic
-#endif  // _IV_PHONIC_RPARSER_H_