antlr4-native 2.2.0 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8bed5c4bbb574d8ad5cbef067bee9fff704a1f0bf8d5fa0b522393d5fa771be3
-  data.tar.gz: 44dcb2987d8f3c04bc2f9a4769853b6bdfbf390b0aebdff9fcc783b98a389896
+  metadata.gz: e0ba82bae327f0b02ac099beab9b01d8efef600b21dd1f0bac9438303af8d9fc
+  data.tar.gz: f840a00ac13adf1282c77fc2b2376da1b9c986cc32d9affa2232ab0d01c6ae57
 SHA512:
-  metadata.gz: 25961945b37da84af109e20b39728f57108d3e7490165d936898c95a76bec6add1e18086609c0085f14614efcd801b31723a17032a111ec86236f31709f52ec6
-  data.tar.gz: 20e79181660b66a56c94ef35fccd369c7590ed676703854f91aafa7147398fea220fb26c6e7f7ccb1bc3810fcc4c64f0db0018dda887b6344f6ae97930e24f3e
+  metadata.gz: a9aec0e46b565763c2e77c6e7af5108e24b7c348afda4d0073a5f99a52813feeb8204b8a554ed3dc500898aa4bb6a023c5646d5f0965c7d0bb25f2acb831889a
+  data.tar.gz: 508ed703bf98e7584c998cbed39b73f788a99cf5b65469b9c82d0adba5467c6bc1bed458ad26bde81dc5d32a73ef755a89a9516c9aa36ee5a550a3516dd30a7f

data/README.md

@@ -79,6 +79,7 @@ Finally, if you override `#initialize` in your visitor subclasses, don't forget
 
 ## Caveats
 
+1. Avoid retaining references to contexts, tokens, etc anywhere in your Ruby code. Contexts (i.e. the `ctx` variables in the examples above) and other objects that are created by ANTLR's C++ runtime are automatically cleaned up without the Ruby interpreter's knowledge. You'll almost surely see a segfault if you retain a reference to one of these objects and try to use it after the call to `Parser#visit`.
 1. Due to an ANTLR limitation, parsers cannot be used in a multi-threaded environment, even if each parser instance is used entirely in the context of a single thread (i.e. parsers are not shared between threads). According to the ANTLR C++ developers, parsers should be threadsafe. Unfortunately firsthand experience has proven otherwise. Your mileage may vary.
 1. The description of this gem says "(almost) any ANTLR4 grammar" because many grammars contain target-specific code. For example, the Python3 grammar referenced in the examples above contains inline Java code that the C++ compiler won't understand. You'll need to port any such code to C++ before you'll be able to compile and use the native extension.
 

data/antlr4-native.gemspec

@@ -15,5 +15,5 @@ Gem::Specification.new do |s|
   s.require_path = 'lib'
   s.files = Dir['{lib,spec,vendor}/**/*', 'Gemfile', 'README.md', 'Rakefile', 'antlr4-native.gemspec']
 
-  s.add_runtime_dependency "rice", "~> 4.0"
+  s.add_runtime_dependency "rice", "~> 4.11"
 end

data/lib/antlr4-native/context.rb

@@ -56,7 +56,7 @@ module Antlr4Native
           class To_Ruby<#{parser_ns}::#{name}*> {
           public:
             VALUE convert(#{parser_ns}::#{name}* const &x) {
-              if (!x) return Nil;
+              if (!x) return Qnil;
               return Data_Object<#{parser_ns}::#{name}>(x, false, #{proxy_class_variable});
             }
           };
@@ -65,7 +65,7 @@ module Antlr4Native
           class To_Ruby<#{name}Proxy*> {
           public:
             VALUE convert(#{name}Proxy* const &x) {
-              if (!x) return Nil;
+              if (!x) return Qnil;
               return Data_Object<#{name}Proxy>(x, false, #{proxy_class_variable});
             }
           };
@@ -114,11 +114,11 @@ module Antlr4Native
 
               for (auto child : getChildren()) {
                 if (ctx == detail::From_Ruby<ContextProxy>().convert(child.value()).getOriginal()) {
-                  return child;
+                  return Rice::Object(child);
                 }
               }
 
-              return Nil;
+              return Qnil;
             }
           END
         end

data/lib/antlr4-native/generator.rb

@@ -97,7 +97,7 @@ module Antlr4Native
           class To_Ruby<Token*> {
           public:
             VALUE convert(Token* const &x) {
-              if (!x) return Nil;
+              if (!x) return Qnil;
               return Data_Object<Token>(x, false, rb_cToken);
             }
           };
@@ -106,7 +106,7 @@ module Antlr4Native
           class To_Ruby<tree::ParseTree*> {
           public:
             VALUE convert(tree::ParseTree* const &x) {
-              if (!x) return Nil;
+              if (!x) return Qnil;
               return Data_Object<tree::ParseTree>(x, false, rb_cParseTree);
             }
           };
@@ -115,7 +115,7 @@ module Antlr4Native
           class To_Ruby<tree::TerminalNode*> {
           public:
             VALUE convert(tree::TerminalNode* const &x) {
-              if (!x) return Nil;
+              if (!x) return Qnil;
               return Data_Object<tree::TerminalNode>(x, false, rb_cTerminalNode);
             }
           };
@@ -153,7 +153,7 @@ module Antlr4Native
               for (auto it = orig -> children.begin(); it != orig -> children.end(); it ++) {
                 Object parseTree = ContextProxy::wrapParseTree(*it);
 
-                if (parseTree != Nil) {
+                if (!parseTree.is_nil()) {
                   children.push(parseTree);
                 }
               }
@@ -162,7 +162,7 @@ module Antlr4Native
           }
 
           Object getParent() {
-            return orig == nullptr ? Nil : ContextProxy::wrapParseTree(orig -> parent);
+            return orig == nullptr ? Rice::Object(Qnil) : ContextProxy::wrapParseTree(orig -> parent);
           }
 
           size_t childCount() {
@@ -300,13 +300,26 @@ module Antlr4Native
         };
 
         namespace Rice::detail {
+          template <>
+          struct Type<ParserProxy*> {
+            static bool verify() {
+              return true;
+            }
+          };
+
           template <>
           class To_Ruby<ParserProxy*> {
           public:
+            To_Ruby() = default;
+
+            explicit To_Ruby(Arg* arg) : arg_(arg) {}
+
             VALUE convert(ParserProxy* const &x) {
-              if (!x) return Nil;
+              if (!x) return Qnil;
               return Data_Object<ParserProxy>(x, false, rb_cParser);
             }
+          private:
+            Arg* arg_ = nullptr;
           };
         }
       END
@@ -346,8 +359,8 @@ module Antlr4Native
           rb_cParser = define_class_under<ParserProxy>(rb_m#{parser_ns}, "Parser")
             .define_singleton_function("parse", &ParserProxy::parse)
             .define_singleton_function("parse_file", &ParserProxy::parseFile)
-            .define_method("#{parser_root_method}", &ParserProxy::#{parser_root_method}, Return().keepAlive())
-            .define_method("visit", &ParserProxy::visit, Return().keepAlive());
+            .define_method("#{parser_root_method}", &ParserProxy::#{parser_root_method})
+            .define_method("visit", &ParserProxy::visit);
 
         #{class_wrappers_str('  ')}
         }
@@ -372,7 +385,7 @@ module Antlr4Native
               TerminalNodeProxy proxy(node);
               return detail::To_Ruby<TerminalNodeProxy>().convert(proxy);
             } else {
-              return Nil;
+              return Qnil;
             }
           }
         END

data/lib/antlr4-native/version.rb

@@ -1,3 +1,3 @@
 module Antlr4Native
-  VERSION = '2.2.0'
+  VERSION = '2.3.0'
 end

data/spec/lua-parser-rb/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :development do
+  gem 'pry-byebug'
+  gem 'antlr4-native', path: '../../../'
+end
+
+group :development, :test do
+  gem 'rake'
+end

data/spec/lua-parser-rb/Lua.g4

@@ -0,0 +1,335 @@
+/*
+BSD License
+
+Copyright (c) 2013, Kazunori Sakamoto
+Copyright (c) 2016, Alexander Alexeev
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the NAME of Rainer Schuster nor the NAMEs of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+This grammar file derived from:
+
+    Lua 5.3 Reference Manual
+    http://www.lua.org/manual/5.3/manual.html
+
+    Lua 5.2 Reference Manual
+    http://www.lua.org/manual/5.2/manual.html
+
+    Lua 5.1 grammar written by Nicolai Mainiero
+    http://www.antlr3.org/grammar/1178608849736/Lua.g
+
+Tested by Kazunori Sakamoto with Test suite for Lua 5.2 (http://www.lua.org/tests/5.2/)
+
+Tested by Alexander Alexeev with Test suite for Lua 5.3 http://www.lua.org/tests/lua-5.3.2-tests.tar.gz 
+*/
+
+grammar Lua;
+
+chunk
+    : block EOF
+    ;
+
+block
+    : stat* retstat?
+    ;
+
+stat
+    : ';'
+    | varlist '=' explist
+    | functioncall
+    | label
+    | 'break'
+    | 'goto' NAME
+    | 'do' block 'end'
+    | 'while' exp 'do' block 'end'
+    | 'repeat' block 'until' exp
+    | 'if' exp 'then' block ('elseif' exp 'then' block)* ('else' block)? 'end'
+    | 'for' NAME '=' exp ',' exp (',' exp)? 'do' block 'end'
+    | 'for' namelist 'in' explist 'do' block 'end'
+    | 'function' funcname funcbody
+    | 'local' 'function' NAME funcbody
+    | 'local' namelist ('=' explist)?
+    ;
+
+retstat
+    : 'return' explist? ';'?
+    ;
+
+label
+    : '::' NAME '::'
+    ;
+
+funcname
+    : NAME ('.' NAME)* (':' NAME)?
+    ;
+
+varlist
+    : var (',' var)*
+    ;
+
+namelist
+    : NAME (',' NAME)*
+    ;
+
+explist
+    : exp (',' exp)*
+    ;
+
+exp
+    : 'nil' | 'false' | 'true'
+    | number
+    | string
+    | '...'
+    | functiondef
+    | prefixexp
+    | tableconstructor
+    | <assoc=right> exp operatorPower exp
+    | operatorUnary exp
+    | exp operatorMulDivMod exp
+    | exp operatorAddSub exp
+    | <assoc=right> exp operatorStrcat exp
+    | exp operatorComparison exp
+    | exp operatorAnd exp
+    | exp operatorOr exp
+    | exp operatorBitwise exp
+    ;
+
+prefixexp
+    : varOrExp nameAndArgs*
+    ;
+
+functioncall
+    : varOrExp nameAndArgs+
+    ;
+
+varOrExp
+    : var | '(' exp ')'
+    ;
+
+var
+    : (NAME | '(' exp ')' varSuffix) varSuffix*
+    ;
+
+varSuffix
+    : nameAndArgs* ('[' exp ']' | '.' NAME)
+    ;
+
+nameAndArgs
+    : (':' NAME)? args
+    ;
+
+/*
+var
+    : NAME | prefixexp '[' exp ']' | prefixexp '.' NAME
+    ;
+
+prefixexp
+    : var | functioncall | '(' exp ')'
+    ;
+
+functioncall
+    : prefixexp args | prefixexp ':' NAME args 
+    ;
+*/
+
+args
+    : '(' explist? ')' | tableconstructor | string
+    ;
+
+functiondef
+    : 'function' funcbody
+    ;
+
+funcbody
+    : '(' parlist? ')' block 'end'
+    ;
+
+parlist
+    : namelist (',' '...')? | '...'
+    ;
+
+tableconstructor
+    : '{' fieldlist? '}'
+    ;
+
+fieldlist
+    : field (fieldsep field)* fieldsep?
+    ;
+
+field
+    : '[' exp ']' '=' exp | NAME '=' exp | exp
+    ;
+
+fieldsep
+    : ',' | ';'
+    ;
+
+operatorOr 
+	: 'or';
+
+operatorAnd 
+	: 'and';
+
+operatorComparison 
+	: '<' | '>' | '<=' | '>=' | '~=' | '==';
+
+operatorStrcat
+	: '..';
+
+operatorAddSub
+	: '+' | '-';
+
+operatorMulDivMod
+	: '*' | '/' | '%' | '//';
+
+operatorBitwise
+	: '&' | '|' | '~' | '<<' | '>>';
+
+operatorUnary
+    : 'not' | '#' | '-' | '~';
+
+operatorPower
+    : '^';
+
+number
+    : INT | HEX | FLOAT | HEX_FLOAT
+    ;
+
+string
+    : NORMALSTRING | CHARSTRING | LONGSTRING
+    ;
+
+// LEXER
+
+NAME
+    : [a-zA-Z_][a-zA-Z_0-9]*
+    ;
+
+NORMALSTRING
+    : '"' ( EscapeSequence | ~('\\'|'"') )* '"' 
+    ;
+
+CHARSTRING
+    : '\'' ( EscapeSequence | ~('\''|'\\') )* '\''
+    ;
+
+LONGSTRING
+    : '[' NESTED_STR ']'
+    ;
+
+fragment
+NESTED_STR
+    : '=' NESTED_STR '='
+    | '[' .*? ']'
+    ;
+
+INT
+    : Digit+
+    ;
+
+HEX
+    : '0' [xX] HexDigit+
+    ;
+
+FLOAT
+    : Digit+ '.' Digit* ExponentPart?
+    | '.' Digit+ ExponentPart?
+    | Digit+ ExponentPart
+    ;
+
+HEX_FLOAT
+    : '0' [xX] HexDigit+ '.' HexDigit* HexExponentPart?
+    | '0' [xX] '.' HexDigit+ HexExponentPart?
+    | '0' [xX] HexDigit+ HexExponentPart
+    ;
+
+fragment
+ExponentPart
+    : [eE] [+-]? Digit+
+    ;
+
+fragment
+HexExponentPart
+    : [pP] [+-]? Digit+
+    ;
+
+fragment
+EscapeSequence
+    : '\\' [abfnrtvz"'\\]
+    | '\\' '\r'? '\n'
+    | DecimalEscape
+    | HexEscape
+    | UtfEscape
+    ;
+    
+fragment
+DecimalEscape
+    : '\\' Digit
+    | '\\' Digit Digit
+    | '\\' [0-2] Digit Digit
+    ;
+    
+fragment
+HexEscape
+    : '\\' 'x' HexDigit HexDigit
+    ;
+
+fragment
+UtfEscape
+    : '\\' 'u{' HexDigit+ '}'
+    ;
+
+fragment
+Digit
+    : [0-9]
+    ;
+
+fragment
+HexDigit
+    : [0-9a-fA-F]
+    ;
+
+COMMENT
+    : '--[' NESTED_STR ']' -> channel(HIDDEN)
+    ;
+    
+LINE_COMMENT
+    : '--'
+    (                                               // --
+    | '[' '='*                                      // --[==
+    | '[' '='* ~('='|'['|'\r'|'\n') ~('\r'|'\n')*   // --[==AA
+    | ~('['|'\r'|'\n') ~('\r'|'\n')*                // --AAA
+    ) ('\r\n'|'\r'|'\n'|EOF)
+    -> channel(HIDDEN)
+    ;
+    
+WS  
+    : [ \t\u000C\r\n]+ -> skip
+    ;
+
+SHEBANG
+    : '#' '!' ~('\n'|'\r')* -> channel(HIDDEN)
+    ;

data/spec/lua-parser-rb/Rakefile

@@ -0,0 +1,28 @@
+require 'bundler'
+
+require 'antlr4-native'
+require 'etc'
+
+def ruby_installer?
+  Object.const_defined?(:RubyInstaller)
+end
+
+Bundler::GemHelper.install_tasks
+
+task :generate do
+  generator = Antlr4Native::Generator.new(
+    grammar_files:      ["./Lua.g4"],
+    output_dir:         'ext/',
+    parser_root_method: 'chunk'
+  )
+
+  generator.generate
+end
+
+task :compile do
+  Dir.chdir(File.join(%w(ext lua_parser))) do
+    load 'extconf.rb'
+    RubyInstaller::Runtime.enable_msys_apps if ruby_installer?
+    exec "make -j #{Etc.nprocessors}"
+  end
+end

data/spec/lua-parser-rb/ext/lua_parser/extconf.rb

@@ -0,0 +1,38 @@
+require 'mkmf-rice'
+
+extension_name = 'lua_parser'
+dir_config(extension_name)
+
+have_library('stdc++')
+
+$CFLAGS << ' -std=c++14'
+
+if enable_config('static')
+  $defs.push '-DANTLR4CPP_STATIC' unless $defs.include?('-DANTLR4CPP_STATIC')
+end
+
+include_paths = [
+  '.',
+  'antlrgen',
+  'antlr4-upstream/runtime/Cpp/runtime/src',
+  'antlr4-upstream/runtime/Cpp/runtime/src/atn',
+  'antlr4-upstream/runtime/Cpp/runtime/src/dfa',
+  'antlr4-upstream/runtime/Cpp/runtime/src/misc',
+  'antlr4-upstream/runtime/Cpp/runtime/src/support',
+  'antlr4-upstream/runtime/Cpp/runtime/src/tree',
+  'antlr4-upstream/runtime/Cpp/runtime/src/tree/pattern',
+  'antlr4-upstream/runtime/Cpp/runtime/src/tree/xpath'
+]
+
+$srcs = []
+
+include_paths.each do |include_path|
+  $INCFLAGS << " -I#{include_path}"
+  $VPATH << include_path
+
+  Dir.glob("#{include_path}/*.cpp").each do |path|
+    $srcs << path
+  end
+end
+
+create_makefile(extension_name)

data/spec/lua-parser-rb/lib/lua_parser/version.rb

@@ -0,0 +1,3 @@
+module LuaParser
+  VERSION = '1.0.0'
+end

data/spec/lua-parser-rb/lib/lua_parser.rb

@@ -0,0 +1,2 @@
+# load the native extension
+require File.expand_path(File.join('..', 'ext', 'lua_parser', 'lua_parser'), __dir__)

data/spec/lua-parser-rb/lua_parser.gemspec

@@ -0,0 +1,31 @@
+$:.unshift File.join(File.dirname(__FILE__), 'lib')
+require 'lua_parser/version'
+
+Gem::Specification.new do |s|
+  s.name     = 'lua_parser'
+  s.version  = ::LuaParser::VERSION
+  s.authors  = ['Mickey Mouse']
+  s.email    = ['mickey@disney.com']
+  s.homepage = 'https://github.com/mickeymouse/lua-parser-rb'
+
+  s.description = s.summary = 'A Lua parser for Ruby'
+
+  s.platform = Gem::Platform::RUBY
+
+  s.add_dependency 'rice', '~> 4.0'
+
+  s.extensions = File.join(*%w(ext lua_parser extconf.rb))
+
+  s.require_path = 'lib'
+  s.files = Dir[
+    '{lib,spec}/**/*',
+    'ext/lua_parser/*.{cpp,h}',
+    'ext/lua_parser/extconf.rb',
+    'ext/lua_parser/antlrgen/*',
+    'ext/lua_parser/antlr4-upstream/runtime/Cpp/runtime/src/**/*.{cpp,h}',
+    'Gemfile',
+    'README.md',
+    'Rakefile',
+    'lua_parser.gemspec'
+  ]
+end

data/spec/lua-parser-rb/parse_test.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "lua_parser"
+
+class FuncVisitor < LuaParser::Visitor
+  def visit_functioncall(ctx)
+    puts ctx.var_or_exp.text
+    visit_children(ctx)
+  end
+end
+
+Dir.glob('lua/**/*.lua').each do |file_name|
+  # this file contains some weird non-UTF8 strings, so let's just skip it
+  next if File.basename(file_name) == "strings.lua"
+
+  lua_code = File.read(file_name)
+  parser = LuaParser::Parser.parse(lua_code)
+  visitor = FuncVisitor.new
+  parser.visit(visitor)
+end

data/spec/run-lua-test.sh

@@ -0,0 +1,5 @@
+#! /bin/bash
+
+antlr_version=$(bundle exec ruby -Ilib -rantlr4-native -e "puts Antlr4Native::Generator::ANTLR_VERSION")
+docker build --build-arg ANTLR_VERSION="${antlr_version}" -t antlr4-native-rb:latest .
+docker run -t antlr4-native-rb:latest bundle exec ruby parse_test.rb

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: antlr4-native
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.3.0
 platform: ruby
 authors:
 - Cameron Dutro
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-21 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice
@@ -16,14 +15,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.11'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '4.0'
+        version: '4.11'
 description: Create a Ruby native extension from any ANTLR4 grammar.
 email:
 - camertron@gmail.com
@@ -43,11 +42,19 @@ files:
 - lib/antlr4-native/string_helpers.rb
 - lib/antlr4-native/version.rb
 - lib/antlr4-native/visitor_generator.rb
+- spec/lua-parser-rb/Gemfile
+- spec/lua-parser-rb/Lua.g4
+- spec/lua-parser-rb/Rakefile
+- spec/lua-parser-rb/ext/lua_parser/extconf.rb
+- spec/lua-parser-rb/lib/lua_parser.rb
+- spec/lua-parser-rb/lib/lua_parser/version.rb
+- spec/lua-parser-rb/lua_parser.gemspec
+- spec/lua-parser-rb/parse_test.rb
+- spec/run-lua-test.sh
 - vendor/antlr-4.10.1-complete.jar
 homepage: http://github.com/camertron/antlr4-native-rb
 licenses: []
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -62,8 +69,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
-signing_key:
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Create a Ruby native extension from any ANTLR4 grammar.
 test_files: []