antlr4-native 2.1.0 → 2.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15c061ecc5402e131b0bb299f653624452168b857a20010f6243e1415d74fe43
-  data.tar.gz: c9c1dcbdcb46daedcdfbe219dee93d1f18512fbbeeb26808a5d2acb3281db73c
+  metadata.gz: 417a38bde6ca6dfdeca982ed80f0258c00c18feeda5cfecbaab7a5784e941501
+  data.tar.gz: 99531b3de905a08734dc2f421dc8b4e488850ab0021a6619c0581c6a810590d7
 SHA512:
-  metadata.gz: 0c973edc74b86e8fbf2193832c3555aa1e93a52bc60dc9736752c0be35a18cee3a1dd5d68cb262603467e0586a4eb22c74b38589246eb6d34edba6d4e4fe54c2
-  data.tar.gz: e6b89735c4083c6c51b6260f751048aeec4187da2a93d286af7cf2778d490fb5d82a9dff958af7285229f790f3a8e1f537581a5946feda441ebf95e1bd93ffa8
+  metadata.gz: bad5784b07eb34c221c4a1a7a8311dd9c6998a837c5b5f13605564367aa15269055c3046e4894d531b4e9512467200896f0eb0495f08f46972593367d1143012
+  data.tar.gz: a0f02bfbbbff0fb8d06849623f8ced4046ebb2feb28a1f21d48bd03f2d017887c41a289777f30f4016205d01a49e3c77d8df9a83352a9da6b05ba43dc81148b3

data/README.md

@@ -79,6 +79,7 @@ Finally, if you override `#initialize` in your visitor subclasses, don't forget
 
 ## Caveats
 
+1. Avoid retaining references to contexts, tokens, etc anywhere in your Ruby code. Contexts (i.e. the `ctx` variables in the examples above) and other objects that are created by ANTLR's C++ runtime are automatically cleaned up without the Ruby interpreter's knowledge. You'll almost surely see a segfault if you retain a reference to one of these objects and try to use it after the call to `Parser#visit`.
 1. Due to an ANTLR limitation, parsers cannot be used in a multi-threaded environment, even if each parser instance is used entirely in the context of a single thread (i.e. parsers are not shared between threads). According to the ANTLR C++ developers, parsers should be threadsafe. Unfortunately firsthand experience has proven otherwise. Your mileage may vary.
 1. The description of this gem says "(almost) any ANTLR4 grammar" because many grammars contain target-specific code. For example, the Python3 grammar referenced in the examples above contains inline Java code that the C++ compiler won't understand. You'll need to port any such code to C++ before you'll be able to compile and use the native extension.
 

data/lib/antlr4-native/generator.rb

@@ -25,7 +25,7 @@ module Antlr4Native
     end
 
     def gem_name
-      @gem_name ||= dasherize(parser_ns)
+      @gem_name ||= underscore(parser_ns)
     end
 
     def antlr_ns
@@ -316,7 +316,7 @@ module Antlr4Native
       <<~END
         extern "C"
         void Init_#{ext_name}() {
-          Module rb_m#{parser_ns} = define_module("#{parser_ns}");
+          Module rb_m#{parser_ns} = define_module("#{capitalize(parser_ns)}");
 
           rb_cToken = define_class_under<Token>(rb_m#{parser_ns}, "Token")
             .define_method("text", &Token::getText)
@@ -346,8 +346,8 @@ module Antlr4Native
           rb_cParser = define_class_under<ParserProxy>(rb_m#{parser_ns}, "Parser")
             .define_singleton_function("parse", &ParserProxy::parse)
             .define_singleton_function("parse_file", &ParserProxy::parseFile)
-            .define_method("#{parser_root_method}", &ParserProxy::#{parser_root_method}, Return().keepAlive())
-            .define_method("visit", &ParserProxy::visit, Return().keepAlive());
+            .define_method("#{parser_root_method}", &ParserProxy::#{parser_root_method})
+            .define_method("visit", &ParserProxy::visit);
 
         #{class_wrappers_str('  ')}
         }

data/lib/antlr4-native/version.rb

@@ -1,3 +1,3 @@
 module Antlr4Native
-  VERSION = '2.1.0'
+  VERSION = '2.2.1'
 end

data/spec/lua-parser-rb/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :development do
+  gem 'pry-byebug'
+  gem 'antlr4-native', path: '../../../'
+end
+
+group :development, :test do
+  gem 'rake'
+end

data/spec/lua-parser-rb/Lua.g4

@@ -0,0 +1,335 @@
+/*
+BSD License
+
+Copyright (c) 2013, Kazunori Sakamoto
+Copyright (c) 2016, Alexander Alexeev
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+3. Neither the NAME of Rainer Schuster nor the NAMEs of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+This grammar file derived from:
+
+    Lua 5.3 Reference Manual
+    http://www.lua.org/manual/5.3/manual.html
+
+    Lua 5.2 Reference Manual
+    http://www.lua.org/manual/5.2/manual.html
+
+    Lua 5.1 grammar written by Nicolai Mainiero
+    http://www.antlr3.org/grammar/1178608849736/Lua.g
+
+Tested by Kazunori Sakamoto with Test suite for Lua 5.2 (http://www.lua.org/tests/5.2/)
+
+Tested by Alexander Alexeev with Test suite for Lua 5.3 http://www.lua.org/tests/lua-5.3.2-tests.tar.gz 
+*/
+
+grammar Lua;
+
+chunk
+    : block EOF
+    ;
+
+block
+    : stat* retstat?
+    ;
+
+stat
+    : ';'
+    | varlist '=' explist
+    | functioncall
+    | label
+    | 'break'
+    | 'goto' NAME
+    | 'do' block 'end'
+    | 'while' exp 'do' block 'end'
+    | 'repeat' block 'until' exp
+    | 'if' exp 'then' block ('elseif' exp 'then' block)* ('else' block)? 'end'
+    | 'for' NAME '=' exp ',' exp (',' exp)? 'do' block 'end'
+    | 'for' namelist 'in' explist 'do' block 'end'
+    | 'function' funcname funcbody
+    | 'local' 'function' NAME funcbody
+    | 'local' namelist ('=' explist)?
+    ;
+
+retstat
+    : 'return' explist? ';'?
+    ;
+
+label
+    : '::' NAME '::'
+    ;
+
+funcname
+    : NAME ('.' NAME)* (':' NAME)?
+    ;
+
+varlist
+    : var (',' var)*
+    ;
+
+namelist
+    : NAME (',' NAME)*
+    ;
+
+explist
+    : exp (',' exp)*
+    ;
+
+exp
+    : 'nil' | 'false' | 'true'
+    | number
+    | string
+    | '...'
+    | functiondef
+    | prefixexp
+    | tableconstructor
+    | <assoc=right> exp operatorPower exp
+    | operatorUnary exp
+    | exp operatorMulDivMod exp
+    | exp operatorAddSub exp
+    | <assoc=right> exp operatorStrcat exp
+    | exp operatorComparison exp
+    | exp operatorAnd exp
+    | exp operatorOr exp
+    | exp operatorBitwise exp
+    ;
+
+prefixexp
+    : varOrExp nameAndArgs*
+    ;
+
+functioncall
+    : varOrExp nameAndArgs+
+    ;
+
+varOrExp
+    : var | '(' exp ')'
+    ;
+
+var
+    : (NAME | '(' exp ')' varSuffix) varSuffix*
+    ;
+
+varSuffix
+    : nameAndArgs* ('[' exp ']' | '.' NAME)
+    ;
+
+nameAndArgs
+    : (':' NAME)? args
+    ;
+
+/*
+var
+    : NAME | prefixexp '[' exp ']' | prefixexp '.' NAME
+    ;
+
+prefixexp
+    : var | functioncall | '(' exp ')'
+    ;
+
+functioncall
+    : prefixexp args | prefixexp ':' NAME args 
+    ;
+*/
+
+args
+    : '(' explist? ')' | tableconstructor | string
+    ;
+
+functiondef
+    : 'function' funcbody
+    ;
+
+funcbody
+    : '(' parlist? ')' block 'end'
+    ;
+
+parlist
+    : namelist (',' '...')? | '...'
+    ;
+
+tableconstructor
+    : '{' fieldlist? '}'
+    ;
+
+fieldlist
+    : field (fieldsep field)* fieldsep?
+    ;
+
+field
+    : '[' exp ']' '=' exp | NAME '=' exp | exp
+    ;
+
+fieldsep
+    : ',' | ';'
+    ;
+
+operatorOr 
+	: 'or';
+
+operatorAnd 
+	: 'and';
+
+operatorComparison 
+	: '<' | '>' | '<=' | '>=' | '~=' | '==';
+
+operatorStrcat
+	: '..';
+
+operatorAddSub
+	: '+' | '-';
+
+operatorMulDivMod
+	: '*' | '/' | '%' | '//';
+
+operatorBitwise
+	: '&' | '|' | '~' | '<<' | '>>';
+
+operatorUnary
+    : 'not' | '#' | '-' | '~';
+
+operatorPower
+    : '^';
+
+number
+    : INT | HEX | FLOAT | HEX_FLOAT
+    ;
+
+string
+    : NORMALSTRING | CHARSTRING | LONGSTRING
+    ;
+
+// LEXER
+
+NAME
+    : [a-zA-Z_][a-zA-Z_0-9]*
+    ;
+
+NORMALSTRING
+    : '"' ( EscapeSequence | ~('\\'|'"') )* '"' 
+    ;
+
+CHARSTRING
+    : '\'' ( EscapeSequence | ~('\''|'\\') )* '\''
+    ;
+
+LONGSTRING
+    : '[' NESTED_STR ']'
+    ;
+
+fragment
+NESTED_STR
+    : '=' NESTED_STR '='
+    | '[' .*? ']'
+    ;
+
+INT
+    : Digit+
+    ;
+
+HEX
+    : '0' [xX] HexDigit+
+    ;
+
+FLOAT
+    : Digit+ '.' Digit* ExponentPart?
+    | '.' Digit+ ExponentPart?
+    | Digit+ ExponentPart
+    ;
+
+HEX_FLOAT
+    : '0' [xX] HexDigit+ '.' HexDigit* HexExponentPart?
+    | '0' [xX] '.' HexDigit+ HexExponentPart?
+    | '0' [xX] HexDigit+ HexExponentPart
+    ;
+
+fragment
+ExponentPart
+    : [eE] [+-]? Digit+
+    ;
+
+fragment
+HexExponentPart
+    : [pP] [+-]? Digit+
+    ;
+
+fragment
+EscapeSequence
+    : '\\' [abfnrtvz"'\\]
+    | '\\' '\r'? '\n'
+    | DecimalEscape
+    | HexEscape
+    | UtfEscape
+    ;
+    
+fragment
+DecimalEscape
+    : '\\' Digit
+    | '\\' Digit Digit
+    | '\\' [0-2] Digit Digit
+    ;
+    
+fragment
+HexEscape
+    : '\\' 'x' HexDigit HexDigit
+    ;
+
+fragment
+UtfEscape
+    : '\\' 'u{' HexDigit+ '}'
+    ;
+
+fragment
+Digit
+    : [0-9]
+    ;
+
+fragment
+HexDigit
+    : [0-9a-fA-F]
+    ;
+
+COMMENT
+    : '--[' NESTED_STR ']' -> channel(HIDDEN)
+    ;
+    
+LINE_COMMENT
+    : '--'
+    (                                               // --
+    | '[' '='*                                      // --[==
+    | '[' '='* ~('='|'['|'\r'|'\n') ~('\r'|'\n')*   // --[==AA
+    | ~('['|'\r'|'\n') ~('\r'|'\n')*                // --AAA
+    ) ('\r\n'|'\r'|'\n'|EOF)
+    -> channel(HIDDEN)
+    ;
+    
+WS  
+    : [ \t\u000C\r\n]+ -> skip
+    ;
+
+SHEBANG
+    : '#' '!' ~('\n'|'\r')* -> channel(HIDDEN)
+    ;

data/spec/lua-parser-rb/Rakefile

@@ -0,0 +1,28 @@
+require 'bundler'
+
+require 'antlr4-native'
+require 'etc'
+
+def ruby_installer?
+  Object.const_defined?(:RubyInstaller)
+end
+
+Bundler::GemHelper.install_tasks
+
+task :generate do
+  generator = Antlr4Native::Generator.new(
+    grammar_files:      ["./Lua.g4"],
+    output_dir:         'ext/',
+    parser_root_method: 'chunk'
+  )
+
+  generator.generate
+end
+
+task :compile do
+  Dir.chdir(File.join(%w(ext lua_parser))) do
+    load 'extconf.rb'
+    RubyInstaller::Runtime.enable_msys_apps if ruby_installer?
+    exec "make -j #{Etc.nprocessors}"
+  end
+end

data/spec/lua-parser-rb/ext/lua_parser/extconf.rb

@@ -0,0 +1,38 @@
+require 'mkmf-rice'
+
+extension_name = 'lua_parser'
+dir_config(extension_name)
+
+have_library('stdc++')
+
+$CFLAGS << ' -std=c++14'
+
+if enable_config('static')
+  $defs.push '-DANTLR4CPP_STATIC' unless $defs.include?('-DANTLR4CPP_STATIC')
+end
+
+include_paths = [
+  '.',
+  'antlrgen',
+  'antlr4-upstream/runtime/Cpp/runtime/src',
+  'antlr4-upstream/runtime/Cpp/runtime/src/atn',
+  'antlr4-upstream/runtime/Cpp/runtime/src/dfa',
+  'antlr4-upstream/runtime/Cpp/runtime/src/misc',
+  'antlr4-upstream/runtime/Cpp/runtime/src/support',
+  'antlr4-upstream/runtime/Cpp/runtime/src/tree',
+  'antlr4-upstream/runtime/Cpp/runtime/src/tree/pattern',
+  'antlr4-upstream/runtime/Cpp/runtime/src/tree/xpath'
+]
+
+$srcs = []
+
+include_paths.each do |include_path|
+  $INCFLAGS << " -I#{include_path}"
+  $VPATH << include_path
+
+  Dir.glob("#{include_path}/*.cpp").each do |path|
+    $srcs << path
+  end
+end
+
+create_makefile(extension_name)

data/spec/lua-parser-rb/lib/lua_parser/version.rb

@@ -0,0 +1,3 @@
+module LuaParser
+  VERSION = '1.0.0'
+end

data/spec/lua-parser-rb/lib/lua_parser.rb

@@ -0,0 +1,2 @@
+# load the native extension
+require File.expand_path(File.join('..', 'ext', 'lua_parser', 'lua_parser'), __dir__)

data/spec/lua-parser-rb/lua_parser.gemspec

@@ -0,0 +1,31 @@
+$:.unshift File.join(File.dirname(__FILE__), 'lib')
+require 'lua_parser/version'
+
+Gem::Specification.new do |s|
+  s.name     = 'lua_parser'
+  s.version  = ::LuaParser::VERSION
+  s.authors  = ['Mickey Mouse']
+  s.email    = ['mickey@disney.com']
+  s.homepage = 'https://github.com/mickeymouse/lua-parser-rb'
+
+  s.description = s.summary = 'A Lua parser for Ruby'
+
+  s.platform = Gem::Platform::RUBY
+
+  s.add_dependency 'rice', '~> 4.0'
+
+  s.extensions = File.join(*%w(ext lua_parser extconf.rb))
+
+  s.require_path = 'lib'
+  s.files = Dir[
+    '{lib,spec}/**/*',
+    'ext/lua_parser/*.{cpp,h}',
+    'ext/lua_parser/extconf.rb',
+    'ext/lua_parser/antlrgen/*',
+    'ext/lua_parser/antlr4-upstream/runtime/Cpp/runtime/src/**/*.{cpp,h}',
+    'Gemfile',
+    'README.md',
+    'Rakefile',
+    'lua_parser.gemspec'
+  ]
+end

data/spec/lua-parser-rb/parse_test.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require "lua_parser"
+
+class FuncVisitor < LuaParser::Visitor
+  def visit_functioncall(ctx)
+    puts ctx.var_or_exp.text
+    visit_children(ctx)
+  end
+end
+
+Dir.glob('lua/**/*.lua').each do |file_name|
+  # this file contains some weird non-UTF8 strings, so let's just skip it
+  next if File.basename(file_name) == "strings.lua"
+
+  lua_code = File.read(file_name)
+  parser = LuaParser::Parser.parse(lua_code)
+  visitor = FuncVisitor.new
+  parser.visit(visitor)
+end

data/spec/run-lua-test.sh

@@ -0,0 +1,5 @@
+#! /bin/bash
+
+antlr_version=$(bundle exec ruby -Ilib -rantlr4-native -e "puts Antlr4Native::Generator::ANTLR_VERSION")
+docker build --build-arg ANTLR_VERSION="${antlr_version}" -t antlr4-native-rb:latest .
+docker run -t antlr4-native-rb:latest bundle exec ruby parse_test.rb

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: antlr4-native
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.2.1
 platform: ruby
 authors:
 - Cameron Dutro
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-05-12 00:00:00.000000000 Z
+date: 2023-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice
@@ -43,6 +43,15 @@ files:
 - lib/antlr4-native/string_helpers.rb
 - lib/antlr4-native/version.rb
 - lib/antlr4-native/visitor_generator.rb
+- spec/lua-parser-rb/Gemfile
+- spec/lua-parser-rb/Lua.g4
+- spec/lua-parser-rb/Rakefile
+- spec/lua-parser-rb/ext/lua_parser/extconf.rb
+- spec/lua-parser-rb/lib/lua_parser.rb
+- spec/lua-parser-rb/lib/lua_parser/version.rb
+- spec/lua-parser-rb/lua_parser.gemspec
+- spec/lua-parser-rb/parse_test.rb
+- spec/run-lua-test.sh
 - vendor/antlr-4.10.1-complete.jar
 homepage: http://github.com/camertron/antlr4-native-rb
 licenses: []
@@ -62,7 +71,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.4.5
 signing_key:
 specification_version: 4
 summary: Create a Ruby native extension from any ANTLR4 grammar.