python3-parser 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 64bd94c7c78eb31b83c654763d4c850d2c8ea3b9ae219169b24ff9a26a8747db
+  data.tar.gz: e19434ed9e852798b30f7688c093d2527baa7a3ac337898b74b8b53e015f49b4
+SHA512:
+  metadata.gz: d3d6f9737246323e4a93dfb54ca95960e27ef0f7adee22b7b00e5b1d880e70ebb6e97f0a705d132c2bd8df21088bd030673110c64745777878a4743aabdd2deb
+  data.tar.gz: 8b88052b4973af6cd75b9c5a39f5eef107da4992df0dce963bdd03dc44d48b9e3d48a129a469b7301625afb0e1a07f00254ef5a9b5d2a80894a54c204048bdcc

data/Gemfile

@@ -0,0 +1,13 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :development do
+  gem 'antlr4-native'
+  gem 'pry-byebug'
+end
+
+group :development, :test do
+  gem 'rake'
+  gem 'rspec'
+end

data/README.md

@@ -0,0 +1,59 @@
+# python3-parser-rb
+
+A Python 3 parser for Ruby.
+
+## Usage
+
+Parse some Python 3 code and walk over the parse tree with a visitor:
+
+```ruby
+require 'python3-parser'
+
+# from https://rosettacode.org/wiki/Roman_numerals/Decode#Python
+python_code = <<~END
+  _rdecode = dict(zip('MDCLXVI', (1000, 500, 100, 50, 10, 5, 1)))
+
+  def decode(roman):
+      result = 0
+      for r, r1 in zip(roman, roman[1:]):
+          rd, rd1 = _rdecode[r], _rdecode[r1]
+          result += -rd if rd < rd1 else rd
+      return result + _rdecode[roman[-1]]
+
+  if __name__ == '__main__':
+      for r in 'MCMXC MMVIII MDCLXVI'.split():
+          print(r, decode(r))
+END
+
+class MyFuncVisitor < Python3Parser::Visitor
+  def visit_funcdef(ctx)
+    puts ctx.NAME.text
+    visit_children(ctx)
+  end
+end
+
+parser = Python3Parser::Parser.parse(python_code)
+parser.visit(MyFuncVisitor.new)
+```
+
+The script above should print `"decode"` when executed, since `decode` is the only function defined in the Python code snippet.
+
+## Technology
+
+Want to target Ruby with your own ANTLR grammars? Check out the [antlr-gemerator](https://github.com/camertron/antlr-gemerator).
+
+## Caveats
+
+See the caveats listed in [antlr4-native's README](https://github.com/camertron/antlr4-native-rb#caveats).
+
+## System Requirements
+
+See the system requirements listed in [antlr4-native's README](https://github.com/camertron/antlr4-native-rb#system-requirements).
+
+## License
+
+Licensed under the MIT license. See LICENSE.txt for details.
+
+## Authors
+
+* Cameron C. Dutro: http://github.com/camertron

data/Rakefile

@@ -0,0 +1,21 @@
+require 'bundler'
+require 'antlr4-native'
+
+Bundler::GemHelper.install_tasks
+
+task :generate do
+  generator = Antlr4Native::Generator.new(
+    grammar_files:      ['Python3Lexer.g4', 'Python3Parser.g4'],
+    output_dir:         'ext',
+    parser_root_method: 'file_input'
+  )
+
+  generator.generate
+end
+
+task :compile do
+  Dir.chdir(File.join(%w(ext python3-parser))) do
+    load 'extconf.rb'
+    exec 'make -j 4'
+  end
+end

data/ext/python3-parser/Python3LexerBase.cpp

@@ -0,0 +1,112 @@
+#include "Python3Lexer.h"
+#include "Python3Parser.h"
+
+Python3LexerBase::Python3LexerBase(antlr4::CharStream *input) : Lexer(input) {
+  tokens = new std::list<antlr4::Token*>();
+  indents = new std::stack<int>();
+  lastToken = nullptr;
+}
+
+Python3LexerBase::~Python3LexerBase() {
+  for (auto i = tokens -> begin(); i != tokens -> end(); i ++) {
+    delete *i;
+  }
+
+  delete tokens;
+  delete indents;
+}
+
+bool Python3LexerBase::atStartOfInput() {
+  return getCharPositionInLine() == 0 && getLine() == 1;
+}
+
+void Python3LexerBase::emit(std::unique_ptr<antlr4::Token> newToken) {
+  auto copy = new antlr4::CommonToken(newToken.get());
+  tokens -> push_back(copy);
+  antlr4::Lexer::emit(std::move(newToken));
+}
+
+void Python3LexerBase::emitp(antlr4::Token* newToken) {
+  auto copy = new antlr4::CommonToken(newToken);
+  tokens -> push_back(copy);
+
+  auto uniqueToken = std::unique_ptr<antlr4::Token>(newToken);
+  antlr4::Lexer::emit(std::move(uniqueToken));
+}
+
+std::unique_ptr<antlr4::Token> Python3LexerBase::nextToken() {
+  // Check if the end-of-file is ahead and there are still some DEDENTS expected.
+  if (_input -> LA(1) == Python3Parser::EOF && !indents -> empty()) {
+    // Remove any trailing EOF tokens from our buffer.
+    tokens -> remove_if(
+      [](auto&& cur) {
+        return cur -> getType() == Python3Parser::EOF;
+      }
+    );
+
+    // // First emit an extra line break that serves as the end of the statement.
+    emitp(commonToken(Python3Parser::NEWLINE, "\n"));
+
+    // Now emit as much DEDENT tokens as needed.
+    while (!indents -> empty()) {
+      emitp(createDedent());
+      indents -> pop();
+    }
+
+    // // Put the EOF back on the token stream.
+    emitp(commonToken(Python3Parser::EOF, "<EOF>"));
+  }
+
+  auto next = antlr4::Lexer::nextToken();
+
+  if (next.get() -> getChannel() == antlr4::Token::DEFAULT_CHANNEL) {
+    // Keep track of the last token on the default channel.
+    lastToken = new antlr4::CommonToken(next.get());
+  }
+
+  if (tokens -> empty()) {
+    return next;
+   } else {
+    auto frontToken = std::unique_ptr<antlr4::Token>(tokens -> front());
+    tokens -> pop_front();
+    return frontToken;
+  }
+}
+
+antlr4::CommonToken* Python3LexerBase::commonToken(size_t type, std::string text) {
+  int stop = getCharIndex() - 1;
+  int start = text.empty() ? stop : stop - text.length() + 1;
+  return new antlr4::CommonToken({ this, _input }, type, antlr4::Lexer::DEFAULT_TOKEN_CHANNEL, start, stop);
+}
+
+antlr4::CommonToken* Python3LexerBase::createDedent() {
+  auto dedent = commonToken(Python3Parser::DEDENT, "");
+  dedent -> setLine(lastToken -> getLine());
+  return dedent;
+}
+
+// Calculates the indentation of the provided spaces, taking the
+// following rules into account:
+//
+// "Tabs are replaced (from left to right) by one to eight spaces
+//  such that the total number of characters up to and including
+//  the replacement is a multiple of eight [...]"
+//
+//  -- https://docs.python.org/3.1/reference/lexical_analysis.html#indentation
+int Python3LexerBase::getIndentationCount(std::string spaces) {
+  int count = 0;
+
+  for (size_t i = 0; i < spaces.length(); i ++) {
+    switch (spaces.at(i)) {
+      case '\t':
+        count += 8 - (count % 8);
+        break;
+
+      default:
+        // A normal space char.
+        count ++;
+    }
+  }
+
+  return count;
+}

data/ext/python3-parser/Python3LexerBase.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <list>
+#include <stack>
+
+#include "antlr4-runtime.h"
+
+class Python3LexerBase : public antlr4::Lexer {
+public:
+  Python3LexerBase(antlr4::CharStream *input);
+  ~Python3LexerBase();
+
+  virtual void emitp(antlr4::Token* newToken);
+
+  virtual void emit(std::unique_ptr<antlr4::Token> newToken) override;
+  virtual std::unique_ptr<antlr4::Token> nextToken() override;
+
+  bool atStartOfInput();
+
+  static int getIndentationCount(std::string spaces);
+
+protected:
+  antlr4::CommonToken* commonToken(size_t type, std::string text);
+  antlr4::CommonToken* createDedent();
+
+  // A queue where extra tokens are pushed on (see the NEWLINE lexer rule).
+  std::list<antlr4::Token*>* tokens;
+  // The stack that keeps track of the indentation level.
+  std::stack<int>* indents;
+  // The amount of opened braces, brackets and parenthesis.
+  int opened = 0;
+  // The most recently produced token.
+  antlr4::Token* lastToken;
+};

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/ANTLRErrorListener.cpp

@@ -0,0 +1,10 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#include "ANTLRErrorListener.h"
+
+antlr4::ANTLRErrorListener::~ANTLRErrorListener()
+{
+}

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/ANTLRErrorListener.h

@@ -0,0 +1,167 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#pragma once
+
+#include "RecognitionException.h"
+
+namespace antlrcpp {
+  class BitSet;
+}
+
+namespace antlr4 {
+
+  /// How to emit recognition errors (an interface in Java).
+  class ANTLR4CPP_PUBLIC ANTLRErrorListener {
+  public:
+    virtual ~ANTLRErrorListener();
+
+    /// <summary>
+    /// Upon syntax error, notify any interested parties. This is not how to
+    /// recover from errors or compute error messages. <seealso cref="ANTLRErrorStrategy"/>
+    /// specifies how to recover from syntax errors and how to compute error
+    /// messages. This listener's job is simply to emit a computed message,
+    /// though it has enough information to create its own message in many cases.
+    /// <p/>
+    /// The <seealso cref="RecognitionException"/> is non-null for all syntax errors except
+    /// when we discover mismatched token errors that we can recover from
+    /// in-line, without returning from the surrounding rule (via the single
+    /// token insertion and deletion mechanism).
+    /// </summary>
+    /// <param name="recognizer">
+    ///        What parser got the error. From this
+    /// 		  object, you can access the context as well
+    /// 		  as the input stream. </param>
+    /// <param name="offendingSymbol">
+    ///        The offending token in the input token
+    /// 		  stream, unless recognizer is a lexer (then it's null). If
+    /// 		  no viable alternative error, {@code e} has token at which we
+    /// 		  started production for the decision. </param>
+    /// <param name="line">
+    /// 		  The line number in the input where the error occurred. </param>
+    /// <param name="charPositionInLine">
+    /// 		  The character position within that line where the error occurred. </param>
+    /// <param name="msg">
+    /// 		  The message to emit. </param>
+    /// <param name="e">
+    ///        The exception generated by the parser that led to
+    ///        the reporting of an error. It is null in the case where
+    ///        the parser was able to recover in line without exiting the
+    ///        surrounding rule. </param>
+    virtual void syntaxError(Recognizer *recognizer, Token *offendingSymbol, size_t line,
+                             size_t charPositionInLine, const std::string &msg, std::exception_ptr e) = 0;
+
+    /**
+     * This method is called by the parser when a full-context prediction
+     * results in an ambiguity.
+     *
+     * <p>Each full-context prediction which does not result in a syntax error
+     * will call either {@link #reportContextSensitivity} or
+     * {@link #reportAmbiguity}.</p>
+     *
+     * <p>When {@code ambigAlts} is not null, it contains the set of potentially
+     * viable alternatives identified by the prediction algorithm. When
+     * {@code ambigAlts} is null, use {@link ATNConfigSet#getAlts} to obtain the
+     * represented alternatives from the {@code configs} argument.</p>
+     *
+     * <p>When {@code exact} is {@code true}, <em>all</em> of the potentially
+     * viable alternatives are truly viable, i.e. this is reporting an exact
+     * ambiguity. When {@code exact} is {@code false}, <em>at least two</em> of
+     * the potentially viable alternatives are viable for the current input, but
+     * the prediction algorithm terminated as soon as it determined that at
+     * least the <em>minimum</em> potentially viable alternative is truly
+     * viable.</p>
+     *
+     * <p>When the {@link PredictionMode#LL_EXACT_AMBIG_DETECTION} prediction
+     * mode is used, the parser is required to identify exact ambiguities so
+     * {@code exact} will always be {@code true}.</p>
+     *
+     * <p>This method is not used by lexers.</p>
+     *
+     * @param recognizer the parser instance
+     * @param dfa the DFA for the current decision
+     * @param startIndex the input index where the decision started
+     * @param stopIndex the input input where the ambiguity was identified
+     * @param exact {@code true} if the ambiguity is exactly known, otherwise
+     * {@code false}. This is always {@code true} when
+     * {@link PredictionMode#LL_EXACT_AMBIG_DETECTION} is used.
+     * @param ambigAlts the potentially ambiguous alternatives, or {@code null}
+     * to indicate that the potentially ambiguous alternatives are the complete
+     * set of represented alternatives in {@code configs}
+     * @param configs the ATN configuration set where the ambiguity was
+     * identified
+     */
+    virtual void reportAmbiguity(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex, bool exact,
+      const antlrcpp::BitSet &ambigAlts, atn::ATNConfigSet *configs) = 0;
+
+    /**
+     * This method is called when an SLL conflict occurs and the parser is about
+     * to use the full context information to make an LL decision.
+     *
+     * <p>If one or more configurations in {@code configs} contains a semantic
+     * predicate, the predicates are evaluated before this method is called. The
+     * subset of alternatives which are still viable after predicates are
+     * evaluated is reported in {@code conflictingAlts}.</p>
+     *
+     * <p>This method is not used by lexers.</p>
+     *
+     * @param recognizer the parser instance
+     * @param dfa the DFA for the current decision
+     * @param startIndex the input index where the decision started
+     * @param stopIndex the input index where the SLL conflict occurred
+     * @param conflictingAlts The specific conflicting alternatives. If this is
+     * {@code null}, the conflicting alternatives are all alternatives
+     * represented in {@code configs}. At the moment, conflictingAlts is non-null
+     * (for the reference implementation, but Sam's optimized version can see this
+     * as null).
+     * @param configs the ATN configuration set where the SLL conflict was
+     * detected
+     */
+    virtual void reportAttemptingFullContext(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex,
+      const antlrcpp::BitSet &conflictingAlts, atn::ATNConfigSet *configs) = 0;
+
+    /**
+     * This method is called by the parser when a full-context prediction has a
+     * unique result.
+     *
+     * <p>Each full-context prediction which does not result in a syntax error
+     * will call either {@link #reportContextSensitivity} or
+     * {@link #reportAmbiguity}.</p>
+     *
+     * <p>For prediction implementations that only evaluate full-context
+     * predictions when an SLL conflict is found (including the default
+     * {@link ParserATNSimulator} implementation), this method reports cases
+     * where SLL conflicts were resolved to unique full-context predictions,
+     * i.e. the decision was context-sensitive. This report does not necessarily
+     * indicate a problem, and it may appear even in completely unambiguous
+     * grammars.</p>
+     *
+     * <p>{@code configs} may have more than one represented alternative if the
+     * full-context prediction algorithm does not evaluate predicates before
+     * beginning the full-context prediction. In all cases, the final prediction
+     * is passed as the {@code prediction} argument.</p>
+     *
+     * <p>Note that the definition of "context sensitivity" in this method
+     * differs from the concept in {@link DecisionInfo#contextSensitivities}.
+     * This method reports all instances where an SLL conflict occurred but LL
+     * parsing produced a unique result, whether or not that unique result
+     * matches the minimum alternative in the SLL conflicting set.</p>
+     *
+     * <p>This method is not used by lexers.</p>
+     *
+     * @param recognizer the parser instance
+     * @param dfa the DFA for the current decision
+     * @param startIndex the input index where the decision started
+     * @param stopIndex the input index where the context sensitivity was
+     * finally determined
+     * @param prediction the unambiguous result of the full-context prediction
+     * @param configs the ATN configuration set where the unambiguous prediction
+     * was determined
+     */
+    virtual void reportContextSensitivity(Parser *recognizer, const dfa::DFA &dfa, size_t startIndex, size_t stopIndex,
+      size_t prediction, atn::ATNConfigSet *configs) = 0;
+  };
+
+} // namespace antlr4

data/ext/python3-parser/antlr4-upstream/runtime/Cpp/runtime/src/ANTLRErrorStrategy.cpp

@@ -0,0 +1,10 @@
+/* Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+#include "ANTLRErrorStrategy.h"
+
+antlr4::ANTLRErrorStrategy::~ANTLRErrorStrategy()
+{
+}