expressir 1.4.3 → 2.1.0

data/ext/express_parser/extconf.rb

@@ -1,63 +0,0 @@
-require "rubygems"
-require "mkmf"
-
-extension_name = "express_parser"
-cross_build =  enable_config("cross-build")
-
-antlr4_src = "antlr4-upstream/runtime/Cpp/runtime/src"
-src_paths = [
-  ".",
-  "antlrgen",
-  antlr4_src.to_s,
-  "#{antlr4_src}/atn",
-  "#{antlr4_src}/dfa",
-  "#{antlr4_src}/internal",
-  "#{antlr4_src}/misc",
-  "#{antlr4_src}/support",
-  "#{antlr4_src}/tree",
-  "#{antlr4_src}/tree/pattern",
-  "#{antlr4_src}/tree/xpath",
-]
-
-if cross_build
-  rice_subdir = "rice-embed"
-  rice_root = Gem.loaded_specs["rice"].full_gem_path
-  rice_src = File.join(rice_root, "include", "rice")
-  rice_embed = File.join(__dir__, rice_subdir)
-  unless File.exist?(rice_embed)
-    FileUtils.mkdir_p(rice_embed)
-    FileUtils.cp_r(Dir.glob(rice_src), rice_embed)
-    File.delete(*Dir.glob("#{rice_embed}/**/*.o"))
-    File.delete(*Dir.glob("#{rice_embed}/**/Makefile"))
-  end
-
-  src_paths.push("#{rice_subdir}/include")
-
-  $INCFLAGS << " -I#{__dir__}/#{rice_subdir}"
-
-  case RbConfig::CONFIG["target_os"]
-  when /mingw32|mswin/
-    # workaround for 'w64-mingw32-as: express_parser.o: too many sections'
-    $CXXFLAGS << " -Wa,-mbig-obj"
-    # workaround for LoadError: 127: The specified procedure could not be found.
-    $DLDFLAGS << " -static-libgcc -static-libstdc++"
-  when /darwin/
-    $CXXFLAGS << " -mmacosx-version-min=10.15 -fno-c++-static-destructors"
-    $DLDFLAGS << " -mmacosx-version-min=10.15"
-  end
-else
-  require "mkmf-rice"
-  dir_config(extension_name)
-end
-
-$CPPFLAGS << " -std=c++17 -DANTLR4CPP_STATIC -DHAVE_CXX11 -O2 -fno-omit-frame-pointer"
-$INCFLAGS << " -I#{__dir__}/#{antlr4_src}"
-$srcs = []
-
-src_paths.each do |src_path|
-  abs_src_path = File.join(__dir__, src_path)
-  $VPATH << abs_src_path
-  $srcs.push(*Dir["#{abs_src_path}/*.cpp"])
-end
-
-create_makefile "expressir/express/#{extension_name}"

data/rakelib/antlr4-native.rake

@@ -1,173 +0,0 @@
-require "fileutils"
-require "antlr4-native"
-require "rake"
-
-def create_class_declarations(parser_source_lines)
-  i = parser_source_lines.index { |x| x == "Class rb_cContextProxy;" }
-  parser_source_lines[i] += <<~CPP.split("\n").map { |x| x == "" ? x : x.to_s }.join("\n")
-
-    Class rb_cParserExt;
-    Class rb_cTokenExt;
-
-  CPP
-end
-
-def create_tp_class_definition(parser_source_lines)
-  i = parser_source_lines.index { |x| x == "class ContextProxy {" }
-  parser_source_lines[i - 2] += <<~CPP.split("\n").map { |x| x == "" ? x : x.to_s }.join("\n")
-
-
-    class TokenProxy : public Object {
-      public:
-        TokenProxy(Token* orig) {
-          this -> orig = orig;
-        }
-
-        std::string getText() {
-          return orig->getText();
-        }
-
-        size_t getChannel() {
-          return orig->getChannel();
-        }
-
-        size_t getTokenIndex() {
-          return orig->getTokenIndex();
-        }
-
-      private:
-        Token * orig = nullptr;
-    };
-
-  CPP
-end
-
-def create_pp_class_definition(parser_source_lines)
-  i = parser_source_lines.index { |x| x == "extern \"C\"" }
-  parser_source_lines[i - 2] += <<~CPP.split("\n").map { |x| x == "" ? x : x.to_s }.join("\n")
-
-    class ParserProxyExt : public Object {
-      public:
-        ParserProxyExt(string file) {
-          ifstream stream;
-          stream.open(file);
-          input = new ANTLRInputStream(stream);
-          lexer = new ExpressLexer(input);
-          tokens = new CommonTokenStream(lexer);
-          parser = new ExpressParser(tokens);
-          stream.close();
-        };
-
-        ~ParserProxyExt() {
-          delete parser;
-          delete tokens;
-          delete lexer;
-          delete input;
-        }
-
-        Object syntax() {
-          auto ctx = parser -> syntax();
-
-          SyntaxContextProxy proxy((ExpressParser::SyntaxContext*) ctx);
-          return detail::To_Ruby<SyntaxContextProxy>().convert(proxy);
-        }
-
-        Object getTokens() {
-          std::vector<TokenProxy> tk;
-          for (auto token : tokens -> getTokens()) {
-            tk.push_back(TokenProxy(token));
-          }
-          return detail::To_Ruby<std::vector<TokenProxy>>().convert(tk);
-        }
-
-        Object visit(VisitorProxy* visitor) {
-          auto result = visitor -> visit(parser -> syntax());
-
-          lexer -> reset();
-          parser -> reset();
-
-          try {
-            return std::any_cast<Object>(result);
-          } catch(std::bad_any_cast) {
-            return Qnil;
-          }
-        }
-
-      private:
-        ANTLRInputStream* input;
-        ExpressLexer* lexer;
-        CommonTokenStream* tokens;
-        ExpressParser* parser;
-    };
-
-
-  CPP
-end
-
-def create_class_api(parser_source_lines)
-  i = parser_source_lines.index { |x| x == "    .define_method(\"visit\", &ParserProxy::visit);" }
-  parser_source_lines[i] += <<~CPP.split("\n").map { |x| x == "" ? x : "  #{x}" }.join("\n")
-
-
-    rb_cTokenExt = define_class_under<TokenProxy>(rb_mExpressParser, "TokenExt")
-      .define_method("text", &TokenProxy::getText)
-      .define_method("channel", &TokenProxy::getChannel)
-      .define_method("token_index", &TokenProxy::getTokenIndex);
-
-    rb_cParserExt = define_class_under<ParserProxyExt>(rb_mExpressParser, "ParserExt")
-      .define_constructor(Constructor<ParserProxyExt, string>())
-      .define_method("syntax", &ParserProxyExt::syntax)
-      .define_method("tokens", &ParserProxyExt::getTokens, Return().keepAlive())
-      .define_method("visit", &ParserProxyExt::visit);
-
-    define_vector<std::vector<TokenProxy>>("TokenVector");
-
-  CPP
-end
-
-def create_vector_definition(parser_source_lines)
-  i = parser_source_lines.index { |x| x == "    .define_method(\"visit\", &ParserProxy::visit);" }
-  parser_source_lines[i] += <<~CPP.split("\n").map { |x| x == "" ? x : "  #{x}" }.join("\n")
-
-  CPP
-end
-
-def generate_extended_parser
-  # Generate extended parser that provide Ruby access to token stream
-  parser_source_file = File.join("ext", "express_parser", "express_parser.cpp")
-  parser_source_lines = File.read(parser_source_file)
-    .gsub!("bad_cast", "bad_any_cast")
-    .gsub!("return detail::To_Ruby<Token*>().convert(token)", "return detail::To_Ruby<TokenProxy>().convert(TokenProxy(token))")
-    .split("\n")
-  create_class_declarations(parser_source_lines)
-  create_tp_class_definition(parser_source_lines)
-  create_pp_class_definition(parser_source_lines)
-  create_class_api(parser_source_lines)
-  File.write(parser_source_file, "#{parser_source_lines.join("\n")}\n")
-end
-
-desc "Generate parser  (Usage: 'rake generate <grammar_file>')"
-task "generate" do
-  grammar_file = ARGV[1]
-  if grammar_file.nil?
-    grammar_file = File.expand_path(File.join("..", "ext", "express-grammar", "Express.g4"), __dir__)
-  end
-
-  puts "Generating parser from '#{grammar_file}'"
-  # ANTLR does weird things if the grammar file isn't in the current working directory
-  temp_grammar_file = File.join(FileUtils.pwd, File.basename(grammar_file))
-  FileUtils.cp(grammar_file, temp_grammar_file)
-
-  # generate parser
-  generator = Antlr4Native::Generator.new(
-    grammar_files: [File.basename(temp_grammar_file)],
-    output_dir: "ext",
-    parser_root_method: "syntax",
-  )
-  generator.generate
-
-  puts "Generating extended parser"
-  generate_extended_parser
-  # cleanup
-  FileUtils.rm(temp_grammar_file)
-end

data/rakelib/cross-ruby.rake

@@ -1,403 +0,0 @@
-require "rbconfig"
-require "shellwords"
-
-WINDOWS_PLATFORM_REGEX = /mingw|mswin/.freeze
-LINUX_GNU_PLATFORM_REGEX = /linux-gnu/.freeze
-LINUX_MUSL_PLATFORM_REGEX = /linux-musl/.freeze
-DARWIN_PLATFORM_REGEX = /darwin/.freeze
-GLIBC_MIN_VERSION = "2.17".freeze
-
-CrossRuby = Struct.new(:version, :host) do
-  def dll_staging_path
-    "tmp/#{platform}/stage/lib/expressir/express/#{minor_ver}/express_parser.#{dll_ext}"
-  end
-
-  def darwin?
-    !!(platform =~ DARWIN_PLATFORM_REGEX)
-  end
-
-  def linux_gnu?
-    !!(platform =~ LINUX_GNU_PLATFORM_REGEX)
-  end
-
-  def linux_musl?
-    !!(platform =~ LINUX_MUSL_PLATFORM_REGEX)
-  end
-
-  def windows?
-    !!(platform =~ WINDOWS_PLATFORM_REGEX)
-  end
-
-  def ver
-    @ver ||= version[/\A[^-]+/]
-  end
-
-  def minor_ver
-    @minor_ver ||= ver[/\A\d\.\d(?=\.)/]
-  end
-
-  def minor_ver_digi
-    @minor_ver_digi = minor_ver.delete(".").to_i
-  end
-
-  def ucrt?
-    minor_ver_digi >= 31
-  end
-
-  def api_ver_suffix
-    case minor_ver
-    when nil
-      raise "CrossRuby.api_ver_suffix: unsupported version: #{ver}"
-    else
-      minor_ver.delete(".") << "0"
-    end
-  end
-
-  # rubocop:disable Metrics/MethodLength
-  # rubocop:disable Metrics/CyclomaticComplexity
-  def platform
-    @platform ||=
-      case host
-      when /\Ax86_64.*mingw32/
-        if ucrt?
-          "x64-mingw-ucrt"
-        else
-          "x64-mingw32"
-        end
-      when "x86_64-linux-gnu"
-        "x86_64-linux-gnu"
-      when "aarch64-linux-gnu"
-        "aarch64-linux-gnu"
-      when "x86_64-linux-musl"
-        "x86_64-linux-musl"
-      when "aarch64-linux-musl"
-        "aarch64-linux-musl"
-      when /\Ax86_64-darwin/
-        "x86_64-darwin"
-      when /\Aarm64-darwin/
-        "arm64-darwin"
-      else
-        raise "CrossRuby.platform: unsupported host: #{host}"
-      end
-  end
-
-  def tag
-    @tag ||=
-      case platform
-      when "x86_64-linux-gnu"
-        "x86_64-linux"
-      when "aarch64-linux-gnu"
-        "aarch64-linux"
-      else
-        platform
-      end
-  end
-
-  def tool(name)
-    (@binutils_prefix ||=
-       case platform
-       when /x64-mingw(32|-ucrt)/
-         "x86_64-w64-mingw32-"
-       when /(x86_64|aarch64)-linux-(gnu|musl)/
-         # We do believe that we are on Linux and can use native tools
-         ""
-       when /x86_64.*darwin/
-         "x86_64-apple-darwin-"
-       when /a.*64.*darwin/
-         "aarch64-apple-darwin-"
-       else
-         raise "CrossRuby.tool: unmatched platform: #{platform}"
-       end) + name
-  end
-
-  def target_file_format
-    case platform
-    when /64-mingw(32|-ucrt)/
-      "pei-x86-64"
-    when /x86_64-linux-(gnu|musl)/
-      "elf64-x86-64"
-    when /aarch64-linux-(gnu|musl)/
-      "elf64-little"
-    when "x86_64-darwin"
-      "Mach-O 64-bit x86-64"
-    when "arm64-darwin"
-      "Mach-O arm64"
-    else
-      raise "CrossRuby.target_file_format: unmatched platform: #{platform}"
-    end
-  end
-  # rubocop:enable Metrics/MethodLength
-  # rubocop:enable Metrics/CyclomaticComplexity
-
-  def dll_ext
-    darwin? ? "bundle" : "so"
-  end
-
-  def verify_format(dump, dll)
-    format_match = (/file format #{Regexp.quote(target_file_format)}\s/ === dump)
-    format_error = "Unexpected file format for '#{dll}', '#{target_file_format}' required"
-    raise format_error unless format_match
-  end
-
-  def verify_entry_windows(dump, dll)
-    unless /Table.*\sInit_express_parser\s/mi === dump
-      raise "Export function Init_express_parser not in dll #{dll}"
-    end
-  end
-
-  def verify_entry_linux(dll)
-    nm = `#{["env", "LANG=C", tool("nm"), "-D", dll].shelljoin}`
-    unless nm.include?(" T Init_express_parser")
-      raise "Export function Init_express_parser not in dll #{dll}"
-    end
-  end
-
-  def verify_entry_darwin(dll)
-    nm = `#{["env", "LANG=C", tool("nm"), "-g", dll].shelljoin}`
-    unless / T _?Init_express_parser/.match?(nm)
-      raise "Export function Init_express_parser not in dll #{dll}"
-    end
-  end
-
-  def verify_entry(dump, dll)
-    case platform
-    when WINDOWS_PLATFORM_REGEX
-      verify_entry_windows(dump, dll)
-    when LINUX_GNU_PLATFORM_REGEX, LINUX_MUSL_PLATFORM_REGEX
-      verify_entry_linux(dll)
-    when DARWIN_PLATFORM_REGEX
-      verify_entry_darwin(dll)
-    else
-      raise "CrossRuby.verify_entry: unmatched platform: #{platform}"
-    end
-  end
-
-  # rubocop:disable Metrics/MethodLength
-  def allowed_dlls_ucrt
-    ["kernel32.dll",
-     "api-ms-win-crt-convert-l1-1-0.dll",
-     "api-ms-win-crt-environment-l1-1-0.dll",
-     "api-ms-win-crt-heap-l1-1-0.dll",
-     "api-ms-win-crt-locale-l1-1-0.dll",
-     "api-ms-win-crt-private-l1-1-0.dll",
-     "api-ms-win-crt-runtime-l1-1-0.dll",
-     "api-ms-win-crt-stdio-l1-1-0.dll",
-     "api-ms-win-crt-string-l1-1-0.dll",
-     "api-ms-win-crt-time-l1-1-0.dll",
-     "api-ms-win-crt-filesystem-l1-1-0.dll",
-     "api-ms-win-crt-math-l1-1-0.dll",
-     "libwinpthread-1.dll",
-     "x64-ucrt-ruby#{api_ver_suffix}.dll"]
-  end
-  # rubocop:enable Metrics/MethodLength
-
-  def allowed_dlls_mingw
-    [
-      "kernel32.dll",
-      "msvcrt.dll",
-      "libwinpthread-1.dll",
-      "x64-msvcrt-ruby#{api_ver_suffix}.dll",
-    ]
-  end
-
-  def allowed_dlls_windows
-    if ucrt?
-      allowed_dlls_ucrt
-    else
-      allowed_dlls_mingw
-    end
-  end
-
-  def allowed_dlls_linux_gnu
-    suffix = (platform == "x86_64-linux-gnu" ? "x86-64" : "aarch64")
-    [
-      "ld-linux-#{suffix}.so",
-      "libc.so",
-      "libm.so",
-      "libstdc++.so",
-      "libgcc_s.so",
-      "libpthread.so",
-    ]
-  end
-
-  def allowed_dlls_linux_musl
-    suffix = (platform == "x86_64-linux-gnu" ? "x86-64" : "aarch64")
-    [
-      "ld-linux-#{suffix}.so",
-      "libstdc++.so",
-      "libc.so",
-      "libgcc_s.so",
-    ]
-  end
-
-  def allowed_dlls_darwin
-    [
-      "/usr/lib/libSystem.B.dylib",
-      "/usr/lib/libc++.1.dylib",
-    ]
-  end
-
-  def allowed_dlls # rubocop:disable Metrics/MethodLength
-    case platform
-    when WINDOWS_PLATFORM_REGEX
-      allowed_dlls_windows
-    when LINUX_GNU_PLATFORM_REGEX
-      allowed_dlls_linux_gnu
-    when LINUX_MUSL_PLATFORM_REGEX
-      allowed_dlls_linux_musl
-    when DARWIN_PLATFORM_REGEX
-      allowed_dlls_darwin
-    else
-      raise "CrossRuby.allowed_dlls: unmatched platform: #{platform}"
-    end
-  end
-
-  def actual_dlls_linux(dump)
-    dump.scan(/NEEDED\s+(.*)/).map(&:first).uniq
-  end
-
-  def actual_dlls_windows(dump)
-    dump.scan(/DLL Name: (.*)$/).map { |x| x.first.downcase }.uniq
-  end
-
-  def actual_dlls_darwin(dll)
-    ldd = `#{[tool("otool"), "-L", dll].shelljoin}`
-    ldd.scan(/^\t([^ ]+) /).map(&:first).uniq
-  end
-
-  def actual_dlls(dump, dll)
-    case platform
-    when DARWIN_PLATFORM_REGEX
-      actual_dlls_darwin(dll)
-    when LINUX_GNU_PLATFORM_REGEX, LINUX_MUSL_PLATFORM_REGEX
-      actual_dlls_linux(dump)
-    when WINDOWS_PLATFORM_REGEX
-      actual_dlls_windows(dump)
-    else
-      raise "CrossRuby.actual_dlls: unmatched platform: #{platform}"
-    end
-  end
-
-  def verify_imports(dump, dll)
-    l = actual_dlls(dump, dll)
-    libs = allowed_dlls
-    l.delete_if { |ln| libs.any? { |lib| ln.include?(lib) } }
-    unless l.empty?
-      raise "Unexpected references in '#{dll}' : #{l}"
-    end
-  end
-
-  def lib_ref_versions(data)
-    # Build a hash of library versions like {"LIBUDEV"=>"183", "GLIBC"=>"2.17"}
-    data.each.with_object({}) do |(lib, ver), h|
-      if !h[lib] || ver.split(".").map(&:to_i).pack("C*") > h[lib].split(".").map(&:to_i).pack("C*")
-        h[lib] = ver
-      end
-    end
-  end
-
-  def verify_glibc_version(dump, dll)
-    ref_versions_data = dump.scan(/0x[\da-f]+ 0x[\da-f]+ \d+ (\w+)_([\d.]+)$/i)
-    ref_ver = lib_ref_versions(ref_versions_data)
-
-    unless ref_ver["GLIBC"].delete(".").to_i <= GLIBC_MIN_VERSION.delete(".").to_i
-      raise "Unexpected GLIBC version #{ref_ver['GLIBC']} for #{dll},  #{GLIBC_MIN_VERSION} or lower is expected"
-    end
-  end
-end
-
-CROSS_RUBIES = File.read(".cross_rubies").split("\n").filter_map do |line|
-  case line
-  when /\A([^#]+):([^#]+)/
-    CrossRuby.new($1, $2)
-  end
-end
-
-ENV["RUBY_CC_VERSION"] = CROSS_RUBIES.map(&:ver).uniq.join(":")
-
-require "rake_compiler_dock"
-
-def verify_dll(dll, cross_ruby)
-  dump = `#{["env", "LANG=C", cross_ruby.tool("objdump"), "-p", dll].shelljoin}`
-  cross_ruby.verify_format(dump, dll)
-  cross_ruby.verify_entry(dump, dll)
-  cross_ruby.verify_imports(dump, dll)
-  #  Not sure if it is required, probably not
-  #  I am keeping related code as a reference for future advances
-  #  cross_ruby.verify_glibc_version(dump, dll) if cross_ruby.linux_gnu?
-
-  puts "#{dll}: passed shared library sanity checks"
-end
-
-CROSS_RUBIES.each do |cross_ruby|
-  unless Rake::Task.task_defined?(cross_ruby.dll_staging_path)
-    task cross_ruby.dll_staging_path do |t|
-      verify_dll t.name, cross_ruby
-    end
-  end
-end
-
-def gem_builder(plat)
-  # use Task#invoke because the pkg/*gem task is defined at runtime
-  Rake::Task["native:#{plat}"].invoke
-  Rake::Task["pkg/#{GEMSPEC.full_name}-#{Gem::Platform.new(plat)}.gem"].invoke
-end
-
-REDHAT_PREREQ = "sudo yum install -y git".freeze
-UBUNTU_PREREQ = "sudo apt-get update -y && sudo apt-get install -y automake autoconf libtool build-essential".freeze
-
-def pre_req(plat)
-  case plat
-  when /linux/
-    "if [[ $(awk -F= '/^NAME/{print $2}' /etc/os-release) == '\"Ubuntu\"' ]]; then #{UBUNTU_PREREQ}; else #{REDHAT_PREREQ}; fi"
-  else
-    UBUNTU_PREREQ.to_s
-  end
-end
-
-namespace "gem" do
-  CROSS_RUBIES.find_all { |cr| cr.windows? || cr.linux_gnu? || cr.linux_musl? || cr.darwin? }.map(&:platform).uniq.each do |plat|
-    desc "build native gem for #{plat} platform"
-    task plat do
-      RakeCompilerDock.sh <<~RCD, platform: plat
-        #{pre_req(plat)} && gem install bundler --no-document &&
-        bundle && bundle exec rake gem:#{plat}:builder MAKE="nice make -j`nproc`"
-      RCD
-    end
-
-    namespace plat do
-      desc "build native gem for #{plat} platform (guest container)"
-      task "builder" do
-        gem_builder(plat)
-      end
-    end
-  end
-
-  desc "build native gems for windows"
-  multitask "windows" => CROSS_RUBIES.find_all(&:windows?).map(&:platform).uniq
-
-  desc "build native gems for linux-gnu"
-  multitask "linux-gnu" => CROSS_RUBIES.find_all(&:linux_gnu?).map(&:platform).uniq
-
-  desc "build native gems for linux-musl"
-  multitask "linux-musl" => CROSS_RUBIES.find_all(&:linux_musl?).map(&:platform).uniq
-
-  desc "build native gems for darwin"
-  multitask "darwin" => CROSS_RUBIES.find_all(&:darwin?).map(&:platform).uniq
-end
-
-require "rake/extensiontask"
-
-Rake::ExtensionTask.new("express_parser", GEMSPEC) do |ext|
-  ext.ext_dir = "ext/express_parser"
-  ext.lib_dir = File.join(*["lib", "expressir", "express", ENV.fetch("FAT_DIR", nil)].compact)
-  ext.config_options << ENV.fetch("EXTOPTS", nil)
-  ext.cross_compile  = true
-  ext.cross_platform = CROSS_RUBIES.map(&:platform).uniq
-  ext.cross_config_options << "--enable-cross-build"
-
-  ext.cross_compiling do |spec|
-    spec.files.reject! { |path| File.fnmatch?("ext/*", path) }
-    spec.dependencies.reject! { |dep| dep.name == "rice" }
-  end
-end

data/spec/acceptance/version_spec.rb

@@ -1,30 +0,0 @@
-require "spec_helper"
-
-RSpec.describe "Expressir" do
-  describe "version" do
-    it "has a version number" do |example|
-      GC.stress = ENV["GC_STRESS"] == "true"
-      puts "Running tests in GC stress mode. It may take a couple of hours ..." if GC.stress
-
-      print "\n[#{example.description}] "
-      expect(Expressir::VERSION).not_to be nil
-
-      # Validate Object Space
-      GC.start
-      GC.verify_compaction_references
-      GC.verify_internal_consistency
-    end
-
-    it "displays the current verison" do |example|
-      print "\n[#{example.description}] "
-      command = %w(version)
-      output = capture_stdout { Expressir::Cli.start(command) }
-      expect(output).to include(Expressir::VERSION)
-
-      # Validate Object Space
-      GC.start
-      GC.verify_compaction_references
-      GC.verify_internal_consistency
-    end
-  end
-end