expressir 1.3.0.pre.5-arm64-darwin → 1.3.2-arm64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 394364cf50899f945816b0f9e57b124c644bd25df6d553fa171b39a2b3197740
-  data.tar.gz: ce30caa38f4389d8752bfbb96852d574c9a04257aeb60b053cce737a7d948243
+  metadata.gz: 14c845e7ff989ff6fd579ec4dd2a0387da92a089955c01b8f8b92be84f9e4216
+  data.tar.gz: 65d821e16366aef97ed212e831030ad5f9fc699cd9a1b74089664d8edd8e45ea
 SHA512:
-  metadata.gz: 7f77c869bf76fb467bdea8cb6edb9a387872603eecea1a3b4c30867f7eee668cc19be34bc9a70a7ccbb637eac86f6f3af07c5c881e5e1cc65be6ba1f5e3c160d
-  data.tar.gz: f9e1049a5e9172eda80f922972f3ca794dbeb33b0fd81862f9a017f57fe02c59128fd4c3f56da367b2c7ff2a0a88969493d7a9da2bb266e42a36e95272be4ce4
+  metadata.gz: dbf20c7784c913fa15099952f1fec474fdf0097add1a1202af9ca8e62522e65a8c0adc3f2fabb25c0b9b1eb781ec2fbbaf927bdb0d18bfc9fcbdf6a0b1e747b7
+  data.tar.gz: e6fcfe824eaa26cbd881dac7f88ecf1b8000cb04676175b4ad19912b08497dd83c757072bde434f5b16490916e9dcc4559844aacf95bbd8ea5a1db1db229905f

data/.cross_rubies

@@ -1,20 +1,28 @@
 3.2.0:x86_64-w64-mingw32
 3.2.0:x86_64-linux-gnu
+3.2.0:x86_64-linux-musl
 3.2.0:aarch64-linux-gnu
+3.2.0:aarch64-linux-musl
 3.2.0:x86_64-darwin
 3.2.0:arm64-darwin
 3.1.0:x86_64-w64-mingw32
 3.1.0:x86_64-linux-gnu
+3.1.0:x86_64-linux-musl
 3.1.0:aarch64-linux-gnu
+3.1.0:aarch64-linux-musl
 3.1.0:x86_64-darwin
 3.1.0:arm64-darwin
 3.0.0:x86_64-w64-mingw32
 3.0.0:x86_64-linux-gnu
+3.0.0:x86_64-linux-musl
 3.0.0:aarch64-linux-gnu
+3.0.0:aarch64-linux-musl
 3.0.0:x86_64-darwin
 3.0.0:arm64-darwin
 2.7.0:x86_64-w64-mingw32
 2.7.0:x86_64-linux-gnu
+2.7.0:x86_64-linux-musl
 2.7.0:aarch64-linux-gnu
+2.7.0:aarch64-linux-musl
 2.7.0:x86_64-darwin
 2.7.0:arm64-darwin

data/.github/workflows/codeql.yml

@@ -0,0 +1,47 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ main ]
+  schedule:
+    - cron: '0 23 * * 3'
+  workflow_dispatch:
+
+concurrency:
+  group: '${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.ref_name }}'
+  cancel-in-progress: true
+
+env:
+  BUNDLER_VER: 2.4.22
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Install Ruby
+        uses: ruby/setup-ruby@master
+        with:
+          ruby-version: 3.1
+          bundler: ${{ env.BUNDLER_VER }}
+          bundler-cache: false
+
+      - name: Bundle
+        run: bundle install --jobs 4 --retry 3
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v2
+        with:
+          languages: "ruby, cpp"
+
+      - name: Build native extension
+        run: bundle exec rake compile
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v2

data/.github/workflows/rake.yml

@@ -8,6 +8,7 @@ on:
       - '**.adoc'
       - '**.md'
       - .github/workflows/release.yml
+      - .github/workflows/codeql.yml
   pull_request:
   workflow_dispatch:
 
@@ -16,6 +17,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
+  RUBYGEM_VER: 3.4.22
   BUNDLER_VER: 2.4.22
 # Forcing bundler version to ensure that it is consistent everywhere and
 # does not cause bundler gem reinstalls
@@ -77,11 +79,54 @@ jobs:
         run: bundle install --jobs 4 --retry 3
 
       - name: Process cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         id: cache
         with:
           path: lib/expressir/express/express_parser.*
-          key: v4-${{ matrix.os }}-${{ matrix.ruby }}-${{ hashFiles('ext/express-parser/extconf.rb', 'ext/express-parser/antlrgen/**', 'ext/express-parser/express_parser.cpp', '.git/modules/ext/express-parser/antlr4-upstream/HEAD') }}
+          key: v4-${{ matrix.os }}-${{ matrix.ruby }}-${{ hashFiles('ext/express_parser/extconf.rb', 'ext/express_parser/antlrgen/**', 'ext/express_parser/express_parser.cpp', '.git/modules/ext/express_parser/antlr4-upstream/HEAD') }}
+
+      - name: Build native extension
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: bundle exec rake compile
+
+      - name: Run tests
+        run: |
+          bundle exec rake
+          cat .rspec_status
+
+  rake-alpine:
+    name: test on ruby-${{ matrix.ruby }} alpine container
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '3.2', '3.1', '3.0', '2.7' ]
+    container: ruby:${{ matrix.ruby }}-alpine
+    steps:
+      - name: Install packages
+        run: |
+          apk --no-cache --upgrade add build-base git bash tar
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Upgrade RubyGems
+        if: matrix.ruby == '3.0' || matrix.ruby == '2.7'
+        run: gem update --system ${{ env.RUBYGEM_VER }} --no-document
+
+      - name: Bundle
+        run: |
+          git config --global --add safe.directory $PWD
+          bundle install --jobs 4 --retry 3
+
+      - name: Process cache
+        uses: actions/cache@v4
+        id: cache
+        with:
+          path: lib/expressir/express/express_parser.*
+          key: v4-alpine-${{ matrix.ruby }}-${{ hashFiles('ext/express_parser/extconf.rb', 'ext/express_parser/antlrgen/**', 'ext/express_parser/express_parser.cpp', '.git/modules/ext/express_parser/antlr4-upstream/HEAD') }}
 
       - name: Build native extension
         if: steps.cache.outputs.cache-hit != 'true'
@@ -115,7 +160,7 @@ jobs:
         run: gem build expressir.gemspec
 
       - name: Package gem without native extension
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: pkg-ruby
           path: expressir-*.gem
@@ -125,7 +170,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        platform: [ linux-gnu, windows, darwin ]
+        platform: [ linux-gnu, linux-musl, windows, darwin ]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -154,7 +199,7 @@ jobs:
         run: bundle exec rake gem:${{ matrix.platform }}
 
       - name: Package gem with native extension
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: pkg-${{ matrix.platform }}
           path: pkg/*.gem
@@ -173,7 +218,7 @@ jobs:
         uses: ruby/setup-ruby@master
         with:
           ruby-version: ${{ matrix.ruby }}
-          bundler-cache: true
+          bundler-cache: false
           bundler: ${{ env.BUNDLER_VER }}
 
       - name: Checkout
@@ -185,7 +230,7 @@ jobs:
         run: bundle install --jobs 4 --retry 3
 
       - name: Download packaged gem
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
          name: pkg-ruby
          path: pkg
@@ -195,7 +240,50 @@ jobs:
 
       - name: Verify
         run: |
-          cd $(ruby -e "puts RbConfig::TOPDIR + '\/lib\/ruby\/gems\/' + RbConfig::CONFIG['ruby_version'] + '\/gems\/expressir*'")
+          cd $(ruby -e "puts Gem::Specification.find_by_name('expressir').gem_dir")
+          ruby bin/rspec
+          cat .rspec_status || echo ".rspec_status was not found"
+
+  verify-ruby-linux-musl:
+    name: verify ruby gem on ruby-${{ matrix.ruby }} on alpine container
+    needs: pack-ruby
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '3.2', '3.1', '3.0', '2.7' ]
+    container: ruby:${{ matrix.ruby }}-alpine
+    steps:
+      - name: Install packages
+        run: |
+          apk --no-cache --upgrade add build-base git bash tar
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Upgrade RubyGems
+        if: matrix.ruby == '3.0' || matrix.ruby == '2.7'
+        run: gem update --system ${{ env.RUBYGEM_VER }} --no-document
+
+      - name: Bundle
+        run: |
+          git config --global --add safe.directory $PWD
+          bundle install --jobs 4 --retry 3
+
+      - name: Download packaged gem
+        uses: actions/download-artifact@v4
+        with:
+         name: pkg-ruby
+         path: pkg
+
+      - name: Install gem
+        run: gem install -l pkg/expressir-*.gem
+
+      - name: Verify
+        run: |
+          cd $(ruby -e "puts Gem::Specification.find_by_name('expressir').gem_dir")
           ruby bin/rspec
           cat .rspec_status || echo ".rspec_status was not found"
 
@@ -212,7 +300,7 @@ jobs:
         uses: ruby/setup-ruby@master
         with:
           ruby-version: ${{ matrix.ruby }}
-          bundler-cache: true
+          bundler-cache: false
           bundler: ${{ env.BUNDLER_VER }}
 
       - name: Checkout
@@ -222,7 +310,7 @@ jobs:
         run: bundle install --jobs 4 --retry 3
 
       - name: Download packaged gem
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
          name: pkg-darwin
          path: pkg
@@ -233,7 +321,7 @@ jobs:
 
       - name: Verify
         run: |
-          cd $(ruby -e "puts RbConfig::TOPDIR + '\/lib\/ruby\/gems\/' + RbConfig::CONFIG['ruby_version'] + '\/gems\/expressir*'")
+          cd $(ruby -e "puts Gem::Specification.find_by_name('expressir').gem_dir")
           ruby bin/rspec
           cat .rspec_status || echo ".rspec_status was not found"
 
@@ -250,9 +338,13 @@ jobs:
         uses: ruby/setup-ruby@master
         with:
           ruby-version: ${{ matrix.ruby }}
-          bundler-cache: true
+          bundler-cache: false
           bundler: ${{ env.BUNDLER_VER }}
 
+      - name: Upgrade RubyGems
+        if: matrix.ruby == '3.0' || matrix.ruby == '2.7'
+        run: gem update --system ${{ env.RUBYGEM_VER }} --no-document
+
       - name: Checkout
         uses: actions/checkout@v4
 
@@ -260,7 +352,7 @@ jobs:
         run: bundle install --jobs 4 --retry 3
 
       - name: Download packaged gem
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
          name: pkg-linux-gnu
          path: pkg
@@ -270,7 +362,7 @@ jobs:
 
       - name: Verify
         run: |
-          cd $(ruby -e "puts RbConfig::TOPDIR + '\/lib\/ruby\/gems\/' + RbConfig::CONFIG['ruby_version'] + '\/gems\/expressir*'")
+          cd $(ruby -e "puts Gem::Specification.find_by_name('expressir').gem_dir")
           ruby bin/rspec
           cat .rspec_status || echo ".rspec_status was not found"
 
@@ -288,7 +380,7 @@ jobs:
         uses: ruby/setup-ruby@master
         with:
           ruby-version: ${{ matrix.ruby }}
-          bundler-cache: true
+          bundler-cache: false
           bundler: ${{ env.BUNDLER_VER }}
 
       - name: Checkout
@@ -298,7 +390,7 @@ jobs:
         run: bundle install --jobs 4 --retry 3
 
       - name: Download packaged gem
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
          name: pkg-windows
          path: pkg
@@ -308,6 +400,49 @@ jobs:
 
       - name: Verify
         run: |
-          cd $(ruby -e "puts RbConfig::TOPDIR + '\/lib\/ruby\/gems\/' + RbConfig::CONFIG['ruby_version'] + '\/gems\/expressir*'")
+          cd $(ruby -e "puts Gem::Specification.find_by_name('expressir').gem_dir")
+          ruby bin/rspec
+          cat .rspec_status || echo ".rspec_status was not found"
+
+  verify-linux-musl:
+    name: verify Linux (musl) binary gem on ruby-${{ matrix.ruby }}
+    needs: pack
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '3.2', '3.1', '3.0', '2.7' ]
+    container: ruby:${{ matrix.ruby }}-alpine
+    steps:
+      - name: Install packages
+        run: |
+          apk --no-cache --upgrade add build-base git bash tar
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Upgrade RubyGems
+        if: matrix.ruby == '3.0' || matrix.ruby == '2.7'
+        run: gem update --system ${{ env.RUBYGEM_VER }} --no-document
+
+      - name: Bundle
+        run: |
+          git config --global --add safe.directory $PWD
+          bundle install --jobs 4 --retry 3
+
+      - name: Download packaged gem
+        uses: actions/download-artifact@v4
+        with:
+         name: pkg-linux-musl
+         path: pkg
+
+      - name: Install binary gem
+        run: gem install -l pkg/expressir-*-$(ruby -e "puts RUBY_PLATFORM").gem
+
+      - name: Verify
+        run: |
+          cd $(ruby -e "puts Gem::Specification.find_by_name('expressir').gem_dir")
           ruby bin/rspec
           cat .rspec_status || echo ".rspec_status was not found"

data/.github/workflows/release.yml

@@ -15,6 +15,9 @@ concurrency:
   group: '${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.ref_name }}'
   cancel-in-progress: true
 
+env:
+  BUNDLER_VER: 2.4.22
+
 jobs:
   bump:
     runs-on: ubuntu-latest
@@ -46,7 +49,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: recursive
-          #ref: main # https://github.com/actions/checkout/issues/439#issuecomment-830862188
+          ref: main # https://github.com/actions/checkout/issues/439#issuecomment-830862188
 
       - uses: ruby/setup-ruby@master
         with:
@@ -62,13 +65,13 @@ jobs:
           gem build expressir.gemspec | grep -o 'Version: .*' | awk '{print $2}' > version
 
       - if: matrix.platform == 'linux-gnu'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: pkg-ruby
           path: expressir-*.gem
 
       - if: matrix.platform == 'linux-gnu'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: version
           path: version
@@ -84,7 +87,7 @@ jobs:
       # build gem WITH pre-built native extension
       - run: bundle exec rake gem:${{ matrix.platform }}
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           name: pkg-${{ matrix.platform }}
           path: pkg/*.gem
@@ -93,22 +96,22 @@ jobs:
     runs-on: ubuntu-latest
     needs: pack
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: pkg-ruby
           path: pkg
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: pkg-linux-gnu
           path: pkg
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: pkg-windows
           path: pkg
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: pkg-darwin
           path: pkg
@@ -130,6 +133,7 @@ jobs:
           chmod 0600 ~/.gem/credentials
           gem signin
           for gem in pkg/*.gem; do gem push $gem -V; done
+          sleep 15
 
   verify:
     name: Verify published gem on ${{ matrix.os }}
@@ -148,7 +152,7 @@ jobs:
           bundler: ${{ env.BUNDLER_VER }}
 
       - name: Download version
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
          name: version
 
@@ -169,7 +173,7 @@ jobs:
         run: apk --no-cache add bash build-base git ruby-dev gcc g++ automake
 
       - name: Download version
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
          name: version
 

data/.github/workflows/stress.yml

@@ -0,0 +1,53 @@
+name: stress
+
+on:
+  workflow_dispatch:
+
+concurrency:
+  group: '${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.ref_name }}'
+  cancel-in-progress: true
+
+env:
+  BUNDLER_VER: 2.4.22
+  GC_STRESS: true
+
+jobs:
+  rake:
+    name: test on ruby-${{ matrix.ruby }} ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '3.2', '3.1', '3.0', '2.7' ]
+        os: [ ubuntu-latest, windows-latest, macos-latest ]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Setup packages
+        if: startsWith(matrix.os, 'macos')
+        run: brew install autoconf automake libtool
+
+      - name: Install Ruby
+        uses: ruby/setup-ruby@master
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler: ${{ env.BUNDLER_VER }}
+          # Rice gem has issues with bundler cache
+          # more info https://github.com/lutaml/expressir/runs/2097658383?check_suite_focus=true#step:7:2126
+          # but it is not the only issue
+          bundler-cache: false
+
+      - name: Bundle
+        run: bundle install --jobs 4 --retry 3
+
+      - name: Build native extension
+        run: bundle exec rake compile
+
+      - name: Run tests
+        run: |
+          bundle exec rake
+          cat .rspec_status

data/.gitignore

@@ -14,7 +14,7 @@
 /lib/expressir/express/*/express_parser.bundle
 /lib/expressir/express/*/express_parser.so
 /spec/syntax/*-pretty.exp
-/ext/express-parser/rice-embed
+/ext/express_parser/rice-embed
 
 # rspec failure tracking
 .rspec_status

data/.gitmodules

@@ -1,5 +1,5 @@
-[submodule "ext/express-parser/antlr4-upstream"]
-	path = ext/express-parser/antlr4-upstream
+[submodule "ext/express_parser/antlr4-upstream"]
+	path = ext/express_parser/antlr4-upstream
 	url = https://github.com/antlr/antlr4
 [submodule "ext/express-grammar"]
 	path = ext/express-grammar

data/exe/generate-parser

@@ -18,7 +18,7 @@ generator = Antlr4Native::Generator.new(
 generator.generate
 
 # fix issues with generated parser
-parser_source_file = File.join("ext", "express-parser", "express_parser.cpp")
+parser_source_file = File.join("ext", "express_parser", "express_parser.cpp")
 parser_source_lines = File.read(parser_source_file).split("\n")
 
 # - add ParserProxy tokens method, simple compensation for missing exposed BufferedTokenStream

data/expressir.gemspec

@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0")
 
   spec.files         = `git ls-files`.split("\n")\
-    + Dir.glob("ext/express-parser/antlr4-upstream/runtime/Cpp/runtime/**/*")
+    + Dir.glob("ext/express_parser/antlr4-upstream/runtime/Cpp/runtime/**/*")
 
   spec.test_files    = `git ls-files -- {spec}/*`.split("\n")
 
@@ -28,18 +28,18 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.executables   = %w[expressir]
 
-  spec.extensions = File.join(*%w(ext express-parser extconf.rb))
+  spec.extensions = File.join(*%w(ext express_parser extconf.rb))
 
-  spec.add_runtime_dependency "rice", "~> 4.1"
+  spec.add_runtime_dependency "rice", "~> 4.2"
   spec.add_runtime_dependency "thor", "~> 1.0"
-  spec.add_development_dependency "antlr4-native", "~> 2.1.0"
+  spec.add_development_dependency "antlr4-native", "~> 2.2"
   spec.add_development_dependency "asciidoctor", "~> 2.0.13"
   spec.add_development_dependency "bundler", "~> 2.3"
   spec.add_development_dependency "byebug", "~> 11.1"
   spec.add_development_dependency "pry", "~> 0.12.2"
   spec.add_development_dependency "rake", "~> 13.0"
   spec.add_development_dependency "rake-compiler", "~> 1.2"
-  spec.add_development_dependency "rake-compiler-dock", "~> 1.3"
+  spec.add_development_dependency "rake-compiler-dock", "~> 1.5"
   spec.add_development_dependency "rspec", "~> 3.11"
   spec.add_development_dependency "rubocop", "1.58"
   spec.add_development_dependency "rubocop-performance", "~> 1.19"

data/lib/expressir/express/parser.rb

@@ -39,7 +39,6 @@ module Expressir
       # so in this class we keep those C++ structure marked for GC so they are not freed
         @parser = ::ExpressParser::ParserExt.new(file.to_s)
         @parse_tree = @parser.syntax()
-
         @visitor = Visitor.new(@parser.tokens, include_source: include_source)
         @repository = @visitor.visit(@parse_tree)
 

data/lib/expressir/express/visitor.rb

@@ -40,7 +40,7 @@ module Expressir
 
       private_constant :REMARK_CHANNEL
 
-      # @param [Array<::ExpressParser::Token>] tokens
+      # @param [::ExpressParser::TokenVector] Rice-wrapped std::vector<TokenProxy>
       # @param [Boolean] include_source attach original source code to model elements
       def initialize(tokens, include_source: nil)
         @tokens = tokens
@@ -101,7 +101,11 @@ module Expressir
           [ctx.start.token_index, ctx.stop.token_index]
         end
 
-        @tokens[start_index..stop_index]
+        selected_tokens = []
+        (start_index..stop_index).each do |i|
+          selected_tokens << @tokens[i]
+        end
+        selected_tokens
       end
 
       def attach_source(ctx, node)
@@ -150,15 +154,14 @@ module Expressir
       end
 
       def attach_remarks(ctx, node)
-        @remark_tokens = get_tokens(ctx)
-        @remark_tokens = @remark_tokens.select{ |x| x.channel == REMARK_CHANNEL
-        }
+        remark_tokens = get_tokens(ctx)
+        remark_tokens = remark_tokens.select{ |x| x.channel == REMARK_CHANNEL }
 
         # skip already attached remarks
-        @remark_tokens = @remark_tokens.select{|x| !@attached_remark_tokens.include?(x)}
+        remark_tokens = remark_tokens.select{|x| !@attached_remark_tokens.include?(x.token_index)}
 
         # parse remarks, find remark targets
-        tagged_remark_tokens = @remark_tokens.map do |remark_token|
+        tagged_remark_tokens = remark_tokens.map do |remark_token|
           _, remark_tag, remark_text = if remark_token.text.start_with?('--')
             remark_token.text.match(/^--"([^"]*)"(.*)$/).to_a
           else
@@ -181,7 +184,7 @@ module Expressir
           remark_target.remarks << remark_text
 
           # mark remark as attached, so that it is not attached again at higher nesting level
-          @attached_remark_tokens << remark_token
+          @attached_remark_tokens << remark_token.token_index
         end
       end
 

data/lib/expressir/model/declarations/schema.rb

@@ -90,13 +90,13 @@ module Expressir
           return [] unless parent
 
           interfaces.flat_map do |interface|
-            schema = parent.children_by_id[interface.schema.id.downcase]
+            schema = parent.children_by_id[interface.schema.id.safe_downcase]
             if schema
               schema_safe_children = schema.safe_children
-              schema_safe_children_by_id = schema_safe_children.select{|x| x.id}.map{|x| [x.id.downcase, x]}.to_h
+              schema_safe_children_by_id = schema_safe_children.select{|x| x.id}.map{|x| [x.id.safe_downcase, x]}.to_h
               if interface.items.length > 0
                 interface.items.map do |interface_item|
-                  base_item = schema_safe_children_by_id[interface_item.ref.id.downcase]
+                  base_item = schema_safe_children_by_id[interface_item.ref.id.safe_downcase]
                   if base_item
                     id = interface_item.id || base_item.id
                     create_interfaced_item(id, base_item)

data/lib/expressir/model/model_element.rb

@@ -46,7 +46,7 @@ module Expressir
       def find(path)
         return self if path.empty?
 
-        path_parts = path.downcase.split(/\./).map do |current_path|
+        path_parts = path.safe_downcase.split(/\./).map do |current_path|
           _, _, current_path = current_path.rpartition(":") # ignore prefix
           current_path
         end
@@ -82,7 +82,7 @@ module Expressir
 
       # @return [Hash<String, Declaration>]
       def children_by_id
-        @children_by_id ||= children.select{|x| x.id}.map{|x| [x.id.downcase, x]}.to_h
+        @children_by_id ||= children.select{|x| x.id}.map{|x| [x.id.safe_downcase, x]}.to_h
       end
 
       # @return [nil]
@@ -90,7 +90,7 @@ module Expressir
         @children_by_id = nil
         nil
       end
-      
+
       # @param [String] root_path
       # @param [Express::Formatter] formatter
       # @param [Boolean] include_empty
@@ -131,7 +131,7 @@ module Expressir
 
         hash
       end
-      
+
       # @param [Hash] hash
       # @param [String] root_path
       # @return [ModelElement]

data/lib/expressir/version.rb

@@ -1,3 +1,3 @@
 module Expressir
-  VERSION = "1.3.0.pre.5".freeze
+  VERSION = "1.3.2".freeze
 end

data/lib/expressir.rb

@@ -7,6 +7,26 @@ Dir[File.join(__dir__, "expressir", "express", "*.rb")].sort.each do |fea|
   require fea
 end
 
+# ..........................................................
+# https://bugs.ruby-lang.org/issues/19319
+# The issue is that this bug is fixed for 3.1 and above,
+# but not for 3.0 or 2.7, so we need a "safe" function
+# ..........................................................
+
+if RUBY_VERSION < "3.1"
+  class String
+    def safe_downcase
+      each_char.map(&:downcase).join
+    end
+  end
+else
+  class String
+    def safe_downcase
+      downcase
+    end
+  end
+end
+
 module Expressir
   class Error < StandardError; end
 

data/rakelib/antlr4-native.rake

@@ -72,12 +72,12 @@ def create_pp_class_definition(parser_source_lines)
           return detail::To_Ruby<SyntaxContextProxy>().convert(proxy);
         }
 
-        Array getTokens() {
-          Array a;
+        Object getTokens() {
+          std::vector<TokenProxy> tk;
           for (auto token : tokens -> getTokens()) {
-            a.push(new TokenProxy(token));
+            tk.push_back(TokenProxy(token));
           }
-          return a;
+          return detail::To_Ruby<std::vector<TokenProxy>>().convert(tk);
         }
 
         Object visit(VisitorProxy* visitor) {
@@ -88,8 +88,8 @@ def create_pp_class_definition(parser_source_lines)
 
           try {
             return std::any_cast<Object>(result);
-          } catch(std::bad_cast) {
-           return Qnil;
+          } catch(std::bad_any_cast) {
+            return Qnil;
           }
         }
 
@@ -105,7 +105,7 @@ def create_pp_class_definition(parser_source_lines)
 end
 
 def create_class_api(parser_source_lines)
-  i = parser_source_lines.index { |x| x == "    .define_method(\"visit\", &ParserProxy::visit, Return().keepAlive());" }
+  i = parser_source_lines.index { |x| x == "    .define_method(\"visit\", &ParserProxy::visit);" }
   parser_source_lines[i] += <<~CPP.split("\n").map { |x| x == "" ? x : "  #{x}" }.join("\n")
 
 
@@ -116,17 +116,29 @@ def create_class_api(parser_source_lines)
 
     rb_cParserExt = define_class_under<ParserProxyExt>(rb_mExpressParser, "ParserExt")
       .define_constructor(Constructor<ParserProxyExt, string>())
-      .define_method("syntax", &ParserProxyExt::syntax, Return().keepAlive())
-      .define_method("tokens", &ParserProxyExt::getTokens)
-      .define_method("visit", &ParserProxyExt::visit, Return().keepAlive());
+      .define_method("syntax", &ParserProxyExt::syntax)
+      .define_method("tokens", &ParserProxyExt::getTokens, Return().keepAlive())
+      .define_method("visit", &ParserProxyExt::visit);
+
+    define_vector<std::vector<TokenProxy>>("TokenVector");
+
+  CPP
+end
+
+def create_vector_definition(parser_source_lines)
+  i = parser_source_lines.index { |x| x == "    .define_method(\"visit\", &ParserProxy::visit);" }
+  parser_source_lines[i] += <<~CPP.split("\n").map { |x| x == "" ? x : "  #{x}" }.join("\n")
 
   CPP
 end
 
 def generate_extended_parser
   # Generate extended parser that provide Ruby access to token stream
-  parser_source_file = File.join("ext", "express-parser", "express_parser.cpp")
-  parser_source_lines = File.read(parser_source_file).split("\n")
+  parser_source_file = File.join("ext", "express_parser", "express_parser.cpp")
+  parser_source_lines = File.read(parser_source_file)
+    .gsub!("bad_cast", "bad_any_cast")
+    .gsub!("return detail::To_Ruby<Token*>().convert(token)", "return detail::To_Ruby<TokenProxy>().convert(TokenProxy(token))")
+    .split("\n")
   create_class_declarations(parser_source_lines)
   create_tp_class_definition(parser_source_lines)
   create_pp_class_definition(parser_source_lines)

data/rakelib/cross-ruby.rake

@@ -3,6 +3,7 @@ require "shellwords"
 
 WINDOWS_PLATFORM_REGEX = /mingw|mswin/.freeze
 LINUX_GNU_PLATFORM_REGEX = /linux-gnu/.freeze
+LINUX_MUSL_PLATFORM_REGEX = /linux-musl/.freeze
 DARWIN_PLATFORM_REGEX = /darwin/.freeze
 GLIBC_MIN_VERSION = "2.17".freeze
 
@@ -11,16 +12,20 @@ CrossRuby = Struct.new(:version, :host) do
     "tmp/#{platform}/stage/lib/expressir/express/#{minor_ver}/express_parser.#{dll_ext}"
   end
 
-  def windows?
-    !!(platform =~ WINDOWS_PLATFORM_REGEX)
+  def darwin?
+    !!(platform =~ DARWIN_PLATFORM_REGEX)
   end
 
   def linux_gnu?
     !!(platform =~ LINUX_GNU_PLATFORM_REGEX)
   end
 
-  def darwin?
-    !!(platform =~ DARWIN_PLATFORM_REGEX)
+  def linux_musl?
+    !!(platform =~ LINUX_MUSL_PLATFORM_REGEX)
+  end
+
+  def windows?
+    !!(platform =~ WINDOWS_PLATFORM_REGEX)
   end
 
   def ver
@@ -63,6 +68,10 @@ CrossRuby = Struct.new(:version, :host) do
         "x86_64-linux-gnu"
       when "aarch64-linux-gnu"
         "aarch64-linux-gnu"
+      when "x86_64-linux-musl"
+        "x86_64-linux-musl"
+      when "aarch64-linux-musl"
+        "aarch64-linux-musl"
       when /\Ax86_64-darwin/
         "x86_64-darwin"
       when /\Aarm64-darwin/
@@ -89,7 +98,7 @@ CrossRuby = Struct.new(:version, :host) do
        case platform
        when /x64-mingw(32|-ucrt)/
          "x86_64-w64-mingw32-"
-       when /(x86_64|aarch64)-linux-gnu/
+       when /(x86_64|aarch64)-linux-(gnu|musl)/
          # We do believe that we are on Linux and can use native tools
          ""
        when /x86_64.*darwin/
@@ -105,9 +114,9 @@ CrossRuby = Struct.new(:version, :host) do
     case platform
     when /64-mingw(32|-ucrt)/
       "pei-x86-64"
-    when "x86_64-linux-gnu"
+    when /x86_64-linux-(gnu|musl)/
       "elf64-x86-64"
-    when "aarch64-linux-gnu"
+    when /aarch64-linux-(gnu|musl)/
       "elf64-little"
     when "x86_64-darwin"
       "Mach-O 64-bit x86-64"
@@ -154,7 +163,7 @@ CrossRuby = Struct.new(:version, :host) do
     case platform
     when WINDOWS_PLATFORM_REGEX
       verify_entry_windows(dump, dll)
-    when LINUX_GNU_PLATFORM_REGEX
+    when LINUX_GNU_PLATFORM_REGEX, LINUX_MUSL_PLATFORM_REGEX
       verify_entry_linux(dll)
     when DARWIN_PLATFORM_REGEX
       verify_entry_darwin(dll)
@@ -199,7 +208,7 @@ CrossRuby = Struct.new(:version, :host) do
     end
   end
 
-  def allowed_dlls_linux
+  def allowed_dlls_linux_gnu
     suffix = (platform == "x86_64-linux-gnu" ? "x86-64" : "aarch64")
     [
       "ld-linux-#{suffix}.so",
@@ -211,6 +220,16 @@ CrossRuby = Struct.new(:version, :host) do
     ]
   end
 
+  def allowed_dlls_linux_musl
+    suffix = (platform == "x86_64-linux-gnu" ? "x86-64" : "aarch64")
+    [
+      "ld-linux-#{suffix}.so",
+      "libstdc++.so",
+      "libc.so",
+      "libgcc_s.so",
+    ]
+  end
+
   def allowed_dlls_darwin
     [
       "/usr/lib/libSystem.B.dylib",
@@ -218,12 +237,14 @@ CrossRuby = Struct.new(:version, :host) do
     ]
   end
 
-  def allowed_dlls
+  def allowed_dlls # rubocop:disable Metrics/MethodLength
     case platform
     when WINDOWS_PLATFORM_REGEX
       allowed_dlls_windows
     when LINUX_GNU_PLATFORM_REGEX
-      allowed_dlls_linux
+      allowed_dlls_linux_gnu
+    when LINUX_MUSL_PLATFORM_REGEX
+      allowed_dlls_linux_musl
     when DARWIN_PLATFORM_REGEX
       allowed_dlls_darwin
     else
@@ -248,7 +269,7 @@ CrossRuby = Struct.new(:version, :host) do
     case platform
     when DARWIN_PLATFORM_REGEX
       actual_dlls_darwin(dll)
-    when LINUX_GNU_PLATFORM_REGEX
+    when LINUX_GNU_PLATFORM_REGEX, LINUX_MUSL_PLATFORM_REGEX
       actual_dlls_linux(dump)
     when WINDOWS_PLATFORM_REGEX
       actual_dlls_windows(dump)
@@ -335,15 +356,10 @@ def pre_req(plat)
 end
 
 namespace "gem" do
-  CROSS_RUBIES.find_all { |cr| cr.windows? || cr.linux_gnu? || cr.darwin? }
-    .map { |cr| { platform: cr.platform, tag: cr.tag } }
-    .uniq { |hash| hash[:platform] }.each do |hash|
-    plat = hash[:platform]
-    tag = hash[:tag]
-
+  CROSS_RUBIES.find_all { |cr| cr.windows? || cr.linux_gnu? || cr.linux_musl? || cr.darwin? }.map(&:platform).uniq.each do |plat|
     desc "build native gem for #{plat} platform"
     task plat do
-      RakeCompilerDock.sh <<~RCD, platform: tag
+      RakeCompilerDock.sh <<~RCD, platform: plat
         #{pre_req(plat)} && gem install bundler --no-document &&
         bundle && bundle exec rake gem:#{plat}:builder MAKE="nice make -j`nproc`"
       RCD
@@ -360,9 +376,12 @@ namespace "gem" do
   desc "build native gems for windows"
   multitask "windows" => CROSS_RUBIES.find_all(&:windows?).map(&:platform).uniq
 
-  desc "build native gems for linux"
+  desc "build native gems for linux-gnu"
   multitask "linux-gnu" => CROSS_RUBIES.find_all(&:linux_gnu?).map(&:platform).uniq
 
+  desc "build native gems for linux-musl"
+  multitask "linux-musl" => CROSS_RUBIES.find_all(&:linux_musl?).map(&:platform).uniq
+
   desc "build native gems for darwin"
   multitask "darwin" => CROSS_RUBIES.find_all(&:darwin?).map(&:platform).uniq
 end
@@ -370,7 +389,7 @@ end
 require "rake/extensiontask"
 
 Rake::ExtensionTask.new("express_parser", GEMSPEC) do |ext|
-  ext.ext_dir = "ext/express-parser"
+  ext.ext_dir = "ext/express_parser"
   ext.lib_dir = File.join(*["lib", "expressir", "express", ENV.fetch("FAT_DIR", nil)].compact)
   ext.config_options << ENV.fetch("EXTOPTS", nil)
   ext.cross_compile  = true

data/spec/acceptance/version_spec.rb

@@ -3,6 +3,9 @@ require "spec_helper"
 RSpec.describe "Expressir" do
   describe "version" do
     it "has a version number" do |example|
+      GC.stress = ENV["GC_STRESS"] == "true"
+      puts "Running tests in GC stress mode. It may take a couple of hours ..." if GC.stress
+
       print "\n[#{example.description}] "
       expect(Expressir::VERSION).not_to be nil
 

data/spec/expressir/express/cache_spec.rb

@@ -40,7 +40,7 @@ RSpec.describe Expressir::Express::Cache do
       end.to raise_error(Errno::ENOENT)
     end
 
-    it "parses a file" do |example|
+    it "loads a cache file" do |example|
       print "\n[#{example.description}] "
       temp_file = Tempfile.new
 
@@ -63,7 +63,7 @@ RSpec.describe Expressir::Express::Cache do
       GC.verify_internal_consistency
     end
 
-    it "fails parsing a file from a different Expressir version" do |example|
+    it "fails parsing a cache from a different Expressir version" do |example|
       print "\n[#{example.description}] "
       temp_file = Tempfile.new
 

data/spec/expressir/express/formatter_spec.rb

@@ -11,9 +11,7 @@ RSpec.describe Expressir::Express::Formatter do
       print "\n[#{example.description}] "
       exp_file = Expressir.root_path.join("spec", "syntax", "single.exp")
       formatted_exp_file = Expressir.root_path.join("spec", "syntax", "single_formatted.exp")
-
       repo = Expressir::Express::Parser.from_file(exp_file)
-
       result = Expressir::Express::Formatter.format(repo)
       # File.write(formatted_exp_file, result)
       expected_result = File.read(formatted_exp_file)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: expressir
 version: !ruby/object:Gem::Version
-  version: 1.3.0.pre.5
+  version: 1.3.2
 platform: arm64-darwin
 authors:
 - Ribose Inc.
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-12-16 00:00:00.000000000 Z
+date: 2024-04-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: '2.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: '2.2'
 - !ruby/object:Gem::Dependency
   name: asciidoctor
   requirement: !ruby/object:Gem::Requirement
@@ -128,14 +128,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -216,8 +216,10 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".cross_rubies"
+- ".github/workflows/codeql.yml"
 - ".github/workflows/rake.yml"
 - ".github/workflows/release.yml"
+- ".github/workflows/stress.yml"
 - ".gitignore"
 - ".gitmodules"
 - ".hound.yml"
@@ -379,9 +381,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 3.3.dev
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubygems_version: 3.3.26
 signing_key: