rbs 4.0.0.dev.4 → 4.1.0.pre.2

data/Rakefile

@@ -25,22 +25,23 @@ test_config = lambda do |t|
   t.test_files = FileList["test/**/*_test.rb"].reject do |path|
     path =~ %r{test/stdlib/}
   end
-end
-
-Rake::TestTask.new(test: :compile, &test_config)
-
-unless Gem.win_platform?
-  begin
-    require "ruby_memcheck"
-
-    namespace :test do
-      RubyMemcheck::TestTask.new(valgrind: :compile, &test_config)
+  if defined?(RubyMemcheck)
+    if t.is_a?(RubyMemcheck::TestTask)
+      t.verbose = true
+      t.options = '-v'
     end
-  rescue LoadError => exn
-    STDERR.puts "🚨🚨🚨🚨 Skipping RubyMemcheck: #{exn.inspect} 🚨🚨🚨🚨"
   end
 end
 
+if RUBY_ENGINE == "jruby"
+  # JRuby runs the parser in WebAssembly instead of the C extension, so there is
+  # nothing to compile. The wasm runtime must be assembled first with
+  # `rake wasm:jruby_setup` (which needs CRuby + the WASI SDK).
+  Rake::TestTask.new(:test, &test_config)
+else
+  Rake::TestTask.new(test: :compile, &test_config)
+end
+
 multitask :default => [:test, :stdlib_test, :typecheck_test, :rubocop, :validate, :test_doc]
 
 task :lexer do
@@ -166,6 +167,10 @@ task :templates do
   sh "#{ruby} templates/template.rb include/rbs/ast.h"
   sh "#{ruby} templates/template.rb src/ast.c"
 
+  sh "#{ruby} templates/template.rb include/rbs/serialize.h"
+  sh "#{ruby} templates/template.rb src/serialize.c"
+  sh "#{ruby} templates/template.rb lib/rbs/wasm/serialization_schema.rb"
+
   # Format the generated files
   Rake::Task["format:c"].invoke
 end
@@ -185,7 +190,7 @@ end
 task :validate => :compile do
   require 'yaml'
 
-  sh "#{ruby} #{rbs} validate --exit-error-on-syntax-error"
+  sh "#{ruby} #{rbs} validate"
 
   libs = FileList["stdlib/*"].map {|path| File.basename(path).to_s }
 
@@ -205,8 +210,18 @@ task :validate => :compile do
     libs << "rbs"
   end
 
+  libs.delete("bigdecimal-math") or raise
+  libs.delete("bigdecimal") or raise
+
   libs.each do |lib|
-    sh "#{ruby} #{rbs} -r #{lib} validate --exit-error-on-syntax-error"
+    args = ["-r", lib]
+
+    if lib == "rbs"
+      args << "-r" << "prism"
+      args << "-r" << "logger"
+    end
+
+    sh "#{ruby} #{rbs} #{args.join(' ')} validate"
   end
 end
 
@@ -218,7 +233,7 @@ end
 
 task :stdlib_test => :compile do
   test_files = FileList["test/stdlib/**/*_test.rb"].reject do |path|
-    path =~ %r{Ractor} || path =~ %r{Encoding} || path =~ %r{CGI_test}
+    path =~ %r{Ractor} || path =~ %r{Encoding} || path =~ %r{CGI-escape_test}
   end
 
   if ENV["RANDOMIZE_STDLIB_TEST_ORDER"] == "true"
@@ -227,27 +242,22 @@ task :stdlib_test => :compile do
 
   sh "#{ruby} -Ilib #{bin}/test_runner.rb #{test_files.join(' ')}"
   # TODO: Ractor tests need to be run in a separate process
-  sh "#{ruby} -Ilib #{bin}/test_runner.rb test/stdlib/CGI_test.rb"
+  sh "#{ruby} -Ilib #{bin}/test_runner.rb test/stdlib/CGI-escape_test.rb"
   sh "#{ruby} -Ilib #{bin}/test_runner.rb test/stdlib/Ractor_test.rb"
   sh "#{ruby} -Ilib #{bin}/test_runner.rb test/stdlib/Encoding_test.rb"
 end
 
 task :typecheck_test => :compile do
-  puts
-  puts
-  puts "⛔️⛔️⛔️⛔️⛔️⛔️ Skipping type check test because RBS is incompatible with Steep (#{__FILE__}:#{__LINE__})"
-  puts
-  puts
-  # FileList["test/typecheck/*"].each do |test|
-  #   Dir.chdir(test) do
-  #     expectations = File.join(test, "steep_expectations.yml")
-  #     if File.exist?(expectations)
-  #       sh "steep check --with_expectations"
-  #     else
-  #       sh "steep check"
-  #     end
-  #   end
-  # end
+  FileList["test/typecheck/*"].each do |test|
+    Dir.chdir(test) do
+      expectations = File.join(test, "steep_expectations.yml")
+      if File.exist?(expectations)
+        sh "steep check --with_expectations"
+      else
+        sh "steep check"
+      end
+    end
+  end
 end
 
 task :raap => :compile do
@@ -325,7 +335,7 @@ namespace :generate do
           class <%= target %>SingletonTest < Test::Unit::TestCase
             include TestHelper
 
-            # library "pathname", "securerandom"     # Declare library signatures to load
+            # library "logger", "securerandom"     # Declare library signatures to load
             testing "singleton(::<%= target %>)"
 
           <%- class_methods.each do |method_name, definition| -%>
@@ -344,7 +354,7 @@ namespace :generate do
           class <%= target %>Test < Test::Unit::TestCase
             include TestHelper
 
-            # library "pathname", "securerandom"     # Declare library signatures to load
+            # library "logger", "securerandom"     # Declare library signatures to load
             testing "::<%= target %>"
 
           <%- instance_methods.each do |method_name, definition| -%>
@@ -535,3 +545,325 @@ task :compile_c99 do
 ensure
   ENV.delete("TEST_NO_C23")
 end
+
+task :prepare_bench do
+  ENV.delete("DEBUG")
+  Rake::Task[:"clobber"].invoke
+  Rake::Task[:"templates"].invoke
+  Rake::Task[:"compile"].invoke
+end
+
+task :prepare_profiling do
+  ENV["DEBUG"] = "1"
+  Rake::Task[:"clobber"].invoke
+  Rake::Task[:"templates"].invoke
+  Rake::Task[:"compile"].invoke
+end
+
+namespace :wasm do
+  WASM_DIR = File.expand_path("wasm", __dir__)
+  WASM_OUTPUT = File.join(WASM_DIR, "rbs_parser.wasm")
+
+  # The parser under src/ is plain, self-contained C with no dependency on the
+  # Ruby C API, so it can be compiled to WebAssembly as-is. The only extra
+  # translation unit is the entry-point shim under wasm/.
+  def wasm_source_files
+    Dir.glob(File.join(__dir__, "src/**/*.c")).sort + [File.join(WASM_DIR, "rbs_wasm.c")]
+  end
+
+  # Locate the clang shipped with the WASI SDK.
+  #
+  # The system clang can target wasm32, but the WASI SDK additionally provides
+  # the wasi-libc sysroot and the wasm32 compiler-rt builtins that the link
+  # step needs, so we require it explicitly.
+  def wasi_clang
+    sdk = ENV["WASI_SDK_PATH"]
+    if sdk.nil? || sdk.empty?
+      raise <<~MSG
+        WASI_SDK_PATH is not set.
+
+        Install the WASI SDK from https://github.com/WebAssembly/wasi-sdk/releases
+        and point WASI_SDK_PATH at the extracted directory, for example:
+
+            export WASI_SDK_PATH=/opt/wasi-sdk
+            rake wasm:build
+      MSG
+    end
+
+    clang = File.join(sdk, "bin", "clang")
+    raise "clang not found at #{clang} (is WASI_SDK_PATH correct?)" unless File.executable?(clang)
+
+    clang
+  end
+
+  desc "Build the RBS parser as a WebAssembly module (requires WASI_SDK_PATH)"
+  task :build do
+    mkdir_p WASM_DIR
+    sh wasi_clang,
+       "--target=wasm32-wasip1",
+       # No `main`; the host calls `_initialize` and then the exported functions.
+       "-mexec-model=reactor",
+       "-std=gnu11",
+       "-O2",
+       "-Wno-unused-parameter",
+       "-I#{File.join(__dir__, "include")}",
+       "-o", WASM_OUTPUT,
+       *wasm_source_files
+    puts "Built #{WASM_OUTPUT}"
+  end
+
+  desc "Build and smoke-test the WebAssembly module (requires wasmtime)"
+  task :check => :build do
+    wasmtime = ENV["WASMTIME"] || "wasmtime"
+
+    # `rbs_wasm_selftest` parses a small fixed signature and returns 1 on
+    # success. `--invoke` prints the return value to stdout.
+    output = IO.popen([wasmtime, "run", "--invoke", "rbs_wasm_selftest", WASM_OUTPUT], err: File::NULL, &:read).to_s.strip
+
+    if output == "1"
+      puts "WebAssembly selftest passed."
+    else
+      raise "WebAssembly selftest failed: rbs_wasm_selftest returned #{output.inspect} (expected \"1\")"
+    end
+  end
+
+  # Where the runtime looks for the module and jars by default (see
+  # RBS::WASM::Runtime). These are build artifacts, bundled into the JRuby gem.
+  JRUBY_WASM_DIR = File.expand_path("lib/rbs/wasm", __dir__)
+  CHICORY_VERSION = ENV.fetch("CHICORY_VERSION", "1.7.5")
+  # `compiler` is Chicory's AOT compiler (wasm -> JVM bytecode); the asm* jars
+  # are the ow2 ASM libraries it depends on. Keep ASM_VERSION in sync with what
+  # the pinned Chicory release declares.
+  CHICORY_JARS = %w[wasm runtime log wasi compiler].freeze
+  ASM_VERSION = ENV.fetch("ASM_VERSION", "9.9.1")
+  ASM_JARS = %w[asm asm-tree asm-util asm-commons asm-analysis].freeze
+
+  desc "Download the Chicory and ASM jars the JRuby runtime needs into lib/rbs/wasm/jars"
+  task :vendor_jars do
+    require "open-uri"
+    require "fileutils"
+
+    jars_dir = File.join(JRUBY_WASM_DIR, "jars")
+    FileUtils.mkdir_p(jars_dir)
+
+    downloads = CHICORY_JARS.map { |name| ["#{name}.jar", "https://repo1.maven.org/maven2/com/dylibso/chicory/#{name}/#{CHICORY_VERSION}/#{name}-#{CHICORY_VERSION}.jar"] }
+    downloads += ASM_JARS.map { |name| ["#{name}.jar", "https://repo1.maven.org/maven2/org/ow2/asm/#{name}/#{ASM_VERSION}/#{name}-#{ASM_VERSION}.jar"] }
+
+    downloads.each do |filename, url|
+      puts "Downloading #{url}"
+      URI.open(url) { |io| File.binwrite(File.join(jars_dir, filename), io.read) } # steep:ignore
+    end
+
+    puts "Vendored Chicory #{CHICORY_VERSION} + ASM #{ASM_VERSION} into #{jars_dir}"
+  end
+
+  desc "Assemble everything the JRuby gem needs: the .wasm and the Chicory jars"
+  task :jruby_setup => [:build, :vendor_jars] do
+    cp WASM_OUTPUT, File.join(JRUBY_WASM_DIR, "rbs_parser.wasm")
+    puts "JRuby runtime is ready under #{JRUBY_WASM_DIR}"
+  end
+end
+
+namespace :rust do
+  namespace :rbs do
+    RUST_DIR = File.expand_path("rust", __dir__)
+    RBS_VERSION_FILE = File.join(RUST_DIR, "rbs_version")
+
+    VENDOR_TARGETS = {
+      "ruby-rbs-sys" => %w[include src],
+      "ruby-rbs" => %w[config.yml],
+    }
+
+    desc "Sync vendored RBS source from the pinned version"
+    task :sync do
+      unless File.exist?(RBS_VERSION_FILE)
+        raise "#{RBS_VERSION_FILE} not found. Run `rake rust:rbs:pin[VERSION]` first."
+      end
+
+      version = File.read(RBS_VERSION_FILE).strip
+      raise "#{RBS_VERSION_FILE} is empty" if version.empty?
+
+      puts "Syncing vendor/rbs/ from #{version}..."
+
+      VENDOR_TARGETS.each do |crate, entries|
+        vendor_dir = File.join(RUST_DIR, crate, "vendor", "rbs")
+
+        puts "  Copying files for #{crate}:"
+        chmod_R "u+w", vendor_dir, verbose: false if File.exist?(vendor_dir)
+        rm_rf vendor_dir, verbose: false
+        mkdir_p vendor_dir, verbose: false
+
+        entries.each do |entry|
+          target = File.join(vendor_dir, entry)
+
+          # Extract the entry from the pinned git tag using git archive
+          IO.popen(["git", "archive", "--format=tar", version, "--", entry], "rb") do |tar|
+            IO.popen(["tar", "xf", "-", "-C", vendor_dir], "wb") do |extract|
+              IO.copy_stream(tar, extract)
+            end
+          end
+
+          raise "Failed to extract #{entry} from #{version}" unless File.exist?(target)
+          puts "    #{entry}"
+        end
+
+        # Make files read-only to prevent accidental edits
+        chmod_R "a-w", vendor_dir, verbose: false
+      end
+
+      puts "📦 Synced vendor/rbs/ from #{version} (read-only)"
+    end
+
+    desc "Pin a specific RBS version for Rust crates (e.g., rake rust:rbs:pin[v4.0.3])"
+    task :pin, [:version] do |_t, args|
+      version = args[:version] or raise "Usage: rake rust:rbs:pin[VERSION]"
+
+      # Verify the tag exists
+      unless system("git", "rev-parse", "--verify", "#{version}^{commit}", out: File::NULL, err: File::NULL)
+        raise "Tag #{version} not found"
+      end
+
+      File.write(RBS_VERSION_FILE, "#{version}\n")
+      puts "📌 Pinned RBS version to #{version}"
+
+      Rake::Task["rust:rbs:sync"].invoke
+    end
+
+    desc "Create symlinks from vendor/rbs/ to the repository root (for development/CI)"
+    task :symlink do
+      VENDOR_TARGETS.each do |crate, entries|
+        vendor_dir = File.join(RUST_DIR, crate, "vendor", "rbs")
+
+        puts "Setting up symlinks for #{crate}..."
+        entries.each do |entry|
+          puts "  #{entry} -> repository root"
+        end
+
+        chmod_R "u+w", vendor_dir, verbose: false if File.exist?(vendor_dir)
+        rm_rf vendor_dir, verbose: false
+        mkdir_p vendor_dir, verbose: false
+
+        entries.each do |entry|
+          ln_s File.join("..", "..", "..", "..", entry), File.join(vendor_dir, entry), verbose: false
+        end
+      end
+
+      puts "🔗 Symlinked vendor/rbs/ to repository root"
+    end
+  end
+
+  namespace :publish do
+    def self.prepare_publish_branch(crate_name)
+      dry_run = ENV["RBS_RUST_PUBLISH_DRY_RUN"]
+
+      version_file = File.join(RUST_DIR, "rbs_version")
+
+      unless File.exist?(version_file)
+        raise "#{version_file} not found. Run `rake rust:rbs:pin[VERSION]` first."
+      end
+
+      rbs_version = File.read(version_file).strip
+      raise "#{version_file} is empty" if rbs_version.empty?
+
+      crate_version = File.read(File.join(RUST_DIR, crate_name, "Cargo.toml"))[/^version\s*=\s*"(.+)"/, 1]
+      release_branch = "rust/release-#{crate_name}-#{Time.now.strftime('%Y%m%d%H%M%S')}"
+
+      puts "=" * 60
+      puts "Rust crate publish: #{crate_name}#{dry_run ? " (DRY RUN)" : ""}"
+      puts "=" * 60
+      puts "  RBS source version:  #{rbs_version}"
+      puts "  #{crate_name}:        #{crate_version} (tag: #{crate_name}-v#{crate_version})"
+      puts "  Release branch:      #{release_branch}"
+      puts "=" * 60
+
+      # Check that vendor dirs contain real files, not symlinks
+      entries = VENDOR_TARGETS.fetch(crate_name)
+      entries.each do |entry|
+        path = File.join(RUST_DIR, crate_name, "vendor", "rbs", entry)
+        if File.symlink?(path)
+          raise "#{path} is a symlink. Run `rake rust:rbs:sync` first."
+        end
+        unless File.exist?(path)
+          raise "#{path} does not exist. Run `rake rust:rbs:sync` first."
+        end
+      end
+
+      # Ensure working tree is clean before publishing
+      unless `git status --porcelain`.strip.empty?
+        raise "💢 Working tree is dirty. Please commit or stash your changes before publishing."
+      end
+
+      # Create a release branch with vendor files committed
+      original_branch = `git rev-parse --abbrev-ref HEAD`.strip
+
+      sh "git", "checkout", "-b", release_branch, verbose: false
+      vendor_path = File.join("rust", crate_name, "vendor", "rbs")
+      sh "git", "add", "-f", vendor_path, verbose: false
+      sh "git", "commit", "-m", "Publish #{crate_name} (RBS #{rbs_version})", verbose: false
+
+      [dry_run, crate_version, original_branch]
+    end
+
+    desc "Publish ruby-rbs-sys crate to crates.io (set RBS_RUST_PUBLISH_DRY_RUN=1 for dry-run only)"
+    task :"ruby-rbs-sys" do
+      crate_name = "ruby-rbs-sys"
+      dry_run, crate_version, original_branch = prepare_publish_branch(crate_name)
+
+      begin
+        puts "🔰 Dry-run publishing..."
+
+        Dir.chdir(File.join(RUST_DIR, crate_name)) do
+          sh "cargo", "publish", "--dry-run"
+        end
+
+        puts "✅ Dry-run succeeded!"
+
+        unless dry_run
+          puts "💪 Publishing #{crate_name} for real..."
+
+          Dir.chdir(File.join(RUST_DIR, crate_name)) do
+            sh "cargo", "publish"
+          end
+
+          sh "git", "tag", "#{crate_name}-v#{crate_version}"
+          sh "git", "push", "origin", "#{crate_name}-v#{crate_version}"
+
+          puts "🎉 Published #{crate_name} successfully!"
+        end
+      ensure
+        sh "git", "checkout", original_branch, verbose: false
+      end
+    end
+
+    desc "Publish ruby-rbs crate to crates.io (set RBS_RUST_PUBLISH_DRY_RUN=1 for dry-run only)"
+    task :"ruby-rbs" do
+      crate_name = "ruby-rbs"
+      dry_run, crate_version, original_branch = prepare_publish_branch(crate_name)
+
+      begin
+        puts "🔰 Dry-run publishing..."
+
+        Dir.chdir(File.join(RUST_DIR, crate_name)) do
+          sh "cargo", "publish", "--dry-run", "--no-verify"
+        end
+
+        puts "✅ Dry-run succeeded!"
+
+        unless dry_run
+          puts "💪 Publishing #{crate_name} for real..."
+
+          Dir.chdir(File.join(RUST_DIR, crate_name)) do
+            sh "cargo", "publish"
+          end
+
+          sh "git", "tag", "#{crate_name}-v#{crate_version}"
+          sh "git", "push", "origin", "#{crate_name}-v#{crate_version}"
+
+          puts "🎉 Published #{crate_name} successfully!"
+        end
+      ensure
+        sh "git", "checkout", original_branch, verbose: false
+      end
+    end
+  end
+end

data/Steepfile

@@ -6,6 +6,13 @@ target :lib do
   ignore(
     "lib/rbs/test",
     # "lib/rbs/test.rb"
+
+    # JRuby-only implementations of RBS::Location and RBS::Parser. Like the C
+    # extension, these implement interfaces already described in sig/, and
+    # runtime.rb is Java interop, so they are not type-checked here.
+    "lib/rbs/wasm/location.rb",
+    "lib/rbs/wasm/runtime.rb",
+    "lib/rbs/wasm/parser.rb",
   )
 
   library "pathname", "json", "logger", "monitor", "tsort", "uri", 'dbm', 'pstore', 'singleton', 'shellwords', 'fileutils', 'find', 'digest', 'prettyprint', 'yaml', "psych", "securerandom"
@@ -15,6 +22,7 @@ target :lib do
   signature "stdlib/rdoc/0/"
   signature "stdlib/ripper/0"
   signature "stdlib/pp/0"
+  signature "steep/patch.rbs"
 
   # configure_code_diagnostics do |config|
   #   config[D::Ruby::MethodDefinitionMissing] = :hint