rbs 4.1.0.pre.2-java → 4.1.1.dev.1-java

data/Dockerfile.jruby

@@ -0,0 +1,53 @@
+# JRuby verification image for RBS.
+#
+# RBS can't load its MRI C extension on JRuby, so it parses through a
+# WebAssembly build of the parser (see lib/rbs/wasm and docs/wasm_serialization.md).
+# This single-stage image installs the WASI SDK, compiles rbs_parser.wasm and
+# downloads the Chicory/ASM jars into ~/.m2 (via jar-dependencies), then runs the
+# test suite on JRuby. It mirrors .github/workflows/jruby.yml but is self-contained.
+#
+#   docker build -f Dockerfile.jruby -t rbs-jruby .
+#   docker run --rm rbs-jruby                                  # run the test suite
+#   docker run --rm -e RBS_PLATFORM=java rbs-jruby \
+#     gem build rbs.gemspec                                    # build the -java gem
+#
+# Bundler is intentionally not used: the development Gemfile pulls in CRuby-only
+# C extensions (bigdecimal, stackprof, ...) that cannot build on JRuby. The few
+# gems the suite needs are installed directly, exactly as the CI does.
+
+FROM jruby:10.1.1.0-jdk21
+
+# Keep in sync with .github/workflows/wasm.yml and .github/workflows/jruby.yml.
+ARG WASI_SDK_VERSION=33
+ARG WASI_SDK_RELEASE=33.0
+
+# build-essential supplies cc/make: on JRuby the prism gem builds libprism.so
+# (loaded via FFI) instead of an MRI C extension, so it needs a C toolchain.
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends git curl ca-certificates build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# The WASI SDK provides clang, the wasi-libc sysroot and the wasm32 compiler-rt
+# builtins that `rake wasm:build` needs to compile src/**/*.c to WebAssembly.
+RUN set -eux; \
+    case "$(uname -m)" in \
+      x86_64 | amd64)  wasi_arch=x86_64 ;; \
+      aarch64 | arm64) wasi_arch=arm64  ;; \
+      *) echo "unsupported arch: $(uname -m)" >&2; exit 1 ;; \
+    esac; \
+    mkdir -p /opt/wasi-sdk; \
+    curl -fsSL "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_RELEASE}-${wasi_arch}-linux.tar.gz" \
+      | tar xz --strip-components=1 -C /opt/wasi-sdk
+ENV WASI_SDK_PATH=/opt/wasi-sdk
+
+# Runtime/test gems the suite needs on JRuby (same set as the jruby.yml CI).
+RUN gem install prism rake rake-compiler test-unit rdoc rspec minitest json-schema pry --no-document
+
+WORKDIR /rbs
+COPY . .
+
+# Compile rbs_parser.wasm (clang is a subprocess, so the build is engine
+# independent) and download the Chicory + ASM jars into ~/.m2 via jar-dependencies.
+RUN rake wasm:jruby_setup wasm:install_jars
+
+CMD ["rake", "test"]

data/Rakefile

@@ -55,8 +55,15 @@ task :confirm_lexer => :lexer do
 end
 
 task :confirm_templates => :templates do
-  puts "Testing if generated code under include and src is updated with respect to templates"
-  sh "git diff --exit-code -- include src"
+  puts "Testing if generated code is updated with respect to templates"
+
+  # Every template generates the file it is named after: `templates/<path>.erb` generates `<path>`.
+  generated = Dir.glob("templates/**/*.erb").sort.map { _1.delete_prefix("templates/").delete_suffix(".erb") }
+
+  missing = generated.reject { File.exist?(_1) }
+  raise "Templates without a generated file: #{missing.join(", ")}. Is the `templates` task missing an entry?" unless missing.empty?
+
+  sh "git diff --exit-code -- #{generated.join(" ")}"
 end
 
 # Task to format C code using clang-format
@@ -147,6 +154,9 @@ end
 rule %r{^include/(.*)\.c} => 'templates/%X.c.erb' do |t|
   puts "⚠️⚠️⚠️ #{t.name} is older than #{t.source}. You may need to run `rake templates` ⚠️⚠️⚠️"
 end
+rule %r{^ext/(.*)\.c} => 'templates/%X.c.erb' do |t|
+  puts "⚠️⚠️⚠️ #{t.name} is older than #{t.source}. You may need to run `rake templates` ⚠️⚠️⚠️"
+end
 
 task :annotate do
   sh "bin/generate_docs.sh"
@@ -495,46 +505,329 @@ NOTES
 end
 
 
-desc "Generate changelog template from GH pull requests"
-task :changelog do
-  major, minor, patch, _pre = RBS::VERSION.split(".", 4)
-  major = major.to_i
-  minor = minor.to_i
-  patch = patch.to_i
+# Pull requests with one of these labels are omitted from the changelog.
+CHANGELOG_SKIP_LABELS = ["skip-changelog"]
 
-  if patch == 0
-    milestone = "RBS #{major}.#{minor}"
-  else
-    milestone = "RBS #{major}.#{minor}.x"
-  end
+# Resolves the commit-ish the changelog starts from.
+#
+# `version` is a version number, a tag, or any commit-ish. When it is omitted, the latest tag
+# matching `tag_glob` is used, skipping the ones matching `exclude_globs`.
+#
+def resolve_changelog_base(version, tag_glob:, exclude_globs: [])
+  require "open3"
+
+  from =
+    if version
+      # `4.1.0` and `v4.1.0` both mean the tag `v4.1.0`, while `master` or a SHA is used as is.
+      version.match?(/\A\d/) ? "v#{version}" : version
+    else
+      command = ["git", "describe", "--tags", "--match", tag_glob, "--abbrev=0"]
+      exclude_globs.each { |glob| command.push("--exclude", glob) }
+
+      output, status = Open3.capture2(*command)
+      raise "🚨 Cannot detect the latest tag matching `#{tag_glob}`. Give the previous version explicitly." unless status.success?
+      output.chomp
+    end
+
+  _, status = Open3.capture2("git", "rev-parse", "--verify", "--quiet", "#{from}^{commit}")
+  raise "🚨 No such commit-ish: `#{from}`" unless status.success?
 
-  puts "🔍 Finding pull requests that is associated to milestone `#{milestone}`..."
+  from
+end
 
-  command = [
-    "gh",
-    "pr",
-    "list",
-    "--limit=10000",
-    "--json",
-    "url,title,number",
-    "--search" ,
-    "milestone:\"#{milestone}\" is:merged sort:updated-desc -label:Released"
-  ]
+# Runs a GraphQL query against the repository of the working directory.
+#
+# `body` is the selection set inside `repository`, so a query can use the `$owner` and `$name`
+# variables. Returns the contents of `data.repository`.
+#
+def changelog_graphql(body)
+  require "open3"
+  require "json"
 
+  @changelog_repository ||=
+    begin
+      output, status = Open3.capture2("gh", "repo", "view", "--json", "nameWithOwner", "--jq", ".nameWithOwner")
+      raise status.inspect unless status.success?
+      output.chomp.split("/", 2)
+    end
+  owner, name = @changelog_repository
+
+  query = <<~GRAPHQL
+    query($owner: String!, $name: String!) {
+      repository(owner: $owner, name: $name) {
+        #{body}
+      }
+    }
+  GRAPHQL
+
+  output, status = Open3.capture2(
+    "gh", "api", "graphql",
+    "-f", "query=#{query}",
+    "-f", "owner=#{owner}",
+    "-f", "name=#{name}",
+    binmode: true
+  )
+  raise status.inspect unless status.success?
+
+  # GitHub always answers in UTF-8, while the default external encoding follows the locale. Without
+  # this, a pull request body with an emoji fails to parse under `LANG=C`, as in GitHub Actions.
+  JSON.parse(output.force_encoding(Encoding::UTF_8), symbolize_names: true).dig(:data, :repository)
+end
+
+# Lists the commits between `from` and `HEAD`, newest first.
+#
+# Giving `paths` limits the commits to the ones touching the paths.
+#
+def changelog_commits(from, paths: [])
   require "open3"
+
+  command = ["git", "log", "--format=%H", "#{from}..HEAD"]
+  # `--simplify-merges` keeps the default history simplification from following only one parent of
+  # a merge commit, which can drop the other side. Note that `--full-history` alone is wrong here:
+  # it also lists merge commits that do not touch the paths, bringing back the excluded pull
+  # requests. The two flags produce the same commits as the default mode for this repository today.
+  command.push("--full-history", "--simplify-merges", "--", *paths) unless paths.empty?
+
   output, status = Open3.capture2(*command)
   raise status.inspect unless status.success?
 
-  require "json"
-  json = JSON.parse(output, symbolize_names: true)
+  output.lines.map(&:chomp).reject(&:empty?)
+end
 
-  unless json.empty?
-    puts
-    json.each do |line|
-      puts "* #{line[:title]} ([##{line[:number]}](#{line[:url]}))"
+# Finds the pull requests the commits came from, keeping the order of `commits`.
+#
+# Returns the pull requests for the changelog and the ones omitted by `skip_labels`.
+#
+def changelog_pull_requests(commits, skip_labels: CHANGELOG_SKIP_LABELS)
+  pull_requests = {}
+  skipped = {}
+
+  commits.each_slice(50) do |slice|
+    # Ask GitHub which pull requests the commits came from, so that any merge strategy -- merge
+    # commit, squash, or rebase -- is handled without parsing commit messages.
+    aliases = slice.map.with_index do |commit, index|
+      <<~GRAPHQL
+        c#{index}: object(oid: "#{commit}") {
+          ... on Commit {
+            associatedPullRequests(first: 10) {
+              nodes {
+                number title url merged
+                labels(first: 100) { nodes { name } }
+              }
+            }
+          }
+        }
+      GRAPHQL
+    end
+
+    changelog_graphql(aliases.join("\n")).each_value do |commit|
+      next unless commit
+
+      commit.dig(:associatedPullRequests, :nodes).each do |pr|
+        next unless pr[:merged]
+
+        pr = { number: pr[:number], title: pr[:title], url: pr[:url], labels: pr.dig(:labels, :nodes).map { |label| label[:name] } }
+        if (pr[:labels] & skip_labels).empty?
+          pull_requests[pr[:number]] ||= pr
+        else
+          skipped[pr[:number]] ||= pr
+        end
+      end
+    end
+  end
+
+  [pull_requests.values, skipped.values]
+end
+
+# Fetches the details that help classifying the pull requests: the changed files and the body.
+#
+def changelog_pull_request_details(pull_requests)
+  pull_requests.each_slice(50).flat_map do |slice|
+    aliases = slice.map do |pr|
+      <<~GRAPHQL
+        p#{pr[:number]}: pullRequest(number: #{pr[:number]}) {
+          body
+          author { login }
+          files(first: 100) {
+            nodes { path }
+            pageInfo { hasNextPage }
+          }
+        }
+      GRAPHQL
+    end
+
+    details = changelog_graphql(aliases.join("\n"))
+
+    slice.map do |pr|
+      detail = details[:"p#{pr[:number]}"] or next pr
+
+      pr.merge(
+        author: detail.dig(:author, :login),
+        # The body is a hint for writing the changelog, not a copy source. Keep it short.
+        body: detail[:body].to_s.strip.slice(0, 1000),
+        files: detail.dig(:files, :nodes).map { |file| file[:path] },
+        files_truncated: detail.dig(:files, :pageInfo, :hasNextPage)
+      )
     end
+  end
+end
+
+# Reports the pull requests omitted by their label, so that they do not disappear silently.
+#
+def warn_skipped_pull_requests(skipped, skip_labels)
+  return if skipped.empty?
+
+  numbers = skipped.map { |pr| "##{pr[:number]}" }
+  numbers = numbers.take(20).push("and #{numbers.size - 20} more") if numbers.size > 20
+
+  $stderr.puts
+  $stderr.puts "  (⏭️  Skipped #{skipped.size} pull request(s) labeled #{skip_labels.map { |label| "`#{label}`" }.join(" or ")}: #{numbers.join(", ")})"
+end
+
+# Prints the changelog template listing the pull requests merged between `from` and `HEAD`.
+#
+# The changelog goes to STDOUT and everything else goes to STDERR, so that the output can be
+# piped to another command: `rake gem:changelog | pbcopy`
+#
+def print_changelog(from, paths: [], skip_labels: CHANGELOG_SKIP_LABELS)
+  $stderr.puts "🔍 Finding pull requests merged between `#{from}` and `HEAD`..."
+
+  commits = changelog_commits(from, paths: paths)
+  if commits.empty?
+    $stderr.puts "  (🤔 There is no commit after `#{from}`.)"
+    return
+  end
+
+  pull_requests, skipped = changelog_pull_requests(commits, skip_labels: skip_labels)
+
+  if pull_requests.empty?
+    $stderr.puts "  (🤔 No pull request is associated to the commits after `#{from}`.)"
   else
-    puts "  (🤑 There is no *unreleased* pull request associated to the milestone.)"
+    $stderr.puts
+    pull_requests.each do |pr|
+      puts "* #{pr[:title]} ([##{pr[:number]}](#{pr[:url]}))"
+    end
+    $stdout.flush
+  end
+
+  warn_skipped_pull_requests(skipped, skip_labels)
+end
+
+# Prints the same pull requests as `print_changelog` as JSON, with the details that help
+# classifying them into the sections of CHANGELOG.md.
+#
+# This is the input for the release automation, so it always prints a valid JSON document.
+#
+def print_changelog_json(from, paths: [], skip_labels: CHANGELOG_SKIP_LABELS)
+  require "json"
+
+  $stderr.puts "🔍 Finding pull requests merged between `#{from}` and `HEAD`..."
+
+  commits = changelog_commits(from, paths: paths)
+  pull_requests, skipped = changelog_pull_requests(commits, skip_labels: skip_labels)
+  pull_requests = changelog_pull_request_details(pull_requests)
+
+  $stderr.puts "  (📋 #{pull_requests.size} pull request(s))"
+
+  puts JSON.pretty_generate(
+    {
+      from: from,
+      to: "HEAD",
+      pull_requests: pull_requests,
+      skipped: skipped
+    }
+  )
+  $stdout.flush
+
+  warn_skipped_pull_requests(skipped, skip_labels)
+end
+
+namespace :gem do
+  # The gem is developed in the whole repository except the Rust crate, which has its own release
+  # cycle. Note that this is an *exclusion*, not a list of the directories shipped in the gem:
+  # changes in `test/` or `.github/` are part of the gem's changelog too.
+  # A constant defined in a `namespace` block is a top-level constant, so it needs the prefix.
+  GEM_CHANGELOG_PATHS = [".", ":(exclude)rust"]
+
+  # The tags a release proper starts *after*, rather than at.
+  GEM_PRERELEASE_TAGS = ["v*.pre*", "v*.dev*"]
+
+  # Where the changelog of the release being prepared starts, derived from `RBS::VERSION`:
+  #
+  # * `X.Y.Z.pre.N` documents what changed since `X.Y.Z.pre.N-1`, so it starts from the latest tag.
+  # * `X.Y.Z` documents the whole cycle, the prereleases included, so it skips the prerelease tags
+  #   in between and starts from the previous release proper.
+  #
+  # This is the step that is easy to get wrong by hand: on a release proper the latest tag is a
+  # prerelease, so the obvious default would produce only the tail of the cycle. Passing a version
+  # explicitly overrides all of it.
+  #
+  def changelog_base(version)
+    excluded = Gem::Version.new(RBS::VERSION).prerelease? ? [] : GEM_PRERELEASE_TAGS
+    resolve_changelog_base(version, tag_glob: "v*", exclude_globs: excluded)
+  end
+
+  desc "Generate changelog template from GH pull requests merged since the previous release"
+  task :changelog, [:version] do |_task, args|
+    print_changelog(changelog_base(args[:version]), paths: GEM_CHANGELOG_PATHS)
+  end
+
+  namespace :changelog do
+    desc "Print the pull requests of `gem:changelog` as JSON, with the changed files and body of each"
+    task :json, [:version] do |_task, args|
+      print_changelog_json(changelog_base(args[:version]), paths: GEM_CHANGELOG_PATHS)
+    end
+  end
+
+  desc "Publish the GitHub release for RBS::VERSION, unless it is a `.dev.` version"
+  task :gh_release do
+    require "open3"
+
+    version = Gem::Version.new(RBS::VERSION)
+    major, minor, *_ = RBS::VERSION.split(".")
+    tag = "v#{RBS::VERSION}"
+
+    # There are three kinds of release: `X.Y.Z`, `X.Y.Z.pre.N`, and `X.Y.Z.dev.N`.
+    # The `.dev.N` ones are cut from the development line for people who need a
+    # specific change early; they are not written up in the changelog, so there are
+    # no notes to publish and nothing worth announcing.
+    if version.segments.include?("dev")
+      puts "⏭️  #{RBS::VERSION} is a dev release, so there is no GitHub release to publish."
+      next
+    end
+
+    # The release is created against an existing tag, so that the artifacts and the
+    # notes describe a commit that is already immutable.
+    _, status = Open3.capture2("git", "rev-parse", "--verify", "--quiet", "#{tag}^{commit}")
+    raise "🚨 No such tag: `#{tag}`. Tag the release before creating the GitHub release." unless status.success?
+
+    # The topmost section of the changelog is this release, minus its own heading.
+    # The encoding is explicit because the default external encoding follows the
+    # locale, and the changelog is not ASCII.
+    content = File.read(File.join(__dir__, "CHANGELOG.md"), encoding: Encoding::UTF_8)
+    section = content.scan(/^## \d.*?(?=^## \d)/m)[0] or raise "🚨 Cannot find a release section in CHANGELOG.md"
+    heading, _, body = section.partition("\n")
+    heading.include?(RBS::VERSION) or raise "🚨 CHANGELOG.md starts with `#{heading.strip}`, which is not #{RBS::VERSION}"
+
+    notes = <<~NOTES
+      [Release note](https://github.com/ruby/rbs/wiki/Release-Note-#{major}.#{minor})
+
+      #{body.strip}
+    NOTES
+
+    # Published rather than drafted: the notes are the changelog section that was
+    # already reviewed in the release pull request, so there is nothing left to edit.
+    command = [
+      "gh", "release", "create", tag,
+      "--title=#{RBS::VERSION}",
+      "--notes=#{notes}"
+    ]
+    command << "--prerelease" if version.prerelease?
+
+    output, status = Open3.capture2(*command)
+    raise "🚨 `gh release create` failed: #{status.inspect}" unless status.success?
+
+    puts "📝 Released #{tag}: #{output.chomp}"
   end
 end
 
@@ -627,40 +920,27 @@ namespace :wasm do
     end
   end
 
-  # Where the runtime looks for the module and jars by default (see
-  # RBS::WASM::Runtime). These are build artifacts, bundled into the JRuby gem.
+  # Where the runtime looks for the module by default (see RBS::WASM::Runtime).
   JRUBY_WASM_DIR = File.expand_path("lib/rbs/wasm", __dir__)
-  CHICORY_VERSION = ENV.fetch("CHICORY_VERSION", "1.7.5")
-  # `compiler` is Chicory's AOT compiler (wasm -> JVM bytecode); the asm* jars
-  # are the ow2 ASM libraries it depends on. Keep ASM_VERSION in sync with what
-  # the pinned Chicory release declares.
-  CHICORY_JARS = %w[wasm runtime log wasi compiler].freeze
-  ASM_VERSION = ENV.fetch("ASM_VERSION", "9.9.1")
-  ASM_JARS = %w[asm asm-tree asm-util asm-commons asm-analysis].freeze
-
-  desc "Download the Chicory and ASM jars the JRuby runtime needs into lib/rbs/wasm/jars"
-  task :vendor_jars do
-    require "open-uri"
-    require "fileutils"
-
-    jars_dir = File.join(JRUBY_WASM_DIR, "jars")
-    FileUtils.mkdir_p(jars_dir)
-
-    downloads = CHICORY_JARS.map { |name| ["#{name}.jar", "https://repo1.maven.org/maven2/com/dylibso/chicory/#{name}/#{CHICORY_VERSION}/#{name}-#{CHICORY_VERSION}.jar"] }
-    downloads += ASM_JARS.map { |name| ["#{name}.jar", "https://repo1.maven.org/maven2/org/ow2/asm/#{name}/#{ASM_VERSION}/#{name}-#{ASM_VERSION}.jar"] }
-
-    downloads.each do |filename, url|
-      puts "Downloading #{url}"
-      URI.open(url) { |io| File.binwrite(File.join(jars_dir, filename), io.read) } # steep:ignore
-    end
 
-    puts "Vendored Chicory #{CHICORY_VERSION} + ASM #{ASM_VERSION} into #{jars_dir}"
+  desc "Download the Chicory/ASM jars into the local Maven repository (~/.m2). Run on JRuby."
+  task :install_jars do
+    # Resolves the `jar` requirements from rbs.gemspec via Maven and downloads
+    # them (and their transitive deps) into ~/.m2, the same way `gem install`
+    # does; the jars are not copied into the gem. The platform is forced to java
+    # because Jars::Installer skips non-java gems, and write_require_file is false
+    # because lib/rbs_jars.rb is hand-maintained (the generator mangles the
+    # `com.dylibso.chicory:runtime` artifact id).
+    require "jars/installer"
+    spec = Gem::Specification.load("rbs.gemspec")
+    spec.platform = "java"
+    Jars::Installer.new(spec).install_jars(write_require_file: false)
   end
 
-  desc "Assemble everything the JRuby gem needs: the .wasm and the Chicory jars"
-  task :jruby_setup => [:build, :vendor_jars] do
+  desc "Build rbs_parser.wasm and copy it next to RBS::WASM::Runtime"
+  task :jruby_setup => [:build] do
     cp WASM_OUTPUT, File.join(JRUBY_WASM_DIR, "rbs_parser.wasm")
-    puts "JRuby runtime is ready under #{JRUBY_WASM_DIR}"
+    puts "rbs_parser.wasm is ready under #{JRUBY_WASM_DIR}"
   end
 end
 

data/Steepfile

@@ -13,6 +13,8 @@ target :lib do
     "lib/rbs/wasm/location.rb",
     "lib/rbs/wasm/runtime.rb",
     "lib/rbs/wasm/parser.rb",
+    # jar-dependencies require_jar calls for the JRuby runtime; not type-checked.
+    "lib/rbs_jars.rb",
   )
 
   library "pathname", "json", "logger", "monitor", "tsort", "uri", 'dbm', 'pstore', 'singleton', 'shellwords', 'fileutils', 'find', 'digest', 'prettyprint', 'yaml', "psych", "securerandom"

data/config.yml

@@ -769,6 +769,7 @@ nodes:
         c_type: rbs_location_range
       - name: type_name
         c_type: rbs_type_name
+        optional: true  # NULL when the name is left out, to be inferred from the Ruby code
       - name: type_name_location
         c_type: rbs_location_range
         optional: true
@@ -781,6 +782,7 @@ nodes:
         c_type: rbs_location_range
       - name: type_name
         c_type: rbs_type_name
+        optional: true  # NULL when the name is left out, to be inferred from the Ruby code
       - name: type_name_location
         c_type: rbs_location_range
         optional: true