rbs_protobuf 1.0.0 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 563169bcedcc8c4392d8ad6c428c8fd4f31b5ca3373705439fa94a0e3f05b138
-  data.tar.gz: a55dd8d55cc9b866b101e51bb90ba88aa5e138251c3083b010671b03346d9db8
+  metadata.gz: bcaf6201c92331c28b9fa7d1791711def79ccb4803d81692a220b60d08e640a6
+  data.tar.gz: 93651dffc7e86d657f5ed5298b3ab43042a7e663fd07a502b25fe968ede6cef2
 SHA512:
-  metadata.gz: d0ea93fd55f7b9efb4a864156e428033da4116499c99ef8f222949ae301b16fe302cd00c022ce2ad91ac9216dca508a084a9c59fb5b296dbd5e41d4d8dd6b205
-  data.tar.gz: c143b93fce43ede0ff094a26f2e51d3d72e6b35853852cc11e1b571f74abbe68ec7f579af206bec8213c908bb34d1421986f57a6f14417d316d7b666491c04ce
+  metadata.gz: '09835d51dd74b278a376dc721bbde2527f6e501596dd5239b29a0da5aa96af69cfc08831e2dafc7b6ac67491ef3e5c35b2ec2dca536fbc66864291b3a2109e6a'
+  data.tar.gz: 42b4c2d53a767f7b8732f8d4ee7a74d724ba334280cf006326a1ebfcd5f1dc4031d44404d149ee63e8c66f46da5dee2f519eaa6539d7789fb8b9b5e5a9dfb863

data/.github/workflows/test.yml

@@ -12,12 +12,13 @@ jobs:
     strategy:
       matrix:
         container_tag:
-          - 2.7-bionic
-          - 3.0-focal
-          - 3.1-focal
+          - 3.0-dev-focal
+          - 3.1-dev-jammy
+          - 3.2-dev-jammy
         protoc_version:
-          - 3.18.2
-          - 3.19.4
+          - "3.18.2"
+          - "3.19.4"
+          - "24.0"
     container:
       image: rubylang/ruby:${{ matrix.container_tag }}
     steps:
@@ -31,8 +32,9 @@ jobs:
           mv bin/protoc /usr/local/bin/protoc
           protoc --version
       - uses: actions/checkout@v3
-        with:
-          submodules: true
+      - name: Set up git safe.directory
+        run: |
+          git config --global --add safe.directory /__w/rbs_protobuf/rbs_protobuf
       - name: Run test
         run: |
           ruby -v

data/.gitmodules

@@ -1,3 +0,0 @@
-[submodule "vendor/gem_rbs_collection"]
-	path = vendor/gem_rbs_collection
-	url = git@github.com:ruby/gem_rbs_collection.git

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # CHANGELOG
 
+## 1.2.0 (2023-09-06)
+
+* Add `RBS_PROTOBUF_CONCAT_LEVEL` option ([#38](https://github.com/square/rbs_protobuf/pull/38))
+* Generate rpc method types and interfaces ([#37](https://github.com/square/rbs_protobuf/pull/37))
+* Generate `record` type of each proto message ([#36](https://github.com/square/rbs_protobuf/pull/36))
+* Add filter ([#34](https://github.com/square/rbs_protobuf/pull/34))
+
+## 1.1.0 (2023-08-16)
+
+* Support optional fields in proto3 ([\#32](https://github.com/square/rbs_protobuf/pull/32))
+* Update Steep to 1.5 and RBS to 3.1 ([\#30](https://github.com/square/rbs_protobuf/pull/30))
+
 ## 1.0.0 (2022-04-25)
 
 This is version 1.0.0. 🎉

data/Gemfile

@@ -5,7 +5,7 @@ gemspec
 
 gem "rake", "~> 12.0"
 gem "minitest", "~> 5.0"
-gem "steep"
-gem "rbs"
+gem "steep", require: false
+gem "rbs", "~> 3.1"
 
 gem "protobuf", "~> 3.10"

data/Gemfile.lock

@@ -1,59 +1,72 @@
 PATH
   remote: .
   specs:
-    rbs_protobuf (1.0.0)
+    rbs_protobuf (1.2.0)
       activesupport (>= 4.0)
       rbs (>= 2.2.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.0.2.3)
+    activesupport (7.0.7.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
     ast (2.4.2)
-    concurrent-ruby (1.1.10)
+    concurrent-ruby (1.2.2)
+    csv (3.2.7)
     ffi (1.15.5)
-    i18n (1.10.0)
+    fileutils (1.7.1)
+    i18n (1.14.1)
       concurrent-ruby (~> 1.0)
-    language_server-protocol (3.16.0.3)
-    listen (3.7.1)
+    json (2.6.3)
+    language_server-protocol (3.17.0.3)
+    listen (3.8.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
+    logger (1.5.3)
     middleware (0.1.0)
-    minitest (5.15.0)
-    parallel (1.22.1)
-    parser (3.1.2.0)
+    minitest (5.19.0)
+    parser (3.2.2.3)
       ast (~> 2.4.1)
+      racc
     protobuf (3.10.5)
       activesupport (>= 3.2)
       middleware
       thor
       thread_safe
+    racc (1.7.1)
     rainbow (3.1.1)
     rake (12.3.3)
-    rb-fsevent (0.11.1)
+    rb-fsevent (0.11.2)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
-    rbs (2.3.2)
-    steep (0.52.1)
+    rbs (3.1.3)
+    securerandom (0.2.2)
+    steep (1.5.3)
       activesupport (>= 5.1)
+      concurrent-ruby (>= 1.1.10)
+      csv (>= 3.0.9)
+      fileutils (>= 1.1.0)
+      json (>= 2.1.0)
       language_server-protocol (>= 3.15, < 4.0)
       listen (~> 3.0)
-      parallel (>= 1.0.0)
-      parser (>= 3.0)
+      logger (>= 1.3.0)
+      parser (>= 3.1)
       rainbow (>= 2.2.2, < 4.0)
-      rbs (>= 2.3.2)
+      rbs (>= 3.1.0)
+      securerandom (>= 0.1)
+      strscan (>= 1.0.0)
       terminal-table (>= 2, < 4)
+    strscan (3.0.6)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
     thor (1.2.1)
     thread_safe (0.3.6)
-    tzinfo (2.0.4)
+    tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.1.0)
+    unicode-display_width (2.4.2)
 
 PLATFORMS
   ruby
@@ -62,7 +75,7 @@ DEPENDENCIES
   minitest (~> 5.0)
   protobuf (~> 3.10)
   rake (~> 12.0)
-  rbs
+  rbs (~> 3.1)
   rbs_protobuf!
   steep
 

data/README.md

@@ -82,21 +82,6 @@ Run `protoc` with `--rbs_out` option.
 
 You may need `bundle exec protoc ...` to let bundler set up PATH.
 
-## Type checking
-
-To type check the output, you need to configure your tools to import [gem_rbs_collection](https://github.com/ruby/gem_rbs_collection) with `rbs collection` command.
-
-```yaml
-# Add the dependency in rbs_collection.yaml
-gems:
-  - name: rbs_protobuf
-```
-
-We assume that you don't type check the generated `.pb.rb` code.
-If you want to type check them, you need the definition of `Google::Protobuf`, which can be generated from [`descriptor.proto`](https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/descriptor.proto).
-
-Note: `rbs_protobuf` generates RBS files assuming some types added in [this PR](https://github.com/ruby/gem_rbs_collection/pull/145). Make sure you are using one of the newer versions of `rbs_gem_collection` definitions.
-
 ### Options
 
 * `RBS_PROTOBUF_BACKEND` specifies the Ruby code generator gem. Supported value is `protobuf`. (We will add `google-protobuf` for `google-protobuf` gem.)
@@ -104,6 +89,20 @@ Note: `rbs_protobuf` generates RBS files assuming some types added in [this PR](
 * `RBS_PROTOBUF_NO_NESTED_NAMESPACE` is to make the RBS declarations flat.
 * `RBS_PROTOBUF_EXTENSION` specifies what to do for extensions.
 * `RBS_PROTOBUF_ACCEPT_NIL_ATTR_WRITER` is to allow passing `nil` to required fields.
+* `RBS_PROTOBUF_FILTERS` contains filter Ruby script paths separated by `File::PATH_SEPARATOR`
+* `RBS_PROTOBUF_CONCAT_LEVEL` contains the number of dir levels that groups generated RBS files to concat
+
+## Type checking
+
+To type check the output, make sure your type checker configuration loads type definitions of `protobuf` gem.
+
+```ruby
+# Declare in Gemfile and load it with rbs-collection
+gem 'protobuf'
+```
+
+We assume that you don't type check the generated `.pb.rb` code.
+If you want to type check them, you need the definition of `Google::Protobuf`, which can be generated from [`descriptor.proto`](https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/descriptor.proto).
 
 ## Supported features
 
@@ -118,7 +117,7 @@ Note: `rbs_protobuf` generates RBS files assuming some types added in [this PR](
 | Services                | Only generates classes     |
 | Oneof                   | No support in `protobuf` gem |
 
-### Extensions
+## Extensions
 
 Adding extensions may cause problems if the name of new attribute conflicts.
 
@@ -144,6 +143,27 @@ You can control the behavior with `RBS_PROTOBUF_EXTENSION` environment variable.
 * Any value else: Generates RBS for extensions.
 * undefined: Ignores extensions but print messages to ask you to specify a value.
 
+## Filters
+
+You can apply filters that modifies generated RBS files.
+
+A filter is a proc object with type of `^(String rbs_name, String rbs_content, untyped proto_file) -> [String, String]`:
+It receives the file name of RBS file, the content of RBS file, the source protobuf object, and returns a pair of RBS file name and content.
+
+```ruby
+# example_fitler.rb: It adds a warning comment at the top of the RBS content.
+->(rbs_name, rbs_content, _proto_file) {
+  [
+    rbs_name,
+    "# Don't modify this file. This is generated with rbs_protobuf.\n\n" + rbs_content
+  ]
+}
+```
+
+You can apply filters by setting `RBS_PROTOBUF_FILTERS` environment variable.
+
+    $ RBS_PROTOBUF_BACKEND=protobuf RBS_PROTOBUF_FILTERS=example_filter.rb protoc ...
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test example:typecheck` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/Rakefile

@@ -27,7 +27,8 @@ namespace :example do
       "-I#{proto_path}",
       "example/a.proto",
       "example/b.proto",
-      "example/c.proto"
+      "example/c.proto",
+      "example/d.proto"
     )
     sh(
       { "RBS_PROTOBUF_BACKEND" => "protobuf", "RBS_PROTOBUF_EXTENSION" => "true" },
@@ -35,7 +36,8 @@ namespace :example do
       "--rbs_out=example/protobuf-gem",
       "-Iexample",
       "example/a.proto",
-      "example/b.proto"
+      "example/b.proto",
+      "example/d.proto"
     )
     sh(
       { "RBS_PROTOBUF_BACKEND" => "protobuf", "RBS_PROTOBUF_EXTENSION" => "true", "RUBYOPT" => "-rbundler/setup -Iexample/protobuf-gem -rc.pb" },

data/Steepfile

@@ -2,7 +2,6 @@ D = Steep::Diagnostic
 
 target :lib do
   signature "sig"
-  signature "vendor/gem_rbs_collection/gems/protobuf"
   check "lib"
 
   configure_code_diagnostics do |hash|

data/bin/protoc-gen-dumper

@@ -7,5 +7,7 @@ require "rbs_protobuf"
 input = STDIN.read()
 File.write(ENV["PROTOC_DUMPER_OUT"] || "a.pb.out", input)
 
-response = Google::Protobuf::Compiler::CodeGeneratorResponse.new
+response = Google::Protobuf::Compiler::CodeGeneratorResponse.new(
+  :supported_features => ::Google::Protobuf::Compiler::CodeGeneratorResponse::Feature::FEATURE_PROTO3_OPTIONAL.to_i
+)
 print Google::Protobuf::Compiler::CodeGeneratorResponse.encode(response)

data/bin/setup

@@ -9,13 +9,4 @@ echo ">>> Checking if protoc is available... 👀"
 protoc --version
 
 echo ">>> Setting up gem_rbs_collection... 💎"
-REPODIR=$(cd $(dirname $0)/..; pwd)/tmp/rbs/gem_rbs_collection
-if [ -e $REPODIR ]; then
-  echo "      ${REPODIR} exists"
-else
-  echo "      Cloning git repo into ${REPODIR}..."
-  mkdir -p $REPODIR/..
-  git clone https://github.com/ruby/gem_rbs_collection.git ${REPODIR}
-fi
-
 bundle exec rbs collection install

data/example/d.proto

@@ -0,0 +1,6 @@
+syntax = "proto3";
+
+message Foo {
+  string bar = 2;
+  optional string baz = 1;
+}

data/exe/protoc-gen-rbs

@@ -13,6 +13,20 @@ unless backend
   exit 1
 end
 
+filters = []
+
+if string = ENV["RBS_PROTOBUF_FILTERS"]
+  string.split(File::PATH_SEPARATOR).each do |file|
+    filter = Module.new.instance_eval(File.read(file), file)
+
+    if filter.respond_to?(:call)
+      filters << filter
+    else
+      raise "A filter `#{file}` returns `#{filter.inspect}`, but a proc object of type `^(String rbs_name, String rbs_content, untyped proto_file) -> [String, String]` is expected"
+    end
+  end
+end
+
 input = Google::Protobuf::Compiler::CodeGeneratorRequest.decode(STDIN.read)
 
 translator = case backend
@@ -34,6 +48,7 @@ translator = case backend
 
                RBSProtobuf::Translator::ProtobufGem.new(
                  input,
+                 filters,
                  upcase_enum: upcase_enum,
                  nested_namespace: !no_nested_namespace,
                  extension: extension,
@@ -43,6 +58,9 @@ translator = case backend
                raise NotImplementedError
              end
 
+if level = ENV["RBS_PROTOBUF_CONCAT_LEVEL"]
+  translator.rbs_concat_level = level.to_i
+end
 translator.generate_rbs!
 
 response = translator.response

data/lib/rbs_protobuf/rbs_factory.rb

@@ -46,6 +46,7 @@ module RBSProtobuf
     end
 
     def union_type(type, *types)
+      types = types.compact
       if types.empty?
         type
       else

data/lib/rbs_protobuf/translator/base.rb

@@ -3,10 +3,19 @@ module RBSProtobuf
     class Base
       FieldDescriptorProto = Google::Protobuf::FieldDescriptorProto
 
-      attr_reader :input
+      attr_reader :input, :filters
 
-      def initialize(input)
+      attr_accessor :rbs_concat_level
+
+      def initialize(input, filters = [])
         @input = input
+        @filters = filters
+      end
+
+      def apply_filter(rbs_name, rbs_content, proto_files)
+        filters.inject([rbs_name, rbs_content]) do |(rbs_name, rbs_content), filter| #$ [String, String]
+          filter[rbs_name, rbs_content, proto_files] or return
+        end
       end
 
       def factory
@@ -14,18 +23,65 @@ module RBSProtobuf
       end
 
       def response
-        @response ||= Google::Protobuf::Compiler::CodeGeneratorResponse.new
+        @response ||= Google::Protobuf::Compiler::CodeGeneratorResponse.new(:supported_features => ::Google::Protobuf::Compiler::CodeGeneratorResponse::Feature::FEATURE_PROTO3_OPTIONAL.to_i)
       end
 
       def generate_rbs!
+        rbs_decls = {} #: Hash[Pathname, [Array[RBS::AST::Declarations::t], untyped]]
+
         input.proto_file.each do |file|
-          response.file << Google::Protobuf::Compiler::CodeGeneratorResponse::File.new(
-            name: rbs_name(file.name),
-            content: rbs_content(file)
-          )
+          path = Pathname(file.name).sub_ext(".rbs")
+          decls = rbs_content(file)
+          rbs_decls[path] = [decls, file]
+        end
+
+        rbs_contents = {} #: Hash[Pathname, [String, Array[untyped]]]
+
+        if (level = rbs_concat_level)&.nonzero?
+          groups = rbs_decls.each_key.group_by do |path|
+            path.sub_ext("").to_s.split(File::SEPARATOR).take(level).join(File::SEPARATOR)
+          end
+
+          groups.each do |prefix, paths|
+            path = Pathname(prefix).sub_ext(".rbs")
+
+            decls = [] #: Array[RBS::AST::Declarations::t]
+            files = [] #: Array[untyped]
+
+            paths.each do |path|
+              ds, file = rbs_decls.fetch(path)
+              decls.concat(ds)
+              files.push(file)
+            end
+
+            rbs_contents[path] = [
+              format_rbs(decls: decls),
+              files
+            ]
+          end
+        else
+          rbs_decls.each do |path, (decls, file)|
+            content = format_rbs(decls: decls)
+            rbs_contents[path] = [content, [file]]
+          end
+        end
+
+        rbs_contents.each do |path, (content, files)|
+          if (name, content = apply_filter(path.to_s, content, files))
+            response.file << Google::Protobuf::Compiler::CodeGeneratorResponse::File.new(
+              name: name,
+              content: content
+            )
+          end
         end
       end
 
+      def format_rbs(dirs: [], decls: [])
+        StringIO.new.tap do |io|
+          RBS::Writer.new(out: io).write(dirs + decls)
+        end.string
+      end
+
       def rbs_name(proto_name)
         dirname = File.dirname(proto_name)
         basename = File.basename(proto_name, File.extname(proto_name))
@@ -120,6 +176,15 @@ module RBSProtobuf
           )
         )
       end
+
+      def optional_type(type)
+        case type
+        when RBS::Types::Optional
+          type
+        else
+          RBS::Types::Optional.new(type: type, location: nil)
+        end
+      end
     end
   end
 end