gapic-generator 0.10.1 → 0.45.1

data/lib/gapic/schema/loader.rb

@@ -14,12 +14,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require "google/api/annotations.pb"
-require "google/api/client.pb"
-require "google/api/field_behavior.pb"
-require "google/api/resource.pb"
-require "google/longrunning/operations.pb"
-require "google/protobuf/descriptor.pb"
+require "google/api/annotations_pb"
+require "google/api/client_pb"
+require "google/api/field_behavior_pb"
+require "google/api/resource_pb"
+require "google/api/routing_pb"
+require "google/cloud/extended_operations_pb"
+require "google/longrunning/operations_pb"
+require "google/protobuf/descriptor_pb"
+require "gapic/schema/proto_tools"
 require "gapic/schema/wrappers"
 
 module Gapic
@@ -32,12 +35,21 @@ module Gapic
         @prior_enums = []
       end
 
+      FILE_EXTENSION_NAMES = {
+        "google.api.resource_definition" => [1053, ::Google::Api::ResourceDescriptor, :repeated]
+      }.freeze
+
+      MESSAGE_EXTENSION_NAMES = {
+        "google.api.resource" => [1053, ::Google::Api::ResourceDescriptor]
+      }.freeze
+
       # Loads a file.
       #
       # @param file_descriptor [Google::Protobuf::FileDescriptorProto] the
       #   descriptor of the proto file.
-      # @oaram file_to_generate [Boolean] Whether this file is to be
+      # @param file_to_generate [Boolean] Whether this file is to be
       #   generated.
+      # @return [Gapic::Schema::File]
       def load_file file_descriptor, file_to_generate
         # Setup.
         address = file_descriptor.package.split "."
@@ -45,27 +57,28 @@ module Gapic
         registry = {}
 
         # Load the docs.
-        location = file_descriptor.source_code_info.location
-        docs = location.each_with_object({}) { |l, ans| ans[l.path] = l }
+        location = file_descriptor.source_code_info.location || []
+        docs = location.each_with_object({}) { |l, ans| ans[l.path.to_a] = l }
 
         # Load top-level enums.
-        enums = file_descriptor.enum_type.each_with_index.map do |e, i|
+        enums = (file_descriptor.enum_type || []).each_with_index.map do |e, i|
           load_enum registry, e, address, docs, [5, i]
         end
 
         # Load top-level messages.
-        messages = file_descriptor.message_type.each_with_index.map do |m, i|
+        messages = (file_descriptor.message_type || []).each_with_index.map do |m, i|
           load_message registry, m, address, docs, [4, i]
         end
         messages.each(&method(:update_fields!))
 
         # Load services.
-        services = file_descriptor.service.each_with_index.map do |s, i|
+        services = (file_descriptor.service || []).each_with_index.map do |s, i|
           load_service registry, s, address, docs, [6, i]
         end
 
         # Load top-level resources
-        resource_descriptors = file_descriptor.options[:".google.api.resource_definition"] if file_descriptor.options
+        option_extensions = ProtoTools.parse_options_extensions file_descriptor.options, FILE_EXTENSION_NAMES
+        resource_descriptors = option_extensions["google.api.resource_definition"]
         resources = Array(resource_descriptors).map { |descriptor| Resource.new descriptor }
 
         # Construct and return the file.
@@ -73,6 +86,25 @@ module Gapic
                  services, resources, file_to_generate, registry
       end
 
+      ##
+      # Loads snippet configs from the given directory
+      #
+      # @param path [String] Directory to search for snippet config files
+      # @return [Array<
+      #   Google::Cloud::Tools::SnippetGen::ConfigLanguage::V1::SnippetConfig>]
+      #
+      def load_snippet_configs path
+        return [] unless path
+        Dir.chdir path do
+          Dir.glob("**/*.json").map do |file_path|
+            json = JSON.load_file file_path
+            proto = Google::Cloud::Tools::SnippetGen::ConfigLanguage::V1::SnippetConfig.new underscore_keys json
+            proto.json_representation = json
+            proto
+          end
+        end
+      end
+
       # Updates the fields of a message and it's nested messages.
       #
       # @param message [Message] the message whose fields and nested messages
@@ -102,7 +134,7 @@ module Gapic
         address = address.clone << descriptor.name
 
         # Load Enum Values
-        values = descriptor.value.each_with_index.map do |value, i|
+        values = (descriptor.value || []).each_with_index.map do |value, i|
           load_enum_value registry, value, address, docs, path + [2, i]
         end
 
@@ -149,20 +181,22 @@ module Gapic
         address = address.clone << descriptor.name
 
         # Load Children
-        nested_messages = descriptor.nested_type.each_with_index.map do |m, i|
+        nested_messages = (descriptor.nested_type || []).each_with_index.map do |m, i|
           load_message registry, m, address, docs, path + [3, i]
         end
-        nested_enums = descriptor.enum_type.each_with_index.map do |e, i|
+        nested_enums = (descriptor.enum_type || []).each_with_index.map do |e, i|
           load_enum registry, e, address, docs, path + [4, i]
         end
-        fields = descriptor.field.each_with_index.map do |f, i|
+        fields = (descriptor.field || []).each_with_index.map do |f, i|
           load_field registry, f, address, docs, path + [2, i]
         end
-        extensions = descriptor.extension.each_with_index.map do |e, i|
+        fields.each { |field| field.populate_oneof_siblings! fields }
+        extensions = (descriptor.extension || []).each_with_index.map do |e, i|
           load_field registry, e, address, docs, path + [6, i]
         end
 
-        resource_descriptor = descriptor.options[:".google.api.resource"] if descriptor.options
+        option_extensions = ProtoTools.parse_options_extensions descriptor.options, MESSAGE_EXTENSION_NAMES
+        resource_descriptor = option_extensions["google.api.resource"]
         resource = resource_descriptor ? Resource.new(resource_descriptor) : nil
 
         # Construct, cache, and return.
@@ -209,7 +243,7 @@ module Gapic
         address = address.clone << descriptor.name
 
         # Load children
-        methods = descriptor.method.each_with_index.map do |m, i|
+        methods = (descriptor["method"] || descriptor.method || []).each_with_index.map do |m, i|
           load_method registry, m, address, docs, path + [2, i]
         end
 
@@ -259,6 +293,20 @@ module Gapic
 
       private
 
+      ##
+      # Transforms all keys in the given hash to snake case.
+      #
+      def underscore_keys input
+        case input
+        when Hash
+          input.to_h { |key, value| [ActiveSupport::Inflector.underscore(key), underscore_keys(value)] }
+        when Array
+          input.map { |elem| underscore_keys elem }
+        else
+          input
+        end
+      end
+
       def add_to_registry registry, address, obj
         registry[address.join(".")] = obj
         obj
@@ -266,7 +314,7 @@ module Gapic
 
       def cached_obj collection, type_name
         # Remove leading dot.
-        type_name = type_name[1..-1] if type_name && type_name[0] == "."
+        type_name = type_name[1..] if type_name && type_name[0] == "."
 
         # Create an address from the type & check cache.
         collection.find { |m| m.address == type_name.split(".") }

data/lib/gapic/schema/parameter_schema.rb

@@ -43,10 +43,10 @@ module Gapic
       # @param map_params_list [Array<String>]
       # @return Gapic::Schema::ParameterSchema
       def self.create bool_params_list: [], string_params_list: [], array_params_list: [], map_params_list: []
-        bool_params = bool_params_list.map { |val| [val, val] }.to_h
-        string_params = string_params_list.map { |val| [val, val] }.to_h
-        array_params = array_params_list.map { |val| [val, val] }.to_h
-        map_params = map_params_list.map { |val| [val, val] }.to_h
+        bool_params = bool_params_list.to_h { |val| [val, val] }
+        string_params = string_params_list.to_h { |val| [val, val] }
+        array_params = array_params_list.to_h { |val| [val, val] }
+        map_params = map_params_list.to_h { |val| [val, val] }
 
         ParameterSchema.new bool_params, string_params, array_params, map_params
       end
@@ -58,16 +58,16 @@ module Gapic
       # @param map_aliases [Hash{String => String}]
       # @return Gapic::Schema::ParameterSchema
       def extend_with_aliases bool_aliases: {}, string_aliases: {}, array_aliases: {}, map_aliases: {}
-        bool_params = @bool_params
+        bool_params = @bool_params.dup
         bool_aliases.each { |param_alias, param| bool_params[param_alias] = param if bool_params.key? param }
 
-        string_params = @string_params
+        string_params = @string_params.dup
         string_aliases.each { |param_alias, param| string_params[param_alias] = param if string_params.key? param }
 
-        array_params = @array_params
+        array_params = @array_params.dup
         array_aliases.each { |param_alias, param| array_params[param_alias] = param if array_params.key? param }
 
-        map_params = @map_params
+        map_params = @map_params.dup
         map_aliases.each { |param_alias, param| map_params[param_alias] = param if map_params.key? param }
 
         ParameterSchema.new bool_params, string_params, array_params, map_params

data/lib/gapic/schema/proto_tools.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "stringio"
+
+module Gapic
+  module Schema
+    ##
+    # @private
+    #
+    # A set of logic dealing with parsing raw proto data. This is unfortunately
+    # necessary for getting options extension information from proto
+    # descriptors, because the google-protobuf library implementation is
+    # incomplete. (Specifically, it does not expose Ruby interfaces for Service
+    # and Method descriptors, the objects that normally surface extended
+    # options; see https://github.com/protocolbuffers/protobuf/issues/14891.)
+    #
+    # We hope this code will be temporary until the above protobuf features are
+    # available. In the meantime, we make it private.
+    #
+    module ProtoTools
+      class << self
+        ##
+        # Parse through an options object looking for specific fields.
+        #
+        # The fields should be provided as a hash mapping the option name to
+        # an array of the form `[id, type, flags...]` where the ID is the field
+        # number, the type is the protobuf message class or one of the symbols
+        # `:string`, `:bool`, `:enum`, and currently the only flag supported is
+        # `:repeated`.
+        #
+        # Returns a hash mapping option names found to their values. Repeated
+        # values are represented as arrays (not as protobuf repeated values).
+        #
+        def parse_options_extensions options, extension_config
+          return {} unless options
+          return options.dup.delete_if { |k, _v| !extension_config.key? k } if options.is_a? Hash
+          bytes = options.class.encode options
+          return {} if bytes.empty?
+          option_data_by_id = build_option_data_by_id extension_config
+          results = build_default_results extension_config
+          walk_bytes bytes do |id, _wire_type, data|
+            option_data = option_data_by_id[id]
+            next unless option_data
+            name = option_data[0]
+            type = option_data[1]
+            results[name] =
+              if option_data[2].include? :repeated
+                decode_repeated data, type, results[name]
+              else
+                decode_data data, type
+              end
+          end
+          results
+        end
+
+        ##
+        # Given a byte string of data, parse through it and yield each record
+        # as the field number, wire type number, and data. The data is either
+        # an integer (for VARINT) or byte array (for LEN). I64 and I32 records
+        # are skipped.
+        #
+        def walk_bytes bytes
+          io = StringIO.new bytes
+          until io.eof?
+            tag = read_varint io
+            id = tag >> 3
+            wire_type = tag & 7
+            case wire_type
+            when 0 # VARINT
+              data = read_varint io
+              yield id, wire_type, data
+            when 1 # I64
+              # Skip
+              io.read 8
+            when 2 # LEN
+              len = read_varint io
+              data = io.read len
+              yield id, wire_type, data
+            when 5 # I32
+              # Skip
+              io.read 4
+            else
+              raise "Wire type #{wire_type} not supported (id = #{id})"
+            end
+          end
+        end
+
+        ##
+        # Read and return a single VARINT from the byte stream
+        #
+        def read_varint io
+          acc = 0
+          shift = 0
+          loop do
+            byte = io.getbyte
+            acc += ((byte & 0x7f) << shift)
+            shift += 7
+            break if byte < 0x80
+          end
+          acc
+        end
+
+        ##
+        # Given a data and type, return a decoded form of the data.
+        # If the type is `:string`, the data must be a bytestring and is
+        # returned as is. If the type is `:enum`, the data must be an integer
+        # and is returned as is. If the type is `:bool`, the data must be an
+        # integer and is converted to a boolean. If the type is a protobuf
+        # message class, the data must be a bytestring and is decoded. Any
+        # other type is not supported and results in an exception.
+        #
+        def decode_data data, type
+          case type
+          when :string, :enum
+            data
+          when :bool
+            !data.zero?
+          when ::Class
+            type.decode data
+          else
+            raise "Type #{type} not supported"
+          end
+        end
+
+        ##
+        # Given a data and a repeated type, return a decoded form of the data
+        # as an array.  Supports repeated `:string`, `:enum`, and protobuf
+        # message class types. Any other type results in an exception.
+        #
+        def decode_repeated data, type, acc = nil
+          acc ||= []
+          case type
+          when :string
+            acc << data
+          when :enum
+            if data.is_a? ::String
+              io = StringIO.new data
+              acc << read_varint(io) until io.eof?
+            else
+              acc << data
+            end
+          when ::Class
+            acc << type.decode(data)
+          else
+            raise "Type #{type} not supported"
+          end
+          acc
+        end
+
+        private
+
+        # @private
+        # Internal method to build a hash of default results
+        def build_default_results extension_config
+          extension_config.to_h do |name, config|
+            default =
+              if config[2..].include? :repeated
+                []
+              else
+                case config[1]
+                when :enum
+                  0
+                when :string
+                  ""
+                end
+              end
+            [name, default]
+          end
+        end
+
+        # @private
+        # Internal method to build a hash keyed by id
+        def build_option_data_by_id extension_config
+          extension_config.to_h { |name, config| [config[0], [name, config[1], config[2..]]] }
+        end
+      end
+    end
+  end
+end

data/lib/gapic/schema/request_param_parser.rb

@@ -100,9 +100,8 @@ module Gapic
         # @return [String,Array<String>,Hash{String => String}]
         def parse_validate_param_value param_type, param_name_input, param_config_name, value_str, error_output
           if param_type == :bool && !["true", "false"].include?(unescape(value_str))
-            error_str = "WARNING: parameter #{param_name_input} (recognised as bool " \
-                                          "#{param_config_name}) will be discarded because of " \
-                                          "invalid value. Value should be either 'true' or 'false'."
+            error_str = "WARNING: parameter #{param_name_input} (recognised as bool #{param_config_name}) will be " \
+                        "discarded because of invalid value. Value should be either 'true' or 'false'."
             error_output&.puts error_str
           end
 
@@ -113,10 +112,9 @@ module Gapic
             noncompliant_values = param_value.reject { |pv| allowed_transports.include? pv }
             if noncompliant_values.any?
               noncompliant_values_list = noncompliant_values.join ", "
-              error_str = "WARNING: parameter #{param_name_input} (recognised as string array " \
-                                          "#{param_config_name}) will be discarded because "\
-                                          "it contains invalid values: #{noncompliant_values_list}. "\
-                                          "#{param_config_name} can only contain 'grpc' and/or 'rest' or be empty."
+              error_str = "WARNING: parameter #{param_name_input} (recognised as string array #{param_config_name}) " \
+                          "will be discarded because it contains invalid values: #{noncompliant_values_list}. " \
+                          "#{param_config_name} can only contain 'grpc' and/or 'rest' or be empty."
               error_output&.puts error_str
               param_value = nil
             end

data/lib/gapic/schema/service_config_parser.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "yaml"
+require "google/api/service_pb"
+
+module Gapic
+  module Schema
+    # Contains logic for parsing a subset of service.yaml used
+    # for the service generation
+    module ServiceConfigParser
+      class << self
+        CONFIG_VERSION_KEY = "config_version"
+        NAME_KEY = "name"
+        VERSION_KEY = "version"
+        ID_KEY = "id"
+        TITLE_KEY = "title"
+        APIS_KEY = "apis"
+        HTTP_KEY = "http"
+        HTTP_RULES_KEY = "rules"
+        HTTP_RULES_SELECTOR_KEY = "selector"
+        HTTP_RULES_VERBS_ALLOWED = ["get", "post", "put", "patch", "delete"].freeze
+        HTTP_RULES_BODY_KEY = "body"
+        HTTP_RULES_ADDITIONAL_BINDINGS_KEY = "additional_bindings"
+        PUBS_KEY = "publishing"
+        PUBS_SHORTNAME_KEY = "api_short_name"
+        PUBS_ORGANIZATION_KEY = "organization"
+        PUBS_DOCURL_KEY = "documentation_uri"
+        PUBS_DOCTAG_KEY = "doc_tag_prefix"
+        PUBS_METHOD_SETTINGS = "method_settings"
+        DOCS_KEY = "documentation"
+        DOCS_SUMMARY_KEY = "summary"
+        DOCS_OVERVIEW_KEY = "overview"
+
+        ##
+        # Returns the parsed Google::Api::Service object.
+        # Only supports a limited subset of fields.
+        #
+        # @return [::Google::Api::Service]
+        #
+        def parse_service_yaml service_yaml_text
+          return nil unless service_yaml_text && !service_yaml_text.empty?
+          service_yaml = YAML.safe_load service_yaml_text
+          service = Google::Api::Service.new
+
+          if service_yaml.key? CONFIG_VERSION_KEY
+            config_ver = Google::Protobuf::UInt32Value.new
+            config_ver.value = service_yaml[CONFIG_VERSION_KEY]
+            service.config_version = config_ver
+          end
+
+          service.name = service_yaml[NAME_KEY] if service_yaml.key? NAME_KEY
+          service.id = service_yaml[ID_KEY] if service_yaml.key? ID_KEY
+          service.title = service_yaml[TITLE_KEY] if service_yaml.key? TITLE_KEY
+
+          service.apis += parse_apis service_yaml[APIS_KEY] if service_yaml.key? APIS_KEY
+          service.http = parse_http service_yaml[HTTP_KEY] if service_yaml.key? HTTP_KEY
+          service
+        end
+
+        ##
+        # Populates Gapic::Model::ApiMetadata from a service yaml file.
+        #
+        # @param service_yaml_text [String] YAML content
+        # @param api_metadata [Gapic::Model::ApiMetadata] model to populate
+        #
+        def parse_api_metadata service_yaml_text, api_metadata = nil
+          api_metadata ||= Gapic::Model::ApiMetadata.new
+          if service_yaml_text && !service_yaml_text.empty?
+            service_yaml = YAML.safe_load service_yaml_text
+            pubs_yaml = service_yaml[PUBS_KEY] || {}
+            docs_yaml = service_yaml[DOCS_KEY] || {}
+            api_metadata.update! name: service_yaml[NAME_KEY],
+                                 short_name: pubs_yaml[PUBS_SHORTNAME_KEY],
+                                 title: service_yaml[TITLE_KEY],
+                                 summary: docs_yaml[DOCS_SUMMARY_KEY],
+                                 description: docs_yaml[DOCS_OVERVIEW_KEY],
+                                 organization: pubs_yaml[PUBS_ORGANIZATION_KEY],
+                                 documentation_url: pubs_yaml[PUBS_DOCURL_KEY],
+                                 doc_tag_prefix: pubs_yaml[PUBS_DOCTAG_KEY],
+                                 method_settings: pubs_yaml[PUBS_METHOD_SETTINGS] || {}
+          end
+          api_metadata
+        end
+
+        private
+
+        # Parses the Apis section of the service yaml
+        # @return [Enumerable<::Google::Protobuf::Api>]
+        def parse_apis apis_yaml
+          apis_yaml.map { |api_yaml| parse_api api_yaml }
+        end
+
+        # Parses a single Api from yaml
+        # only supports a limited amount of fields
+        # @return [::Google::Protobuf::Api]
+        def parse_api api_yaml
+          api = ::Google::Protobuf::Api.new
+          api.name = api_yaml[NAME_KEY] if api_yaml.key? NAME_KEY
+          api.version = api_yaml[VERSION_KEY] if api_yaml.key? VERSION_KEY
+          api
+        end
+
+        # Parses a http section of the service yaml
+        # only supports a limited amount of fields
+        # @return [::Google::Api::Http]
+        def parse_http http_yaml
+          http = Google::Api::Http.new
+
+          if http_yaml.key? HTTP_RULES_KEY
+            http.rules += http_yaml[HTTP_RULES_KEY].map { |rule_yaml| parse_http_rule rule_yaml }
+          end
+
+          http
+        end
+
+        # Parses a single Http rule from yaml
+        # only supports a limited amount of fields
+        # NB: this method is recursive when parsing additional_bindings field
+        # @return [::Google::Api::HttpRule]
+        def parse_http_rule rule_yaml
+          rule = Google::Api::HttpRule.new
+          rule.selector = rule_yaml[HTTP_RULES_SELECTOR_KEY] if rule_yaml.key? HTTP_RULES_SELECTOR_KEY
+          verb_path = HTTP_RULES_VERBS_ALLOWED.find { |verb| rule_yaml[verb] }
+          rule.send "#{verb_path}=", rule_yaml[verb_path] if verb_path
+          rule.body = rule_yaml[HTTP_RULES_BODY_KEY] if rule_yaml.key? HTTP_RULES_BODY_KEY
+
+          if rule_yaml.key? HTTP_RULES_ADDITIONAL_BINDINGS_KEY
+            rule.additional_bindings += rule_yaml[HTTP_RULES_ADDITIONAL_BINDINGS_KEY].map do |binding_yaml|
+              parse_http_rule binding_yaml
+            end
+          end
+
+          rule
+        end
+      end
+    end
+  end
+end