protostar 0.1.0.beta1

data/lib/protostar/schema/options.rb

@@ -0,0 +1,367 @@
+# frozen_string_literal: true
+
+module Protostar
+  module Schema
+    # A message-typed value in options: an options message itself, or a
+    # message inside one (a `FeatureSet`, a custom option of a message type).
+    # Fields are kept by name, extensions by their {Field}, and fields nobody
+    # could interpret as their wire bytes.
+    #
+    # Values are Ruby values: Integers, Floats, Strings (binary for bytes),
+    # true and false, enum values as Symbols (an Integer for a number the enum
+    # does not name), repeated fields as frozen Arrays, messages as
+    # MessageValues. Map fields are repeated entry messages.
+    class MessageValue
+      # @return [Hash{Field => Object}] extension values by extension descriptor
+      attr_reader :extensions
+      # @return [String] binary, wire-format bytes of fields that could not be interpreted
+      attr_reader :unknown_fields
+
+      # @param fields [Hash{Symbol => Object}]
+      # @param extensions [Hash{Field => Object}]
+      # @param unknown_fields [String] binary
+      def initialize(fields = Freezer::EMPTY_HASH, extensions = Freezer::EMPTY_HASH, unknown_fields = EMPTY_BYTES)
+        @fields = frozen(fields)
+        @extensions = frozen(extensions)
+        @unknown_fields = if unknown_fields.empty?
+          EMPTY_BYTES
+        else
+          unknown_fields.frozen? ? unknown_fields : unknown_fields.b.freeze
+        end
+      end
+
+      # @param name [Symbol, String] a field of the message
+      # @return [Object, nil] its value, nil when not set
+      def [](name)
+        @fields[name.to_sym]
+      end
+
+      # @return [Boolean] whether the field is set
+      def key?(name)
+        @fields.key?(name.to_sym)
+      end
+      alias_method :set?, :key?
+
+      # @return [Hash{Symbol => Object}] the fields that are set, by name, in the order they were given
+      attr_reader :fields
+
+      # @param key [Field, String] an extension descriptor or its full name
+      # @return [Object, nil] the extension's value
+      def extension(key)
+        return @extensions[key] if key.is_a?(Field)
+        name = key.to_s.delete_prefix(".")
+        @extensions.each { |field, value| return value if field.full_name == name }
+        nil
+      end
+
+      # The same value with every extension, here and in nested values, looked
+      # up again by full name; one that is not found stays as it was. A copy
+      # of a file in another pool keys its options by that pool's extensions.
+      # @api private
+      # @param extension [#call] full name to extension {Field}
+      # @return [MessageValue]
+      def rekey(extension)
+        changed = false
+        again = lambda do |v|
+          case v
+          when MessageValue
+            found = v.rekey(extension)
+            changed ||= !found.equal?(v)
+            found
+          when Array then v.map(&again)
+          else v
+          end
+        end
+        fields = @fields.transform_values(&again)
+        extensions = @extensions.to_h do |field, v|
+          key = extension.call(field.full_name) || field
+          changed ||= !key.equal?(field)
+          [key, again.call(v)]
+        end
+        changed ? self.class.new(fields, extensions, @unknown_fields) : self
+      end
+
+      # @return [Boolean] nothing set, not even unknown fields
+      def empty?
+        @fields.empty? && @extensions.empty? && @unknown_fields.empty?
+      end
+
+      # Protobuf merge semantics: `other`'s fields win, nested messages merge,
+      # repeated fields concatenate
+      # @param other [MessageValue, nil]
+      # @return [MessageValue]
+      def merge(other)
+        return self if other.nil? || other.empty?
+        return other if empty? && other.instance_of?(self.class)
+        fields = MessageValue.merge_values(@fields, other.fields)
+        extensions = MessageValue.merge_values(@extensions, other.extensions)
+        unknown = @unknown_fields.empty? ? other.unknown_fields : (@unknown_fields + other.unknown_fields).freeze
+        self.class.new(fields, extensions, unknown)
+      end
+
+      # @api private
+      def self.merge_values(base, overlay)
+        return base if overlay.empty?
+        return overlay if base.empty?
+        base.merge(overlay) do |_key, old, new|
+          if old.is_a?(MessageValue) && new.is_a?(MessageValue)
+            old.merge(new)
+          elsif old.is_a?(Array) && new.is_a?(Array)
+            (old + new).freeze
+          else
+            new
+          end
+        end.freeze
+      end
+
+      def ==(other)
+        other.instance_of?(self.class) && other.fields == @fields && other.extensions == @extensions &&
+          other.unknown_fields == @unknown_fields
+      end
+      alias_method :eql?, :==
+
+      def hash
+        [self.class, @fields, @extensions, @unknown_fields].hash
+      end
+
+      # A plain Hash for reading: fields by name, extensions under their full
+      # name in parentheses, nested messages as Hashes
+      # @return [Hash]
+      def to_h
+        h = @fields.transform_values { |v| MessageValue.plain(v) }
+        @extensions.each { |field, v| h[:"(#{field.full_name})"] = MessageValue.plain(v) }
+        h[:unknown_fields] = @unknown_fields unless @unknown_fields.empty?
+        h
+      end
+
+      # @api private
+      def self.plain(value)
+        case value
+        when MessageValue then value.to_h
+        when Array then value.map { |v| plain(v) }
+        else value
+        end
+      end
+
+      def inspect
+        parts = @fields.map { |k, v| "#{k}: #{v.inspect}" }
+        @extensions.each { |f, v| parts << "(#{f.full_name}): #{v.inspect}" }
+        parts << "unknown_fields: #{@unknown_fields.bytesize} bytes" unless @unknown_fields.empty?
+        parts.empty? ? "#<#{self.class.name}>" : "#<#{self.class.name} #{parts.join(", ")}>"
+      end
+
+      private
+
+      def frozen(hash)
+        return Freezer::EMPTY_HASH if hash.empty?
+        hash.frozen? ? hash : hash.dup.freeze
+      end
+
+      public
+
+      EMPTY_BYTES = "".b.freeze
+      EMPTY = new.freeze
+    end
+
+    # The declared options of a descriptor: a message of descriptor.proto's
+    # `FileOptions`, `FieldOptions` and so on, one subclass each. Every field
+    # of that message has a reader (booleans also a `?` form); custom options
+    # are {#extension} values; {#features} is what the descriptor resolves to.
+    #
+    #   field.options.deprecated?               # => true
+    #   field.options.packed                    # => nil when not set
+    #   file.options.java_package               # => "com.acme"
+    #   message.options.extension("acme.limits") # => #<Protostar::Schema::MessageValue max: 10>
+    #   field.options[:features]                # => the FeatureSet as declared
+    #   field.options.features                  # => #<Protostar::Schema::Features ...> resolved
+    #
+    # A reader answers what was declared: nil (or an empty Array) when the
+    # option was not set, not descriptor.proto's default.
+    class Options < MessageValue
+      TYPE_NAME = nil
+      FIELDS = Freezer::EMPTY_ARRAY
+
+      class << self
+        # @return [String] full name of the options message, e.g. "google.protobuf.FieldOptions"
+        def type_name
+          self::TYPE_NAME
+        end
+
+        # @api private
+        def define(type_name, *names, booleans: [], repeated: [])
+          klass = Class.new(self)
+          klass.const_set(:TYPE_NAME, -"google.protobuf.#{type_name}")
+          names.each do |name|
+            if repeated.include?(name)
+              klass.define_method(name) { @fields.fetch(name, Freezer::EMPTY_ARRAY) }
+            else
+              klass.define_method(name) { @fields[name] }
+            end
+          end
+          booleans.each do |name|
+            klass.define_method(name) { @fields[name] }
+            klass.define_method(:"#{name}?") { @fields[name] == true }
+          end
+          klass.const_set(:FIELDS, (names + booleans + [:features]).freeze)
+          klass
+        end
+
+        # Options built from what a draft was given: nil, an Options or
+        # MessageValue, or a Hash of field values by name with custom options
+        # under `extensions:` (by full name or {Field}) and wire bytes under
+        # `unknown_fields:`. Message values may be Hashes in the same spelling.
+        # @api private
+        # @param spec [Hash, MessageValue, nil]
+        # @param extension [#call] full name to extension {Field}
+        # @return [Options, nil] unresolved (no {#features})
+        def build(spec, extension, owner)
+          case spec
+          when nil then nil
+          when MessageValue
+            value = spec.rekey(extension)
+            new(value.fields, value.extensions, value.unknown_fields, declared: true)
+          when Hash
+            value = MessageValue.from_hash(spec, extension, owner)
+            new(value.fields, value.extensions, value.unknown_fields, declared: true)
+          else raise Error, "#{owner}: options must be a Hash, got #{spec.class}"
+          end
+        end
+      end
+
+      # @param features [Features, nil] resolved features of the descriptor
+      # @param declared [Boolean] whether the descriptor carries an options message, even an empty one
+      def initialize(fields = Freezer::EMPTY_HASH, extensions = Freezer::EMPTY_HASH, unknown_fields = EMPTY_BYTES,
+        features: nil, declared: false)
+        super(fields, extensions, unknown_fields)
+        @features = features
+        @declared = declared
+      end
+
+      # @return [Features, nil] the features the descriptor resolves to, nil before resolution
+      attr_reader :features
+
+      # @api private
+      # @return [Boolean] whether the descriptor carries an options message (protoc writes `options {}`
+      #   for one declared empty)
+      def declared?
+        @declared
+      end
+
+      # @api private
+      # @return [Options] the same options, bound to resolved features
+      def with_features(features)
+        self.class.new(@fields, @extensions, @unknown_fields, features: features, declared: @declared)
+      end
+
+      # @return [String] e.g. "google.protobuf.FieldOptions"
+      def type_name
+        self.class.type_name
+      end
+
+      def inspect
+        @declared ? super : "#<#{self.class.name} (none declared)>"
+      end
+
+      # Returned by descriptors before they are resolved
+      EMPTY = new.freeze
+
+      # These options as the descriptor.proto message they are (a
+      # `Protostar::Google::Protobuf::FieldOptions`, say), custom options as
+      # extension bytes the message keeps as unknown fields
+      # @return [Protostar::Message]
+      def to_proto
+        klass = Proto.types.const_get(type_name.delete_prefix("google.protobuf."))
+        klass.decode(encode(klass.descriptor))
+      end
+    end
+
+    FileOptions = Options.define("FileOptions", :java_package, :java_outer_classname, :optimize_for, :go_package,
+      :objc_class_prefix, :csharp_namespace, :swift_prefix, :php_class_prefix, :php_namespace,
+      :php_metadata_namespace, :ruby_package,
+      booleans: %i[java_multiple_files java_generate_equals_and_hash java_string_check_utf8 cc_generic_services
+        java_generic_services py_generic_services deprecated cc_enable_arenas])
+    MessageOptions = Options.define("MessageOptions",
+      booleans: %i[message_set_wire_format no_standard_descriptor_accessor deprecated map_entry
+        deprecated_legacy_json_field_conflicts])
+    FieldOptions = Options.define("FieldOptions", :ctype, :jstype, :retention, :targets, :edition_defaults,
+      :feature_support, booleans: %i[packed lazy unverified_lazy deprecated weak debug_redact],
+      repeated: %i[targets edition_defaults])
+    OneofOptions = Options.define("OneofOptions")
+    EnumOptions = Options.define("EnumOptions",
+      booleans: %i[allow_alias deprecated deprecated_legacy_json_field_conflicts])
+    EnumValueOptions = Options.define("EnumValueOptions", :feature_support, booleans: %i[deprecated debug_redact])
+    ServiceOptions = Options.define("ServiceOptions", booleans: %i[deprecated])
+    MethodOptions = Options.define("MethodOptions", :idempotency_level, booleans: %i[deprecated])
+    ExtensionRangeOptions = Options.define("ExtensionRangeOptions", :declaration, :verification,
+      repeated: %i[declaration])
+
+    class MessageValue
+      # A value from a Hash: fields by name, `extensions:` by full name or
+      # {Field}, `unknown_fields:` as bytes. A field named `extensions` or
+      # `unknown_fields` takes a String key. `__extensions:` and
+      # `__unknown_fields:` (as `[number, wire_type, bytes]` triples) are the
+      # deprecated spellings.
+      # @api private
+      def self.from_hash(hash, extension, owner)
+        fields = {}
+        extensions = {}
+        unknown = EMPTY_BYTES
+        hash.each do |key, value|
+          case key
+          when :extensions, :__extensions
+            Protostar.deprecated("__extensions:", "extensions:") if key == :__extensions
+            value.each do |name, v|
+              field = name.is_a?(Field) ? name : extension.call(name.to_s.delete_prefix("."))
+              raise Error, "#{owner}: option extension #{name} is not defined here or in an import" unless field
+              extensions[field] = from_value(v, extension, owner)
+            end
+          when :unknown_fields then unknown = value.b.freeze
+          when :__unknown_fields
+            Protostar.deprecated("__unknown_fields: [[number, wire_type, bytes]]", "unknown_fields: \"bytes\".b")
+            unknown = unknown_bytes(value)
+          else fields[key.to_sym] = from_value(value, extension, owner)
+          end
+        end
+        new(fields.freeze, extensions.freeze, unknown)
+      end
+
+      # @api private
+      def self.from_value(value, extension, owner)
+        case value
+        when MessageValue then value
+        when Hash then from_hash(value, extension, owner)
+        when Array then value.map { |v| from_value(v, extension, owner) }.freeze
+        else Freezer.call(value)
+        end
+      end
+
+      # @api private
+      def self.unknown_bytes(triples)
+        out = "".b
+        triples.each do |number, wire_type, bytes|
+          w = Wire::Writer.new(out)
+          w.write_tag(number, wire_type)
+          w.flush
+          out << bytes.b
+        end
+        out.freeze
+      end
+
+      # This value in wire format
+      # @param type [Message] the message type it is a value of
+      # @return [String] binary
+      def encode(type)
+        Proto::Codec.encode(type, self)
+      end
+
+      # @param type [Message]
+      # @param bytes [String]
+      # @param extensions [#call, nil] (extendee full name, number) to an extension {Field};
+      #   fields it does not know stay in {#unknown_fields}
+      # @return [MessageValue]
+      def self.decode(type, bytes, extensions = nil)
+        Proto::Codec.decode(type, bytes, extensions)
+      end
+    end
+  end
+end

data/lib/protostar/schema/pool.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+module Protostar
+  module Schema
+    # A registry of built files, safe to use from several threads. The pool
+    # itself is mutable and not shareable between Ractors; the files it holds are.
+    class Pool
+      def initialize
+        @mutex = Mutex.new
+        @files = {}
+        @symbols = {}
+        @packages = {}
+        @extensions = {}
+      end
+
+      # Registers a built file and everything it defines.
+      #
+      # An extension whose extendee and number another file already uses is
+      # accepted with a warning, as protoc does; {#find_extension} keeps
+      # returning the first one.
+      # @param file [File]
+      # @param warn [Boolean] write that warning with `Kernel#warn`; the
+      #   compiler passes false and reports it as a diagnostic instead
+      # @return [File]
+      # @raise [Error] when the file name is taken, a symbol or package it
+      #   defines clashes with one already in the pool, or the file itself uses
+      #   an extension number twice
+      def add(file, warn: true)
+        raise Error, "expected a #{File.name}, got #{file.class}" unless file.is_a?(File)
+        raise Error, "#{file.name} is not frozen; finish its draft with freeze!" unless file.frozen?
+        taken = @mutex.synchronize do
+          check_clashes(file)
+          @files[file.name] = file
+          file.symbols.each { |name, symbol| @symbols[name] = symbol }
+          package_prefixes(file.package).each { |pkg| @packages[pkg] ||= file }
+          file.all_extensions.filter_map do |ext|
+            by_number = (@extensions[ext.extendee.full_name] ||= {})
+            existing = by_number[ext.number]
+            by_number[ext.number] = ext unless existing
+            [ext, existing] if existing
+          end
+        end
+        if warn
+          taken.each do |ext, existing|
+            Kernel.warn("#{file.name}: Extension number #{ext.number} has already been used in " \
+              "\"#{ext.extendee.full_name}\" by extension \"#{existing.full_name}\" defined in #{existing.file.name}.")
+          end
+        end
+        file
+      end
+
+      # @return [File, nil]
+      def file(name)
+        @mutex.synchronize { @files[name] }
+      end
+
+      # @return [Boolean] whether a file of that name is registered
+      def include?(file_name)
+        @mutex.synchronize { @files.key?(file_name) }
+      end
+
+      # @return [Array<File>] in the order they were added
+      def files
+        @mutex.synchronize { @files.values }
+      end
+
+      # @param full_name [String] without a leading dot
+      # @return [Descriptor, nil] any symbol: message, field, oneof, enum, enum value, service or method
+      def find_symbol(full_name)
+        @mutex.synchronize { @symbols[full_name] }
+      end
+
+      # @return [Message, nil]
+      def find_message(full_name)
+        found = find_symbol(full_name)
+        found if found.is_a?(Message)
+      end
+
+      # @return [Enum, nil]
+      def find_enum(full_name)
+        found = find_symbol(full_name)
+        found if found.is_a?(Enum)
+      end
+
+      # @return [Service, nil]
+      def find_service(full_name)
+        found = find_symbol(full_name)
+        found if found.is_a?(Service)
+      end
+
+      # @param extendee [String, Message] the extended message or its full name
+      # @param number [Integer]
+      # @return [Field, nil]
+      def find_extension(extendee, number)
+        extendee = extendee.full_name if extendee.is_a?(Message)
+        @mutex.synchronize { @extensions[extendee]&.[](number) }
+      end
+
+      # @param extendee [String, Message]
+      # @return [Array<Field>] every registered extension of the message, by number
+      def extensions_of(extendee)
+        extendee = extendee.full_name if extendee.is_a?(Message)
+        @mutex.synchronize { (@extensions[extendee] || {}).values.sort_by(&:number) }
+      end
+
+      def inspect
+        "#<#{self.class.name} #{@files.size} files>"
+      end
+
+      # Builds and adds a serialized FileDescriptorProto, unless a file of that name is here
+      # @return [File]
+      def add_serialized_file(bytes)
+        proto = Proto.types::FileDescriptorProto.decode(bytes)
+        file(proto.name) || add(File.from_proto(proto, pool: self))
+      end
+
+      # @see Protostar.load_descriptor_set
+      def load_descriptor_set(bytes)
+        Protostar.load_descriptor_set(bytes, pool: self)
+      end
+
+      private
+
+      def check_clashes(file)
+        raise Error, "file \"#{file.name}\" is already in the pool" if @files.key?(file.name)
+        file.symbols.each do |name, symbol|
+          if (other = @symbols[name])
+            raise Error, "\"#{name}\" is already defined in file \"#{other.file.name}\" (redefined in \"#{file.name}\")"
+          end
+          if (pkg_file = @packages[name])
+            raise Error, "\"#{name}\" is already defined as a package in file \"#{pkg_file.name}\" " \
+              "(redefined as a #{symbol.class.name.split("::").last} in \"#{file.name}\")"
+          end
+        end
+        package_prefixes(file.package).each do |pkg|
+          other = @symbols[pkg] || file.symbols[pkg]
+          next unless other
+          raise Error, "\"#{pkg}\" is already defined (as something other than a package) in file \"#{other.file.name}\""
+        end
+        seen = {}
+        file.all_extensions.each do |ext|
+          key = [ext.extendee.full_name, ext.number]
+          if (existing = seen[key])
+            raise Error, "extension number #{ext.number} of \"#{ext.extendee.full_name}\" is already used by " \
+              "\"#{existing.full_name}\" (redefined by \"#{ext.full_name}\")"
+          end
+          seen[key] = ext
+        end
+      end
+
+      def package_prefixes(package)
+        return [] if package.empty?
+        parts = package.split(".")
+        (1..parts.size).map { |n| parts.first(n).join(".") }
+      end
+    end
+  end
+end
+
+module Protostar
+  @generated_pool = Schema::Pool.new
+
+  class << self
+    # @return [Schema::Pool] the process-wide pool generated `_pb.rb` files build into
+    attr_reader :generated_pool
+
+    # Builds every file of a serialized `FileDescriptorSet` into a pool, in
+    # order. Custom options are decoded against the extensions the pool (and
+    # the file itself) defines; files already in the pool are taken from it.
+    # @param bytes [String] a serialized google.protobuf.FileDescriptorSet
+    # @param pool [Schema::Pool]
+    # @return [Array<Schema::File>]
+    def load_descriptor_set(bytes, pool:)
+      Schema::Proto.each_file(bytes).map { |file_bytes| pool.add_serialized_file(file_bytes) }
+    end
+  end
+
+  Pool = Schema::Pool
+end