protostar 0.1.0.beta1

data/lib/protostar/codec/sizer.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Protostar
+  module Codec
+    # The first encoding pass computes the encoded size of a message and of
+    # every message and packed run inside it, recorded in `sizes` (an
+    # identity Hash that lives for one encode) so the writer knows each
+    # length prefix before it writes the bytes it covers. The per-class
+    # `__size` methods ({EncodeGenerator}) do the work; this module has the
+    # parts they share.
+    # @api private
+    module Sizer
+      F = Message::FieldInfo
+
+      module_function
+
+      # @param msg [Message]
+      # @param sizes [Hash] compare_by_identity
+      # @return [Integer]
+      def message(msg, sizes)
+        msg.__size(sizes)
+      end
+
+      # Size of one value, without its tag; a group's end tag is not included either
+      def value(info, v, sizes)
+        case info.kind
+        when F::INT32, F::INT64, F::UINT32, F::UINT64 then Wire.varint_size(v)
+        when F::ENUM then Wire.varint_size(Encoder.enum_number(info, v))
+        when F::SINT32, F::SINT64 then Wire.varint_size((v << 1) ^ (v >> 63))
+        when F::BOOL then 1
+        when F::STRING, F::BYTES
+          n = v.bytesize
+          Wire.varint_size(n) + n
+        when F::MESSAGE
+          n = v.__size(sizes)
+          Wire.varint_size(n) + n
+        when F::GROUP then v.__size(sizes)
+        when F::DOUBLE, F::FIXED64, F::SFIXED64 then 8
+        else 4
+        end
+      end
+
+      # Payload size of a packed run, without tag or length prefix
+      def packed(info, list)
+        list.each { |e| check(info, e) }
+        case info.kind
+        when F::DOUBLE, F::FIXED64, F::SFIXED64 then 8 * list.size
+        when F::FLOAT, F::FIXED32, F::SFIXED32 then 4 * list.size
+        when F::BOOL then list.size
+        when F::SINT32, F::SINT64 then list.sum { |v| Wire.varint_size((v << 1) ^ (v >> 63)) }
+        when F::ENUM then list.sum { |v| Wire.varint_size(Encoder.enum_number(info, v)) }
+        else list.sum { |v| Wire.varint_size(v) }
+        end
+      end
+
+      # Size of every entry of a map field, tags and length prefixes included
+      def map(info, hash, sizes)
+        tag = info.tag.bytesize
+        total = 0
+        hash.each do |key, val|
+          n = entry(info, key, val, sizes)
+          total += tag + Wire.varint_size(n) + n
+        end
+        total
+      end
+
+      # Size of a map entry message: key field then value field, both always written
+      def entry(info, key, val, sizes)
+        k = info.key
+        v = info.value
+        check(k, key)
+        check(v, val)
+        k.tag.bytesize + value(k, key, sizes) + v.tag.bytesize + value(v, val, sizes)
+      end
+
+      # @param list [Array<UnknownField>]
+      def unknown(list)
+        list.sum { |u| Wire.varint_size((u.number << 3) | u.wire_type) + u.bytes.bytesize }
+      end
+
+      def check(info, value)
+        invalid(info, value) unless Message::Coercion.storable?(info, value)
+      end
+
+      # @raise [EncodeError]
+      def invalid(info, value)
+        raise EncodeError, "#{info.descriptor.full_name} holds #{value.inspect[0, 64]}, which it cannot encode"
+      end
+    end
+  end
+end

data/lib/protostar/codec.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Protostar
+  # Binary encoding and decoding of {Message} objects. Use the methods on
+  # message classes and instances; these modules are their implementation.
+  module Codec
+    # Needed while a class is defined, never after
+    autoload :DecodeGenerator, ::File.expand_path("codec/decode_generator", __dir__)
+    autoload :EncodeGenerator, ::File.expand_path("codec/encode_generator", __dir__)
+  end
+end
+
+require_relative "codec/decoder"
+require_relative "codec/sizer"
+require_relative "codec/encoder"

data/lib/protostar/compiler/ast.rb

@@ -0,0 +1,193 @@
+# frozen_string_literal: true
+
+module Protostar
+  class Compiler
+    # The parse tree of a `.proto` file. It holds what protoc's parser puts
+    # in a `FileDescriptorProto` before linking: names as written, options
+    # uninterpreted, map entries, groups and the synthetic oneofs of proto3
+    # `optional` fields already expanded, as protoc's parser expands them.
+    #
+    # Every node records where its parts start in {Node#positions}, keyed by
+    # the error locations protoc's DescriptorPool reports against (`:name`,
+    # `:number`, `:type`, `:extendee`, `:default_value`, `:option_name`,
+    # `:option_value`, `:input_type`, `:output_type`, `:editions`, `:other`),
+    # each a 0-based `[line, column]`.
+    module AST
+      # @api private
+      class Node
+        # @return [Hash{Symbol => Array(Integer, Integer)}]
+        attr_reader :positions
+        # Linker state: full name and source path
+        attr_accessor :full_name, :path
+
+        def initialize
+          @positions = {}
+        end
+
+        # @return [Array(Integer, Integer), nil]
+        def position(kind)
+          @positions[kind]
+        end
+      end
+
+      # A whole `.proto` file
+      class File < Node
+        attr_accessor :name, :package, :syntax, :edition, :options
+        attr_reader :dependency, :public_dependency, :weak_dependency, :option_dependency,
+          :message_type, :enum_type, :service, :extension, :import_positions
+        # @return [Array<SourceInfo::Location>] in protoc's order
+        attr_accessor :locations
+
+        def initialize
+          super
+          @dependency = []
+          @public_dependency = []
+          @weak_dependency = []
+          @option_dependency = []
+          @message_type = []
+          @enum_type = []
+          @service = []
+          @extension = []
+          @import_positions = {}
+          @locations = []
+        end
+      end
+
+      # The options of one element: the uninterpreted `option` statements and
+      # the few options protoc's parser sets itself (`map_entry`)
+      class Options
+        # @return [Array<Option>]
+        attr_reader :uninterpreted
+        # @return [Hash] options set directly, e.g. `{map_entry: true}`
+        attr_reader :known
+
+        def initialize
+          @uninterpreted = []
+          @known = {}
+        end
+
+        def empty?
+          @uninterpreted.empty? && @known.empty?
+        end
+
+        # A copy whose options carry no positions, as protoc's message copy
+        # leaves them out of its location table
+        def copy
+          other = Options.new
+          other.known.merge!(@known)
+          @uninterpreted.each { |o| other.uninterpreted << o.copy }
+          other
+        end
+      end
+
+      # One uninterpreted option: `(my.ext).sub.field = value`
+      class Option < Node
+        # @return [Array<Array(String, Boolean)>] name parts and whether each is an extension
+        attr_reader :name
+        # @return [Symbol, nil] which value field is set: :identifier_value,
+        #   :positive_int_value, :negative_int_value, :double_value,
+        #   :string_value or :aggregate_value
+        attr_accessor :value_kind
+        attr_accessor :value
+
+        def initialize
+          super
+          @name = []
+        end
+
+        def copy
+          other = Option.new
+          @name.each { |part| other.name << part }
+          other.value_kind = @value_kind
+          other.value = @value
+          other
+        end
+
+        # @return [String] the name as protoc prints it in errors: `(a.b).c`
+        def display_name
+          @name.map { |part, ext| ext ? "(#{part})" : part }.join(".")
+        end
+      end
+
+      # A message, including synthesized map entries and group bodies
+      class Message < Node
+        attr_accessor :name, :options, :visibility
+        attr_accessor :parent, :index
+        attr_reader :field, :nested_type, :enum_type, :extension_range, :extension, :oneof_decl,
+          :reserved_range, :reserved_name
+
+        def initialize
+          super
+          @field = []
+          @nested_type = []
+          @enum_type = []
+          @extension_range = []
+          @extension = []
+          @oneof_decl = []
+          @reserved_range = []
+          @reserved_name = []
+        end
+      end
+
+      # A field or an extension
+      class Field < Node
+        attr_accessor :name, :number, :label, :type, :type_name, :extendee, :default_value, :json_name,
+          :oneof_index, :proto3_optional, :options
+        # Set by the linker: the declaring message (nil at file level), whether
+        # this is an extension, and its index
+        attr_accessor :parent, :extension, :index
+      end
+
+      # A `oneof` declaration
+      class Oneof < Node
+        attr_accessor :name, :options
+        attr_accessor :parent, :index
+      end
+
+      # An `extensions` range (end exclusive) or a `reserved` range
+      # (end exclusive for messages, inclusive for enums)
+      class Range < Node
+        attr_accessor :start, :end, :options
+        attr_accessor :parent, :index
+      end
+
+      # An enum
+      class Enum < Node
+        attr_accessor :name, :options, :visibility
+        attr_accessor :parent, :index
+        attr_reader :value, :reserved_range, :reserved_name
+
+        def initialize
+          super
+          @value = []
+          @reserved_range = []
+          @reserved_name = []
+        end
+      end
+
+      # An enum value
+      class EnumValue < Node
+        attr_accessor :name, :number, :options
+        attr_accessor :parent, :index
+      end
+
+      # A service
+      class Service < Node
+        attr_accessor :name, :options
+        attr_accessor :index
+        attr_reader :rpcs
+
+        def initialize
+          super
+          @rpcs = []
+        end
+      end
+
+      # An rpc
+      class Method < Node
+        attr_accessor :name, :input_type, :output_type, :options, :client_streaming, :server_streaming
+        attr_accessor :parent, :index
+      end
+    end
+  end
+end

data/lib/protostar/compiler/emitters/descriptor_set.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Protostar
+  class Compiler
+    module Emitters
+      # Writes a serialized `google.protobuf.FileDescriptorSet`, as protoc's
+      # `--descriptor_set_out` does
+      class DescriptorSet
+        # @param include_imports [Boolean] also write every file the given ones import,
+        #   dependencies first (`import option` files included, as protoc does)
+        # @param include_source_info [Boolean] keep `source_code_info`
+        # @param pool [Schema::Pool, nil] where `import option` files and their extensions are found
+        def initialize(include_imports: false, include_source_info: false, pool: nil)
+          @include_imports = include_imports
+          @include_source_info = include_source_info
+          @pool = pool
+        end
+
+        # @param files [Array<Schema::File>] the files named on the command line
+        # @param available [Array<Schema::File>] files `import option` names are looked up in
+        #   before the pool, such as everything {Compiler#compile} returned
+        # @return [String] binary
+        def emit(files, available: [])
+          out = "".b
+          w = Wire::Writer.new(out)
+          @available = available.to_h { |f| [f.name, f] }
+          ordered(files).each do |file|
+            w.write_tag(1, Wire::LENGTH_DELIMITED)
+            w.write_bytes(file.serialize(source_info: @include_source_info))
+          end
+          out
+        end
+
+        private
+
+        # protoc's GetTransitiveDependencies: depth first, each file after its imports
+        def ordered(files)
+          return files unless @include_imports
+          seen = {}
+          out = []
+          visit = lambda do |file|
+            next if seen[file.name]
+            seen[file.name] = true
+            file.dependency_files.each { |dep| visit.call(dep) }
+            file.option_dependencies.each do |name|
+              dep = @available[name] || @pool&.file(name)
+              visit.call(dep) if dep
+            end
+            out << file
+          end
+          files.each { |f| visit.call(f) }
+          out
+        end
+      end
+    end
+  end
+end

data/lib/protostar/compiler/emitters/google_ruby.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Protostar
+  class Compiler
+    module Emitters
+      # Writes the `_pb.rb` file protoc's own Ruby generator writes
+      # (ruby_generator.cc): the serialized descriptor added to
+      # google-protobuf's generated pool, and constants looked up from it.
+      class GoogleRuby
+        # @param pool [Schema::Pool, nil] where extensions used in options but defined in
+        #   `import option` files are found
+        def initialize(pool: nil)
+          @pool = pool
+        end
+
+        # @param files [Array<Schema::File>]
+        # @return [Hash{String => String}] output path to source
+        def emit(files)
+          files.to_h { |f| [Naming.require_name(f.name) + ".rb", file(f)] }
+        end
+
+        # @param file [Schema::File]
+        # @return [String]
+        # @raise [Emitters::Error] for a `ruby_package` protoc refuses
+        def file(file)
+          Naming.check_ruby_package(file)
+          p = Printer.new
+          p.print("# frozen_string_literal: true\n# Generated by the protocol buffer compiler.  DO NOT EDIT!\n" \
+            "# source: #{file.name}\n\n")
+          p.print("require 'google/protobuf'\n\n")
+          unless file.dependencies.empty?
+            file.dependencies.each { |dep| p.print("require '#{Naming.require_name(dep)}'\n") }
+            p.print("\n")
+          end
+          p.print("\ndescriptor_data = \"#{GoogleRuby.ruby_escape(GoogleRuby.serialized(file))}\"\n\n" \
+            "pool = ::Google::Protobuf::DescriptorPool.generated_pool\npool.add_serialized_file(descriptor_data)\n\n")
+          modules = Naming.module_path(file)
+          modules.each do |m|
+            p.print("module #{m}\n")
+            p.indent
+          end
+          file.messages.each { |m| message_assignment(p, "", m) }
+          file.enums.each { |e| enum_assignment(p, "", e) }
+          modules.size.times do
+            p.outdent
+            p.print("end\n")
+          end
+          p.to_s
+        end
+
+        # What protoc embeds: the file as `FileDescriptor::CopyTo` writes it,
+        # without source info and with `json_name` only where the .proto set
+        # one. Whether it was set is not kept, so a `json_name` equal to the
+        # derived one is taken as not set.
+        # @return [String] binary
+        def self.serialized(file)
+          file.serialize(source_info: false, default_json_names: false)
+        end
+
+        PRINTABLE = (0x20..0x7E)
+        HEX = /[0-9A-Fa-f]/
+        C_ESCAPES = {"\n" => "\\n", "\r" => "\\r", "\t" => "\\t", "\"" => "\\\"", "'" => "\\'", "\\" => "\\\\"}.freeze
+
+        # absl::CHexEscape, then a backslash before every `#` that would
+        # start an interpolation: ruby_generator.cc's RubyEscape
+        def self.ruby_escape(bytes)
+          escaped = +""
+          last_hex = false
+          bytes.each_byte do |b|
+            c = b.chr
+            hex = false
+            if (e = C_ESCAPES[c])
+              escaped << e
+            elsif !PRINTABLE.cover?(b) || (last_hex && HEX.match?(c))
+              escaped << format("\\x%02x", b)
+              hex = true
+            else
+              escaped << c
+            end
+            last_hex = hex
+          end
+          escaped.gsub(/#(?=[{$@])/) { "\\#" }
+        end
+
+        private
+
+        def message_assignment(p, prefix, message)
+          return if message.map_entry?
+          name = Naming.rubify_constant(message.name)
+          p.print("#{prefix}#{name} = ::Google::Protobuf::DescriptorPool.generated_pool." \
+            "lookup(\"#{message.full_name}\").msgclass\n")
+          message.nested_messages.each { |m| message_assignment(p, "#{prefix}#{name}::", m) }
+          message.nested_enums.each { |e| enum_assignment(p, "#{prefix}#{name}::", e) }
+        end
+
+        def enum_assignment(p, prefix, enum)
+          p.print("#{prefix}#{Naming.rubify_constant(enum.name)} = ::Google::Protobuf::DescriptorPool.generated_pool." \
+            "lookup(\"#{enum.full_name}\").enummodule\n")
+        end
+      end
+    end
+  end
+end

data/lib/protostar/compiler/emitters/grpc.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module Protostar
+  class Compiler
+    module Emitters
+      # Writes the `_services_pb.rb` stubs grpc-tools' Ruby plugin writes
+      # (grpc's src/compiler/ruby_generator.cc). Comments come from source
+      # info, so compile with it.
+      class Grpc
+        # @param files [Array<Schema::File>]
+        # @return [Hash{String => String}] output path to source, for files with services only
+        def emit(files)
+          files.each_with_object({}) do |f, out|
+            source = file(f)
+            out[Grpc.output_name(f.name)] = source unless source.empty?
+          end
+        end
+
+        # @param name [String] "foo/bar.proto"
+        # @return [String] "foo/bar_services_pb.rb"
+        # @raise [Emitters::Error] for a name protoc's plugin refuses
+        def self.output_name(name)
+          unless name.size > 6 && ".proto".include?(name[-1])
+            raise Error, "Invalid proto file name:  must end with .proto"
+          end
+          name[0, name.size - 6] + "_services_pb.rb"
+        end
+
+        # @param file [Schema::File]
+        # @return [String] empty when the file declares no services
+        def file(file)
+          return "" if file.services.empty?
+          @source = file.source_info
+          package = ruby_package(file)
+          p = Printer.new
+          p.print("# Generated by the protocol buffer compiler.  DO NOT EDIT!\n")
+          p.print("# Source: #{file.name} for package '#{package}'\n")
+          file_comments = file_comments(file)
+          unless file_comments.empty?
+            p.print("# Original file comments:\n")
+            p.print(file_comments)
+          end
+          p.print("\nrequire 'grpc'\n")
+          p.print("require '#{file.name.sub(".proto", "_pb")}'\n\n")
+          modules = split(package)
+          modules.each do |m|
+            p.print("module #{Naming.package_to_module(m)}\n")
+            p.indent
+          end
+          file.services.each_with_index { |s, i| service(p, s, [6, i]) }
+          modules.size.times do
+            p.outdent
+            p.print("end\n")
+          end
+          p.to_s
+        ensure
+          @source = nil
+        end
+
+        private
+
+        def service(p, service, path)
+          return if service.rpcs.empty?
+          p.print("module #{modularize(service.name)}\n")
+          p.indent
+          p.print(comments(path, true))
+          p.print("class Service\n")
+          p.indent
+          p.print("\ninclude ::GRPC::GenericService\n\n")
+          p.print("self.marshal_class_method = :encode\nself.unmarshal_class_method = :decode\n")
+          p.print("self.service_name = '#{service.full_name}'\n\n")
+          service.rpcs.each_with_index do |rpc, i|
+            input = type_of(rpc.input_type)
+            input = "stream(#{input})" if rpc.client_streaming?
+            output = type_of(rpc.output_type)
+            output = "stream(#{output})" if rpc.server_streaming?
+            p.print(comments(path + [2, i], true))
+            p.print("rpc :#{rpc.name}, #{input}, #{output}\n")
+            p.print(comments(path + [2, i], false))
+          end
+          p.outdent
+          p.print("end\n\nStub = Service.rpc_stub_class\n")
+          p.outdent
+          p.print("end\n")
+          p.print(comments(path, false))
+        end
+
+        # The package in dotted form, `ruby_package`'s `::` turned into dots
+        def ruby_package(file)
+          file.options.ruby_package&.gsub("::", ".") || file.package
+        end
+
+        # std::getline splitting: no trailing empty piece
+        def split(text)
+          parts = text.split(".", -1)
+          parts.pop if parts.last == ""
+          parts
+        end
+
+        def modularize(name)
+          return name if name.empty?
+          out = name[0].upcase(:ascii)
+          last_underscore = false
+          name[1..].each_char do |c|
+            if last_underscore && c != "_"
+              out << c.upcase(:ascii)
+            elsif c != "_"
+              out << c
+            end
+            last_underscore = c == "_"
+          end
+          out
+        end
+
+        # grpc's RubyTypeOf: the message's Ruby constant path as grpc spells it
+        def type_of(message)
+          name = message.full_name
+          file = message.file
+          if file.options.ruby_package
+            name = name.delete_prefix(file.package) if name.start_with?(file.package)
+            name = name.delete_prefix(".")
+            name = "#{ruby_package(file)}.#{name}"
+          end
+          parts = split(".#{name}")
+          parts.each_with_index.map do |part, i|
+            (i < parts.size - 1) ? modularize(part) : Naming.rubify_constant(part)
+          end.join("::")
+        end
+
+        def lines(text)
+          return [] if text.nil?
+          parts = text.split("\n", -1)
+          parts.pop if parts.last == ""
+          parts
+        end
+
+        # GetPrefixedComments: detached and leading comments, or trailing ones
+        def comments(path, leading)
+          location = @source&.location(path)
+          return "" unless location
+          out = []
+          if leading
+            location.leading_detached_comments.each { |c| out.concat(lines(c)) << "" }
+            out.concat(lines(location.leading_comments))
+          else
+            out.concat(lines(location.trailing_comments))
+          end
+          prefixed(out)
+        end
+
+        def prefixed(out)
+          out.map do |line|
+            if line.empty?
+              "#\n"
+            elsif line.start_with?(" ")
+              "##{line}\n"
+            else
+              "# #{line}\n"
+            end
+          end.join
+        end
+
+        # The comments above the syntax line, printed raw: a `$` comes out doubled
+        def file_comments(file)
+          location = @source&.location([12])
+          return "" unless location
+          out = []
+          location.leading_detached_comments.each { |c| out.concat(lines(c)) << "" }
+          out.concat(lines(location.leading_comments))
+          prefixed(out).gsub("$", "$$")
+        end
+      end
+    end
+  end
+end