protostar 0.1.0.beta1

data/lib/protostar/compiler/emitters/ruby.rb

@@ -0,0 +1,503 @@
+# frozen_string_literal: true
+
+module Protostar
+  class Compiler
+    module Emitters
+      # Writes protostar's own `_pb.rb` files: a {Protostar::DSL} block
+      # describing the file, then constants named as protoc's Ruby generator
+      # names them.
+      class Ruby
+        # source_code_info path components (descriptor.proto field numbers)
+        FILE_MESSAGE = 4
+        FILE_ENUM = 5
+        FILE_SERVICE = 6
+        FILE_EXTENSION = 7
+        MESSAGE_FIELD = 2
+        MESSAGE_NESTED = 3
+        MESSAGE_ENUM = 4
+        MESSAGE_EXTENSION = 6
+        MESSAGE_ONEOF = 8
+        ENUM_VALUE = 2
+        SERVICE_METHOD = 2
+
+        # @param namespaces [Hash{String => String}] Ruby module paths ("A::B") by file name, overriding
+        #   the package-derived modules; the bootstrap puts google.protobuf under Protostar::Google::Protobuf
+        # @param bootstrap_requires [Boolean] require imports of {BOOTSTRAP} files as
+        #   "protostar/google/protobuf/..._pb" rather than by their own names
+        def initialize(namespaces: {}, bootstrap_requires: true)
+          @namespaces = namespaces
+          @bootstrap_requires = bootstrap_requires
+        end
+
+        # @param files [Array<Schema::File>]
+        # @return [Hash{String => String}] output path to source
+        def emit(files)
+          files.to_h { |f| [Naming.require_name(f.name) + ".rb", file(f)] }
+        end
+
+        # @param file [Schema::File]
+        # @return [String] the generated source
+        def file(file)
+          Naming.check_ruby_package(file)
+          @file = file
+          @source = file.source_info
+          @out = +""
+          @out << "# frozen_string_literal: true\n\n"
+          @out << "# Generated by protostar from #{file.name}. Do not edit.\n\n"
+          @out << "require \"protostar\"\n"
+          file.dependencies.each { |dep| @out << "require #{Literal.string(require_path(dep))}\n" }
+          @out << "\n"
+          emit_file(file)
+          emit_constants(file)
+          @out
+        ensure
+          @file = @source = @reachable = nil
+        end
+
+        # Only the {Protostar::DSL.file} statement, without requires and constants
+        # @param file [Schema::File]
+        # @return [String]
+        def definition(file)
+          @file = file
+          @source = file.source_info
+          @out = +""
+          emit_file(file)
+          @out
+        ensure
+          @file = @source = @reachable = nil
+        end
+
+        # @param name [String] an imported file name
+        # @return [String] what a generated file requires for it
+        def require_path(name)
+          if @bootstrap_requires && BOOTSTRAP.include?(name)
+            "protostar/#{Naming.require_name(name)}"
+          else
+            Naming.require_name(name)
+          end
+        end
+
+        private
+
+        def emit_file(file)
+          args = [Literal.string(file.name)]
+          args << "package: #{Literal.string(file.package)}" unless file.package.empty?
+          if file.edition
+            args << "syntax: :editions" unless file.proto_syntax == "editions"
+            args << "edition: #{file.edition.inspect}"
+          elsif file.proto_syntax
+            args << "syntax: #{file.proto_syntax.to_sym.inspect}"
+          end
+          deps = file.dependencies
+          args << ["dependencies: ", deps] unless deps.empty?
+          [[:public_dependencies, file.public_dependencies], [:weak_dependencies, file.weak_dependencies]].each do |key, list|
+            args << ["#{key}: ", list.map { |i| deps[i] }] unless list.empty?
+          end
+          args << ["option_dependencies: ", file.option_dependencies] unless file.option_dependencies.empty?
+          args.concat(option_args(file.options))
+          body = +""
+          file.messages.each_with_index { |m, i| message(body, 1, "f", m, [FILE_MESSAGE, i]) }
+          file.enums.each_with_index { |e, i| enum(body, 1, "f", e, [FILE_ENUM, i]) }
+          extensions(body, 1, "f", file.extensions, [FILE_EXTENSION])
+          file.services.each_with_index { |s, i| service(body, 1, "f", s, [FILE_SERVICE, i]) }
+          if body.empty?
+            @out << call("Protostar::DSL.file", args, 0) << "\n"
+          else
+            @out << call("Protostar::DSL.file", args, 0) << " do |f|\n" << body << "end\n"
+          end
+        end
+
+        def message(out, depth, var, desc, path)
+          comment(out, depth, path)
+          args = [Literal.string(desc.name)]
+          args << "visibility: #{desc.visibility.inspect}" if desc.visibility
+          args.concat(option_args(desc.options))
+          head = call("#{var}.message", args, depth)
+          body = +""
+          message_body(body, depth + 1, desc, path)
+          out << ind(depth) << head << (body.empty? ? "\n" : " do |m|\n#{body}#{ind(depth)}end\n")
+        end
+
+        def message_body(out, depth, desc, path)
+          fields = desc.fields
+          nested = desc.nested_messages
+          oneofs = desc.oneofs
+          owned = owned_nested(desc, fields)
+          canonical = canonical_oneofs?(fields, oneofs)
+          unless canonical
+            oneofs.each do |o|
+              out << ind(depth) << call("m.oneof", [o.name.to_sym.inspect] + option_args(o.options), depth) << "\n"
+            end
+          end
+          items = field_items(fields, canonical)
+          emitted = {}
+          flush = lambda do |limit|
+            nested.each_index do |j|
+              next if j >= limit || emitted[j] || owned.value?(j)
+              emitted[j] = true
+              message(out, depth, "m", nested[j], path + [MESSAGE_NESTED, j])
+            end
+          end
+          items.each do |oneof_index, indexes|
+            owned_here = indexes.filter_map { |i| owned[i] }
+            flush.call(owned_here.max) unless owned_here.empty?
+            if oneof_index
+              comment(out, depth, path + [MESSAGE_ONEOF, oneof_index])
+              o = oneofs[oneof_index]
+              out << ind(depth) << call("m.oneof", [o.name.to_sym.inspect] + option_args(o.options), depth)
+              out << " do |o|\n"
+              indexes.each { |i| field(out, depth + 1, "o", desc, fields[i], owned[i], path + [MESSAGE_FIELD, i], false) }
+              out << ind(depth) << "end\n"
+            else
+              indexes.each { |i| field(out, depth, "m", desc, fields[i], owned[i], path + [MESSAGE_FIELD, i], !canonical) }
+            end
+            owned_here.each { |j| emitted[j] = true }
+          end
+          flush.call(nested.size)
+          desc.nested_enums.each_with_index { |e, i| enum(out, depth, "m", e, path + [MESSAGE_ENUM, i]) }
+          extensions(out, depth, "m", desc.extensions, path + [MESSAGE_EXTENSION])
+          desc.extension_range_descriptors.each do |r|
+            out << ind(depth) << call("m.extensions", [range(r.start, r.end)] + option_args(r.options), depth) << "\n"
+          end
+          reserved = desc.reserved_ranges.map { |r| range(r.begin, r.end) }
+          reserved.concat(desc.reserved_names.map { |n| Literal.string(n) })
+          out << ind(depth) << call("m.reserved", reserved, depth) << "\n" unless reserved.empty?
+        end
+
+        # Fields grouped for emission: [oneof index or nil, [field indexes]]
+        def field_items(fields, canonical)
+          return fields.each_index.map { |i| [nil, [i]] } unless canonical
+          items = []
+          fields.each_with_index do |f, i|
+            o = f.oneof&.index
+            if o && !f.proto3_optional?
+              if items.last && items.last[0] == o
+                items.last[1] << i
+              else
+                items << [o, [i]]
+              end
+            else
+              items << [nil, [i]]
+            end
+          end
+          items
+        end
+
+        # Whether the oneofs are laid out as the DSL's block form and synthetic
+        # oneofs reproduce them: real oneofs in order of their contiguous
+        # members, then one synthetic oneof per proto3 optional field, named
+        # as protoc's parser names them
+        def canonical_oneofs?(fields, oneofs)
+          return true if oneofs.empty?
+          real = []
+          synthetic = []
+          fields.each_with_index do |f, i|
+            next unless (o = f.oneof&.index)
+            if f.proto3_optional?
+              synthetic << [o, f]
+            else
+              return false if real.include?(o) && real.last != o
+              return false if real.include?(o) && fields[i - 1].oneof&.index != o
+              real << o unless real.include?(o)
+            end
+          end
+          return false unless real == (0...real.size).to_a
+          return false unless synthetic.map(&:first) == (real.size...oneofs.size).to_a
+          taken = fields.map(&:name) + oneofs.first(real.size).map(&:name)
+          names = Schema::Draft.synthetic_oneof_names(synthetic.map { |_, f| f.name }, taken)
+          synthetic.zip(names).all? do |(o, _), name|
+            oneofs[o].name == name && !oneofs[o].options.declared?
+          end
+        end
+
+        # Nested types written through `map` or `group`: field index to nested index
+        def owned_nested(desc, fields)
+          owned = {}
+          fields.each_with_index do |f, i|
+            nested = f.message_type
+            next unless nested && nested.containing_message.equal?(desc) && nested.file.equal?(desc.file)
+            owned[i] = nested.index if map_sugar(f, nested) || group_sugar?(f, nested)
+          end
+          # the sugar appends its nested type when called, so the order must agree
+          return {} unless owned.values == owned.values.sort
+          owned
+        end
+
+        # [key type, value type, value type name] when `m.map` makes exactly this entry
+        def map_sugar(f, entry)
+          return nil unless f.declared_label == :repeated && entry.name == Schema::Draft.map_entry_name(f.name)
+          return nil if f.raw_default || f.oneof || f.proto3_optional?
+          return nil unless only?(entry.options, map_entry: true)
+          return nil unless entry.nested_messages.empty? && entry.nested_enums.empty? && entry.extensions.empty? &&
+            entry.extension_range_descriptors.empty? && entry.oneofs.empty? && entry.reserved_ranges.empty? &&
+            entry.reserved_names.empty? && entry.visibility.nil?
+          key, value = entry.fields
+          return nil unless entry.fields.size == 2 && entry_field?(key, "key", 1) && entry_field?(value, "value", 2)
+          return nil if key.type_name
+          features = f.options.declared? ? f.options[:features] : nil
+          [key, value].each do |kv|
+            if features
+              return nil unless only?(kv.options, features: features)
+            elsif kv.options.declared?
+              return nil
+            end
+          end
+          [key.declared_type, value.declared_type, value.type_name && ".#{value.type_name}"]
+        end
+
+        def entry_field?(field, name, number)
+          field.name == name && field.number == number && field.declared_label == :optional &&
+            field.json_name == name && field.raw_default.nil? && field.oneof.nil? && !field.proto3_optional?
+        end
+
+        # Options declared with exactly these fields, no custom or unknown ones
+        def only?(options, **fields)
+          options.declared? && options.fields == fields && options.extensions.empty? && options.unknown_fields.empty?
+        end
+
+        def group_sugar?(f, nested)
+          f.declared_type == :group && f.name == Schema::Draft.group_field_name(nested.name) &&
+            !f.proto3_optional? && f.raw_default.nil?
+        end
+
+        def field(out, depth, var, desc, f, owned, path, explicit_oneof)
+          comment(out, depth, path)
+          nested = owned && desc.nested_messages[owned]
+          if nested && (sugar = map_sugar(f, nested))
+            key_type, value_type, value_type_name = sugar
+            args = [f.name.to_sym.inspect, f.number.to_s, key_type.inspect, value_type.inspect]
+            args << Literal.string(value_type_name) if value_type_name
+            args << "json_name: #{Literal.string(f.json_name)}" if custom_json_name?(f)
+            args.concat(option_args(f.options))
+            out << ind(depth) << call("#{var}.map", args, depth) << "\n"
+            return
+          end
+          if nested
+            args = [Literal.string(nested.name), f.number.to_s]
+            args << "label: #{f.declared_label.inspect}" unless f.declared_label == :optional
+            args << "json_name: #{Literal.string(f.json_name)}" if custom_json_name?(f)
+            args << "oneof: #{f.oneof.name.to_sym.inspect}" if f.oneof && explicit_oneof
+            args.concat(option_args(f.options))
+            args << ["message_options: ", dsl_hash(nested.options)] if nested.options.declared?
+            body = +""
+            message_body(body, depth + 1, nested, path_for_nested(owned, path))
+            head = call("#{var}.group", args, depth)
+            out << ind(depth) << head << (body.empty? ? "\n" : " do |m|\n#{body}#{ind(depth)}end\n")
+            return
+          end
+          args = field_args(f)
+          args << "proto3_optional: true" if f.proto3_optional?
+          args << "oneof: #{f.oneof.name.to_sym.inspect}" if f.oneof && explicit_oneof
+          args.concat(option_args(f.options))
+          out << ind(depth) << call("#{var}.field", args, depth) << "\n"
+        end
+
+        def path_for_nested(index, field_path)
+          field_path[0...-2] + [MESSAGE_NESTED, index]
+        end
+
+        def field_args(f)
+          args = [f.name.to_sym.inspect, f.number.to_s, f.declared_type.inspect]
+          args << Literal.string(".#{f.type_name}") if f.type_name
+          args << "label: #{f.declared_label.inspect}" unless f.declared_label == :optional
+          args << "default: #{Literal.string(f.raw_default)}" if f.raw_default
+          args << "json_name: #{Literal.string(f.json_name)}" if custom_json_name?(f)
+          args
+        end
+
+        def custom_json_name?(f)
+          f.json_name != Schema.json_name(f.name)
+        end
+
+        def extensions(out, depth, var, list, path)
+          i = 0
+          while i < list.size
+            extendee = list[i].extendee
+            out << ind(depth) << "#{var}.extend #{Literal.string(".#{extendee.full_name}")} do |e|\n"
+            while i < list.size && list[i].extendee.equal?(extendee)
+              f = list[i]
+              comment(out, depth + 1, path + [i])
+              args = field_args(f)
+              args << "proto3_optional: true" if f.proto3_optional?
+              args.concat(option_args(f.options))
+              out << ind(depth + 1) << call("e.field", args, depth + 1) << "\n"
+              i += 1
+            end
+            out << ind(depth) << "end\n"
+          end
+        end
+
+        def enum(out, depth, var, desc, path)
+          comment(out, depth, path)
+          args = [Literal.string(desc.name)]
+          args << "visibility: #{desc.visibility.inspect}" if desc.visibility
+          args.concat(option_args(desc.options))
+          out << ind(depth) << call("#{var}.enum", args, depth) << " do |e|\n"
+          desc.values.each_with_index do |v, i|
+            comment(out, depth + 1, path + [ENUM_VALUE, i])
+            args = [v.name.to_sym.inspect, v.number.to_s] + option_args(v.options)
+            out << ind(depth + 1) << call("e.value", args, depth + 1) << "\n"
+          end
+          reserved = desc.reserved_ranges.map { |r| range(r.begin, r.end) }
+          reserved.concat(desc.reserved_names.map { |n| Literal.string(n) })
+          out << ind(depth + 1) << call("e.reserved", reserved, depth + 1) << "\n" unless reserved.empty?
+          out << ind(depth) << "end\n"
+        end
+
+        def service(out, depth, var, desc, path)
+          comment(out, depth, path)
+          args = [Literal.string(desc.name)] + option_args(desc.options)
+          head = call("#{var}.service", args, depth)
+          if desc.rpcs.empty?
+            out << ind(depth) << head << "\n"
+            return
+          end
+          out << ind(depth) << head << " do |s|\n"
+          desc.rpcs.each_with_index do |m, i|
+            comment(out, depth + 1, path + [SERVICE_METHOD, i])
+            name = /\A[A-Za-z_][A-Za-z0-9_]*\z/.match?(m.name) ? m.name.to_sym.inspect : Literal.string(m.name)
+            args = [name, Literal.string(".#{m.input_type.full_name}"), Literal.string(".#{m.output_type.full_name}")]
+            args << "client_streaming: true" if m.client_streaming?
+            args << "server_streaming: true" if m.server_streaming?
+            args.concat(option_args(m.options))
+            out << ind(depth + 1) << call("s.rpc", args, depth + 1) << "\n"
+          end
+          out << ind(depth) << "end\n"
+        end
+
+        # Options as keywords, custom ones under `extensions:`; verbatim
+        # under `options:` when that cannot be said with keywords. Each is a
+        # `[prefix, value]` pair, rendered by {#call}.
+        def option_args(options)
+          return [] unless options.declared?
+          h = dsl_hash(options)
+          return [["options: ", h]] if h.empty? || h.key?(:options)
+          h.map { |key, value| ["#{key}: ", value] }
+        end
+
+        # An options message (or a message inside one) as the DSL spells it:
+        # fields by name, `extensions` by full name, `unknown_fields` as wire
+        # bytes; a field named like either of those two takes a String key. An
+        # extension the file reaches only through `import option` is written
+        # as unknown bytes, as protoc's own generator embeds it: the generated
+        # file does not load that import, so it may not be there to name the
+        # option.
+        def dsl_hash(value)
+          h = {}
+          value.fields.each { |k, v| h[RESERVED_KEYS.include?(k) ? k.to_s : k] = dsl_value(v) }
+          named, unnamed = value.extensions.partition { |f, _| reachable?(f) }
+          h[:extensions] = named.to_h { |f, v| [f.full_name, dsl_value(v)] } unless named.empty?
+          unknown = value.unknown_fields
+          unnamed.each { |f, v| unknown += Schema::MessageValue.new({}, {f => v}).encode(f.extendee) }
+          h[:unknown_fields] = unknown.b unless unknown.empty?
+          h
+        end
+
+        RESERVED_KEYS = %i[extensions unknown_fields].freeze
+
+        # Whether an extension is defined by the file or one it imports, so
+        # that loading the generated file can find it
+        def reachable?(extension)
+          @reachable ||= begin
+            names = {}
+            queue = [@file]
+            until queue.empty?
+              f = queue.shift
+              next if names[f.name]
+              names[f.name] = true
+              queue.concat(f.dependency_files)
+            end
+            names
+          end
+          @reachable.key?(extension.file.name)
+        end
+
+        def dsl_value(value)
+          case value
+          when Schema::MessageValue then dsl_hash(value)
+          when Array then value.map { |v| dsl_value(v) }
+          else value
+          end
+        end
+
+        # A call on one line when it fits. Otherwise the top-level call puts
+        # one argument per line inside parentheses, and calls in blocks keep
+        # their positional arguments on the first line and continue with one
+        # keyword per line. Arguments are source Strings or `[prefix, value]` pairs.
+        def call(receiver, args, depth)
+          flat = args.map { |a| a.is_a?(Array) ? a[0] + Literal.flat(a[1]) : a }
+          one = if args.empty?
+            receiver
+          elsif depth.positive?
+            "#{receiver} #{flat.join(", ")}"
+          else
+            "#{receiver}(#{flat.join(", ")})"
+          end
+          return one if depth * 2 + one.size <= Literal::WIDTH
+          inner = (depth + 1) * 2
+          render = lambda do |a|
+            a.is_a?(Array) ? a[0] + Literal.call(a[1], inner, inner + a[0].size) : a
+          end
+          if depth.zero?
+            return "#{receiver}(\n#{args.map { |a| " " * inner + render.call(a) }.join(",\n")}\n)"
+          end
+          positional, keywords = args.partition { |a| a.is_a?(String) && !KEYWORD.match?(a) }
+          return one if keywords.empty?
+          head = positional.empty? ? receiver : "#{receiver} #{positional.join(", ")}"
+          lines = keywords.map { |a| " " * inner + render.call(a) }
+          "#{head},\n#{lines.join(",\n")}"
+        end
+
+        KEYWORD = /\A[a-z_][a-z0-9_]*: /
+
+        def range(first, last)
+          (first == last) ? first.to_s : "#{first}..#{last}"
+        end
+
+        def comment(out, depth, path)
+          return unless @source
+          location = @source.location(path)
+          text = location&.leading_comments
+          return if text.nil? || text.empty?
+          lines = text.split("\n", -1)
+          lines.pop if lines.last == ""
+          lines.each do |line|
+            line = line.rstrip
+            out << ind(depth) << (line.empty? ? "#" : "##{line}") << "\n"
+          end
+        end
+
+        def ind(depth)
+          "  " * depth
+        end
+
+        def emit_constants(file)
+          names = Naming.module_path(file, @namespaces[file.name])
+          lines = []
+          file.messages.each { |m| message_constants(lines, "", m) }
+          file.enums.each { |e| lines << enum_constant("", e) }
+          file.services.each do |s|
+            lines << "#{Naming.rubify_constant(s.name)} = Protostar::DSL.service(#{Literal.string(s.full_name)})"
+          end
+          return if lines.empty?
+          @out << "\n"
+          names.each_with_index { |n, i| @out << ind(i) << "module #{n}\n" }
+          lines.each { |l| @out << ind(names.size) << l << "\n" }
+          names.size.downto(1) { |i| @out << ind(i - 1) << "end\n" }
+        end
+
+        def message_constants(lines, prefix, message)
+          return if message.map_entry?
+          const = prefix + Naming.rubify_constant(message.name)
+          lines << "#{const} = Protostar::DSL.message(#{Literal.string(message.full_name)})"
+          message.nested_messages.each { |m| message_constants(lines, "#{const}::", m) }
+          message.nested_enums.each { |e| lines << enum_constant("#{const}::", e) }
+        end
+
+        def enum_constant(prefix, enum)
+          "#{prefix}#{Naming.rubify_constant(enum.name)} = Protostar::DSL.enum(#{Literal.string(enum.full_name)})"
+        end
+      end
+    end
+  end
+end