protostar 0.1.0.beta1

data/lib/protostar/compiler/retention.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module Protostar
+  class Compiler
+    # protoc's StripSourceRetentionOptions (`compiler/retention.cc`): a copy
+    # of a file without the options declared `retention = RETENTION_SOURCE`,
+    # without options messages left empty by that, and without the source
+    # locations under anything removed. protoc does this when it writes a
+    # descriptor set or embeds descriptors in generated code; its pool keeps
+    # the options for validating later files.
+    #
+    # @api private
+    module Retention
+      # descriptor.proto field numbers of each descriptor's `options`
+      OPTIONS_FIELD = {
+        Schema::File => 8, Schema::Message => 7, Schema::Field => 8, Schema::Oneof => 2, Schema::Enum => 3,
+        Schema::EnumValue => 3, Schema::Service => 3, Schema::Method => 4, Schema::ExtensionRange => 3
+      }.freeze
+
+      # @param file [Schema::File]
+      # @param pool [Schema::Pool] holds the stripped copies of the file's imports
+      # @param options_type [#call] options message full name to its {Schema::Message}
+      # @return [Schema::File] the stripped copy, not added to the pool
+      def self.strip(file, pool, options_type)
+        stripped = []
+        kept = {}.compare_by_identity
+        source_paths(file).each do |descriptor, path|
+          options = descriptor.options
+          next unless options.declared?
+          path += [OPTIONS_FIELD.fetch(descriptor.class)]
+          value = strip_value(options, options_type.call(options.type_name), path, stripped)
+          if !options.empty? && value.empty?
+            stripped << path
+            next
+          end
+          kept[descriptor] = value
+        end
+        info = file.source_info
+        if info && !stripped.empty?
+          info = Schema::SourceInfo.new(info.locations.reject { |loc| stripped.any? { |s| loc.path[0, s.size] == s } })
+        end
+        Schema::Draft.copy(file, pool: pool, source_info: info) { |descriptor| kept[descriptor] }
+      end
+
+      def self.strip_value(value, type, path, stripped)
+        fields = {}
+        value.fields.each do |name, v|
+          field = type&.fields_by_name&.[](name.to_s)
+          result = field ? strip_field(field, v, path, stripped) : v
+          fields[name] = result unless result.equal?(DROP)
+        end
+        extensions = {}
+        value.extensions.each do |field, v|
+          result = strip_field(field, v, path, stripped)
+          extensions[field] = result unless result.equal?(DROP)
+        end
+        Schema::MessageValue.new(fields, extensions, value.unknown_fields)
+      end
+
+      DROP = Object.new.freeze
+
+      def self.strip_field(field, value, path, stripped)
+        here = path + [field.number]
+        if field.options.retention == :RETENTION_SOURCE
+          stripped << here
+          return DROP
+        end
+        return value unless field.type == :message || field.type == :group
+        if field.repeated?
+          value.each_with_index.map { |v, i| strip_value(v, field.message_type, here + [i], stripped) }
+        else
+          strip_value(value, field.message_type, here, stripped)
+        end
+      end
+
+      # Each descriptor's path in its FileDescriptorProto, as source locations spell it
+      def self.source_paths(file)
+        paths = {file => []}.compare_by_identity
+        visit = lambda do |m, path|
+          paths[m] = path
+          m.fields.each_with_index { |f, i| paths[f] = path + [2, i] }
+          m.nested_messages.each_with_index { |n, i| visit.call(n, path + [3, i]) }
+          m.nested_enums.each_with_index { |e, i| enum_paths(paths, e, path + [4, i]) }
+          m.extension_range_descriptors.each_with_index { |r, i| paths[r] = path + [5, i] }
+          m.extensions.each_with_index { |f, i| paths[f] = path + [6, i] }
+          m.oneofs.each_with_index { |o, i| paths[o] = path + [8, i] }
+        end
+        file.messages.each_with_index { |m, i| visit.call(m, [4, i]) }
+        file.enums.each_with_index { |e, i| enum_paths(paths, e, [5, i]) }
+        file.services.each_with_index do |s, i|
+          paths[s] = [6, i]
+          s.rpcs.each_with_index { |m, j| paths[m] = [6, i, 2, j] }
+        end
+        file.extensions.each_with_index { |f, i| paths[f] = [7, i] }
+        paths
+      end
+
+      def self.enum_paths(paths, enum, path)
+        paths[enum] = path
+        enum.values.each_with_index { |v, i| paths[v] = path + [2, i] }
+      end
+    end
+  end
+end

data/lib/protostar/compiler/run.rb

@@ -0,0 +1,184 @@
+# frozen_string_literal: true
+
+module Protostar
+  class Compiler
+    # One {Compiler#compile} call: loads the named files and their imports
+    # depth first, as protoc's importer does (a file is parsed, then its
+    # imports are built, then the file itself), and collects diagnostics in
+    # protoc's order.
+    #
+    # @api private
+    class Run
+      # @return [Array<Diagnostic>]
+      attr_reader :diagnostics
+
+      # @param compiler [Compiler]
+      # @param names [Array<String>] the files asked for
+      # @param source_info [Boolean]
+      # @param sources [Hash{String => String}] file contents to use instead of reading include paths
+      # @param protoc_cli [Boolean] behave like protoc's command line (see {Linker#initialize});
+      #   false behaves like a bare DescriptorPool, which loads no imports itself
+      # @param all_direct [Boolean] check every file loaded as if it had been named, imports
+      #   included (unused imports, deprecated features), as `protostar lint` wants
+      def initialize(compiler, names, source_info, sources: {}, protoc_cli: true, all_direct: false)
+        @compiler = compiler
+        @pool = compiler.pool
+        @names = names
+        @source_info = source_info
+        @sources = sources
+        @protoc_cli = protoc_cli
+        @all_direct = all_direct
+        @diagnostics = []
+        @pending = []
+        @done = {}
+        @failed = false
+        @order = []
+        @listed = {}
+        @linkers = {}
+      end
+
+      def failed?
+        @failed || @diagnostics.any?(&:error?)
+      end
+
+      # @return [Linker, nil] the linker of a file this run built
+      def linker(name)
+        @linkers[name]
+      end
+
+      # @return [Array<Schema::File>]
+      def call
+        direct = []
+        @names.each do |name|
+          file = load(name, direct: true)
+          if file.nil?
+            @failed = true
+            break
+          end
+          direct << @linkers[name] if @linkers[name]
+        end
+        Validator.cli_checks(direct, self) if @protoc_cli && !failed?
+        @order
+      end
+
+      # Records a diagnostic with a 0-based position (nil for none)
+      # @param spans [Array<Diagnostics::Span>] where the renderer underlines, when known
+      def report(file, position, message, severity = :error, spans: [])
+        line, column = position
+        line = nil if line && line < 0
+        path = @compiler.disk_path(file) if file && !@sources.key?(file) && @protoc_cli
+        @diagnostics << Diagnostic.new(file, line && line + 1, line && column + 1, message, severity, path: path,
+          spans: line ? spans : [])
+      end
+
+      private
+
+      # The problem's own span, as long as the source location that starts
+      # where it is reported, then the places it relates to
+      def spans(ast, problem, position)
+        return [] unless position
+        out = [span(ast, position, nil, true)]
+        problem.related.each do |node, kind, label|
+          at = node.position(kind)
+          out << span(ast, at, label, false) if at
+        end
+        out
+      end
+
+      def span(ast, position, label, primary)
+        line, column = position
+        end_line, end_column = SourceInfo.extent(ast.locations, line, column)
+        Diagnostics::Span.new(line + 1, column + 1, end_line && end_line + 1, end_column && end_column + 1,
+          label: label, primary: primary)
+      end
+
+      # An extension number another file already uses is a warning the linker
+      # reports as a diagnostic, so the pool is asked not to print it too
+      def add_to_pool(name, file)
+        @pool.add(file, warn: false)
+      rescue Schema::Error => e
+        report(name, nil, e.message)
+        nil
+      end
+
+      def list(file)
+        return if @listed[file.name]
+        file.dependency_files.each { |dep| list(dep) }
+        @listed[file.name] = true
+        @order << file
+      end
+
+      def read(name)
+        return @sources[name] if @sources.key?(name)
+        return nil unless @protoc_cli
+        path = @compiler.resolve(name)
+        path && ::File.binread(path)
+      end
+
+      # @return [Schema::File, nil]
+      def load(name, direct: false)
+        return @done[name] if @done.key?(name)
+        if direct && !@protoc_cli && @pool.include?(name)
+          ast = Parser.new(read(name).to_s).parse(name)
+          report(name, ast.position(:other), "A file with this name is already in the pool.")
+          return @done[name] = nil
+        end
+        if (existing = @pool.file(name))
+          list(existing)
+          return @done[name] = existing
+        end
+
+        if (from = @pending.index { |n, _| n == name })
+          chain = @pending[from..].map(&:first) + [name]
+          ast = @pending[from][1]
+          import = (from < @pending.size - 1) ? @pending[from + 1][0] : name
+          report(name, ast.import_positions[import], "File recursively imports itself: #{chain.join(" -> ")}")
+          return nil
+        end
+
+        source = read(name)
+        unless source
+          report(name, nil, "File not found.") if @protoc_cli
+          return @done[name] = nil
+        end
+
+        parser = Parser.new(source)
+        ast = parser.parse(name)
+        parser.errors.each { |line, column, message| report(name, [line, column], message) }
+        parser.warnings.each { |line, column, message| report(name, [line, column], message, :warning) }
+        return @done[name] = nil unless parser.errors.empty?
+
+        @pending << [name, ast]
+        missing = []
+        (ast.dependency + ast.option_dependency).each do |dep|
+          missing << dep if load(dep).nil?
+        end
+        @pending.pop
+        # protoc's BuildFile gives up quietly on a file that imports itself,
+        # once the recursion has been reported
+        return @done[name] = nil if ast.dependency.include?(name)
+
+        linker = Linker.new(ast, pool: @pool, missing: missing, direct: (direct || @all_direct) && @protoc_cli,
+          options_fallback: ->(type) { @compiler.builtin_options_type(type) }, source_info: @source_info,
+          protoc_cli: @protoc_cli)
+        linker.link
+        Validator.new(linker).validate
+        linker.problems.each do |problem|
+          position = if problem.location == :import
+            ast.import_positions[problem.import]
+          else
+            problem.node&.position(problem.location)
+          end
+          report(name, position, problem.message, problem.severity, spans: spans(ast, problem, position))
+        end
+        file = linker.errors? ? nil : linker.file
+        file = add_to_pool(name, file) if file
+        if file
+          @linkers[name] = linker
+          list(file)
+        end
+        @done[name] = file
+      end
+    end
+  end
+end

data/lib/protostar/compiler/source_info.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+module Protostar
+  class Compiler
+    # Builds `SourceCodeInfo` the way protoc does: the parser opens a
+    # {Recorder} for every element as it starts reading it, so locations come
+    # out in pre-order with protoc's paths and spans, and comments are
+    # attached at the end of each declaration. After option interpretation
+    # {.rewrite_option_paths} moves option locations from
+    # `uninterpreted_option` paths to the paths of the options they set.
+    #
+    # @api private
+    module SourceInfo
+      # A `SourceCodeInfo.Location` under construction
+      class Location
+        attr_accessor :path, :span, :leading_comments, :trailing_comments, :leading_detached_comments
+
+        def initialize(path, span)
+          @path = path
+          @span = span
+          @leading_comments = nil
+          @trailing_comments = nil
+          @leading_detached_comments = []
+        end
+
+        def dup_with_path(path)
+          copy = Location.new(path, @span.dup)
+          copy.leading_comments = @leading_comments
+          copy.trailing_comments = @trailing_comments
+          copy.leading_detached_comments = @leading_detached_comments.dup
+          copy
+        end
+
+        # @return [Schema::SourceInfo::Location] the finished location, comments as UTF-8
+        def to_schema
+          Schema::SourceInfo::Location.new(path: @path, span: @span, leading_comments: utf8(@leading_comments),
+            trailing_comments: utf8(@trailing_comments),
+            leading_detached_comments: @leading_detached_comments.map { |c| utf8(c) })
+        end
+
+        private
+
+        def utf8(text)
+          text&.dup&.force_encoding(Encoding::UTF_8)
+        end
+      end
+
+      # protoc's Parser::LocationRecorder. It starts at the current token when
+      # created and, unless ended explicitly, ends at the previous token when
+      # {#finish}ed.
+      class Recorder
+        # @return [Location]
+        attr_reader :location
+        # @return [Array<Location>] where the location was added
+        attr_reader :target
+
+        # @param lexer [Lexer]
+        # @param path [Array<Integer>] the parent's path plus this element's components
+        # @param target [Array<Location>]
+        def initialize(lexer, path, target)
+          @lexer = lexer
+          @target = target
+          token = lexer.current
+          @location = Location.new(path, [token.line, token.column])
+          target << @location
+        end
+
+        # @return [Recorder] a child location, its path extended by `components`
+        def child(*components, target: @target)
+          Recorder.new(@lexer, @location.path + components, target)
+        end
+
+        def add_path(component)
+          @location.path << component
+        end
+
+        def path_size
+          @location.path.size
+        end
+
+        def start_at(token)
+          @location.span[0] = token.line
+          @location.span[1] = token.column
+        end
+
+        def start_at_recorder(other)
+          @location.span[0] = other.location.span[0]
+          @location.span[1] = other.location.span[1]
+        end
+
+        def end_at(token)
+          @location.span << token.line if token.line != @location.span[0]
+          @location.span << token.end_column
+        end
+
+        # The LocationRecorder destructor
+        def finish
+          end_at(@lexer.previous) if @location.span.size <= 2
+        end
+
+        # Records where this element starts, for diagnostics against `node`
+        def record_legacy(node, kind)
+          node.positions[kind] = [@location.span[0], @location.span[1]]
+        end
+
+        # @return [Array(Integer, Integer)]
+        def start
+          [@location.span[0], @location.span[1]]
+        end
+
+        def attach_comments(leading, trailing, detached)
+          @location.leading_comments = leading.dup unless leading.empty?
+          @location.trailing_comments = trailing.dup unless trailing.empty?
+          detached.each { |c| @location.leading_detached_comments << c }
+          detached.clear
+        end
+      end
+
+      # Where the shortest location starting at a position ends: the extent of
+      # the element whose name, number or type is reported there
+      # @param locations [Array<Location>]
+      # @return [Array(Integer, Integer), nil] 0-based line and column just past the end
+      def self.extent(locations, line, column)
+        best = nil
+        locations.each do |loc|
+          span = loc.span
+          next unless span[0] == line && span[1] == column && span.size >= 3
+          finish = (span.size == 4) ? [span[2], span[3]] : [span[0], span[2]]
+          best = finish if best.nil? || (finish <=> best) < 0
+        end
+        best
+      end
+
+      # OptionInterpreter::UpdateSourceCodeInfo: the location of each
+      # interpreted option takes the path of the option it set, and the
+      # locations nested under it (name parts, values) are dropped
+      # @param locations [Array<Location>]
+      # @param interpreted [Hash{Array<Integer> => Array<Integer>}] uninterpreted_option path to option path
+      # @return [Array<Location>]
+      def self.rewrite_option_paths(locations, interpreted)
+        return locations if interpreted.empty?
+        out = []
+        match_src = nil
+        locations.each do |loc|
+          if match_src
+            next if loc.path.size >= match_src.size && loc.path[0, match_src.size] == match_src
+            match_src = nil
+          end
+          dest = interpreted[loc.path]
+          if dest.nil?
+            out << loc
+            next
+          end
+          match_src = loc.path
+          out << loc.dup_with_path(dest.dup)
+        end
+        out
+      end
+    end
+  end
+end

data/lib/protostar/compiler/symbol_table.rb

@@ -0,0 +1,223 @@
+# frozen_string_literal: true
+
+module Protostar
+  class Compiler
+    # The names visible while one file is linked: its own symbols, drafted
+    # as the linker meets them, and everything in the pool. Lookups follow
+    # protoc's DescriptorBuilder (FindSymbol and LookupSymbolNoPlaceholder):
+    # the same first-component scoping rule as {Schema::Resolver}, with a
+    # symbol in another file visible only through a direct import or a chain
+    # of `import public`, and with the failure explanations protoc gives.
+    #
+    # @api private
+    class SymbolTable
+      # A named thing
+      class Symbol
+        # @return [Symbol] :message, :enum, :enum_value, :field, :oneof, :service, :method or :package
+        attr_reader :kind
+        # @return [String]
+        attr_reader :full_name
+        # @return [String] the file defining it
+        attr_reader :file_name
+        # @return [Schema::Descriptor, nil] the descriptor (drafted, for the file being
+        #   linked); nil for a package
+        attr_reader :object
+
+        def initialize(kind:, full_name:, file_name:, object:, local: false)
+          @kind = kind
+          @full_name = full_name
+          @file_name = file_name
+          @object = object
+          @local = local
+          freeze
+        end
+
+        def type?
+          @kind == :message || @kind == :enum
+        end
+
+        def aggregate?
+          @kind == :message || @kind == :package || @kind == :enum || @kind == :service
+        end
+
+        # @return [Boolean] defined by the file being linked
+        def local?
+          @local
+        end
+      end
+
+      SCHEMA_KINDS = {
+        Schema::Message => :message, Schema::Enum => :enum, Schema::EnumValue => :enum_value,
+        Schema::Field => :field, Schema::Oneof => :oneof, Schema::Service => :service, Schema::Method => :method
+      }.freeze
+
+      # @return [String, nil] set when the last failed lookup found the name in a file that is not imported
+      attr_reader :undeclared_file, :undeclared_name
+      # @return [String, nil] set when the last failed lookup resolved a compound name to a missing symbol
+      attr_reader :unresolved_name
+      # @return [Array<String>] direct imports no lookup has used yet
+      attr_reader :unused
+
+      # @param file_name [String] the file being linked
+      # @param package [String]
+      # @param pool [Schema::Pool]
+      # @param dependencies [Array<Schema::File, nil>] direct imports, nil where missing
+      # @param option_dependencies [Array<Schema::File, nil>] `import option` files
+      # @param unused [Array<Schema::File>] imports to report when never used
+      def initialize(file_name, package, pool, dependencies, option_dependencies, unused)
+        @file_name = file_name
+        @package = package
+        @pool = pool
+        @local = {}
+        @visible = {}
+        dependencies.each { |dep| record_public(dep, @visible) }
+        @option_visible = {}
+        option_dependencies.each { |dep| record_public(dep, @option_visible) }
+        @unused = unused.map(&:name)
+        @packages = {}
+        pool.files.each { |f| add_package_names(f.package, f.name) }
+        @local_packages = []
+      end
+
+      # @return [Hash{String => Symbol}] the file's own symbols
+      attr_reader :local
+
+      # Registers a symbol of the file being linked
+      # @param descriptor [Schema::Descriptor] as drafted
+      # @return [Symbol, nil] the existing symbol when the name is taken
+      def add(kind, full_name, descriptor)
+        existing = find_any(full_name)
+        return existing if existing
+        @local[full_name] = Symbol.new(kind: kind, full_name: full_name, file_name: @file_name, object: descriptor,
+          local: true)
+        nil
+      end
+
+      # Registers the file's package and its parents, as AddPackage does
+      # @return [Symbol, nil] a non-package symbol already using one of the names
+      def add_package(package)
+        existing = find_any(package)
+        if existing.nil?
+          @packages[package] ||= @file_name
+          @local_packages << package
+          cut = package.rindex(".")
+          return add_package(package[0, cut]) if cut
+        elsif existing.kind != :package
+          return existing
+        end
+        nil
+      end
+
+      # Any symbol by full name, ignoring visibility
+      # @return [Symbol, nil]
+      def find_any(full_name)
+        if (sym = @local[full_name])
+          return sym
+        end
+        if (descriptor = @pool.find_symbol(full_name))
+          return Symbol.new(kind: SCHEMA_KINDS.fetch(descriptor.class), full_name: full_name,
+            file_name: descriptor.file.name, object: descriptor)
+        end
+        if (owner = @packages[full_name])
+          return Symbol.new(kind: :package, full_name: full_name, file_name: owner, object: nil)
+        end
+        nil
+      end
+
+      # FindSymbolNotEnforcingDeps: any symbol, marking its file's import as used
+      def find_not_enforcing_deps(full_name)
+        sym = find_any(full_name)
+        if sym && sym.kind != :package && (@visible.key?(sym.file_name) || @option_visible.key?(sym.file_name))
+          @unused.delete(sym.file_name)
+        end
+        sym
+      end
+
+      # FindSymbol: a symbol visible from the file being linked, or nil (and
+      # {#undeclared_file} set) when it lives in a file that is not imported
+      def find(full_name)
+        sym = find_not_enforcing_deps(full_name)
+        return nil if sym.nil?
+        return sym if sym.file_name == @file_name || @visible.key?(sym.file_name)
+        return sym if @option_visible.key?(sym.file_name) && sym.kind == :field
+        if sym.kind == :package
+          return sym if in_package?(@package, full_name)
+          return sym if @visible.each_value.any? { |f| in_package?(f.package, full_name) }
+          return sym if @option_visible.each_value.any? { |f| in_package?(f.package, full_name) }
+        end
+        @undeclared_file = sym.file_name
+        @undeclared_name = full_name
+        nil
+      end
+
+      # LookupSymbolNoPlaceholder
+      # @param name [String] as written, maybe with a leading dot
+      # @param relative_to [String] full name of the element doing the lookup
+      # @param types_only [Boolean] skip simple names that are not types
+      # @return [Symbol, nil]
+      def lookup(name, relative_to, types_only: false)
+        @undeclared_file = nil
+        @undeclared_name = nil
+        @unresolved_name = nil
+        return find(name[1..]) if name.start_with?(".")
+
+        dot = name.index(".")
+        first = dot ? name[0, dot] : name
+        scope = relative_to.dup
+        loop do
+          cut = scope.rindex(".")
+          return find(name) if cut.nil?
+          scope = scope[0, cut]
+          candidate = "#{scope}.#{first}"
+          sym = find(candidate)
+          next if sym.nil?
+          if dot
+            next unless sym.aggregate?
+            full = "#{scope}.#{name}"
+            found = find(full)
+            @unresolved_name = full if found.nil?
+            return found
+          end
+          return sym unless types_only && !sym.type?
+        end
+      end
+
+      # AddNotDefinedError's messages for the last failed {#lookup}
+      # @return [Array<String>]
+      def not_defined_messages(name)
+        if @undeclared_file.nil? && @unresolved_name.nil?
+          return ["\"#{name}\" is not defined."]
+        end
+        messages = []
+        if @undeclared_file
+          messages << "\"#{@undeclared_name}\" seems to be defined in \"#{@undeclared_file}\", which is not " \
+            "imported by \"#{@file_name}\".  To use it here, please add the necessary import."
+        end
+        if @unresolved_name
+          messages << "\"#{name}\" is resolved to \"#{@unresolved_name}\", which is not defined. The innermost " \
+            "scope is searched first in name resolution. Consider using a leading '.'(i.e., \".#{name}\") to start " \
+            "from the outermost scope."
+        end
+        messages
+      end
+
+      private
+
+      def record_public(file, into)
+        return if file.nil? || into.key?(file.name)
+        into[file.name] = file
+        file.public_dependencies.each { |i| record_public(file.dependency_files[i], into) }
+      end
+
+      def add_package_names(package, file_name)
+        return if package.empty?
+        parts = package.split(".")
+        parts.size.downto(1) { |n| @packages[parts.first(n).join(".")] ||= file_name }
+      end
+
+      def in_package?(package, name)
+        package == name || (package.start_with?(name) && package.getbyte(name.bytesize) == 46) # "."
+      end
+    end
+  end
+end