rubycc 1.0.0

data/lib/rubycc/link/partial_linker.rb

@@ -0,0 +1,546 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Rubycc
+  module Link
+    # The static-link core, an `ld -r` equivalent: it merges an ordered list of
+    # ET_REL objects (and archive members pulled in lazily) into a single ET_REL
+    # object. Relocations are *retargeted*, not applied — an `ld -r` output keeps
+    # its relocations so a later final link can resolve them; the byte-patching
+    # apply engine belongs to the executable/.so writers, not here.
+    #
+    # The pipeline is two phases over the reader structures directly (no linker
+    # IR, per the roadmap): a selection phase walks the inputs left to right to
+    # decide *which* objects take part — a `.o` joins unconditionally, an archive
+    # contributes only members that resolve a still-undefined symbol (iterated to
+    # a fixpoint within that archive; classic single-pass, so an archive already
+    # passed is never revisited) — and a merge phase concatenates same-named
+    # sections, resolves the global symbol table, and rewrites every relocation
+    # onto the merged sections and symbol table before handing the result to
+    # RelocatableWriter.
+    #
+    # The single subtle point is a relocation against a *section symbol*: its
+    # target section's bytes move to a new offset in the merged section, so the
+    # addend, which is measured from that section's start, must gain the piece's
+    # placement offset. Named-symbol references need no addend fix-up because the
+    # symbol's own value already carries the shift.
+    #
+    # Unresolved symbols are legal in an `ld -r` output and stay UND. COMMON
+    # symbols are diagnosed rather than allocated (out of scope; neither rubycc
+    # nor gcc's -fno-common default emits them). Output is deterministic (N4):
+    # sections appear in first-seen order, symbols in a fixed local-then-global
+    # order, so identical inputs yield byte-identical output.
+    class PartialLinker
+      include ObjFile
+
+      ELFMAG   = "\x7FELF".b
+      AR_MAGIC = "!<arch>\n".b
+
+      # Section header types the merge treats specially. Everything else — any
+      # named PROGBITS/NOBITS/NOTE/*_ARRAY the compiler emits or gcc adds — is
+      # carried through and concatenated by name.
+      SHT_NULL         = 0
+      SHT_SYMTAB       = 2
+      SHT_STRTAB       = 3
+      SHT_RELA         = 4
+      SHT_REL          = 9
+      SHT_NOBITS       = 8
+      SHT_GROUP        = 17
+      SHT_SYMTAB_SHNDX = 18
+      # Section kinds that describe the object's own linking metadata (symbol and
+      # string tables, relocation tables, section groups) — regenerated by the
+      # writer, so never carried from the inputs.
+      SKIPPED_TYPES = [SHT_NULL, SHT_SYMTAB, SHT_STRTAB, SHT_RELA, SHT_REL,
+                       SHT_GROUP, SHT_SYMTAB_SHNDX].freeze
+
+      SHN_UNDEF = 0
+      SHN_ABS   = 0xFFF1
+
+      class << self
+        # Links `inputs` (an ordered array; each element a filesystem path, or the
+        # raw bytes of an ET_REL object or an ar archive) and returns the merged
+        # ET_REL object as an ASCII-8BIT String.
+        def link(inputs)
+          new(inputs).link
+        end
+
+        # Convenience: link and write the merged object to `path`.
+        def link_to(inputs, path)
+          File.binwrite(path, link(inputs))
+        end
+      end
+
+      def initialize(inputs)
+        @inputs = inputs.each_with_index.map { |raw, i| load_input(raw, i) }
+      end
+
+      def link
+        included = select_inputs
+        Merger.new(included).run
+      end
+
+      private
+
+      # A classified input: :object or :archive, the reader, and a label used in
+      # diagnostics (the path, or a synthetic name for raw bytes).
+      Input = Struct.new(:kind, :label, :reader, keyword_init: true)
+
+      # Classifies one raw input. A String that already opens with an ELF or ar
+      # magic is taken as in-memory bytes; anything else is a filesystem path to
+      # read. The resulting bytes are then dispatched to the matching reader.
+      def load_input(raw, index)
+        if raw.is_a?(String) && (raw.b.start_with?(ELFMAG) || raw.b.start_with?(AR_MAGIC))
+          bytes = raw.b
+          label = "input##{index}"
+        else
+          label = raw.to_s
+          bytes = File.binread(label)
+        end
+
+        if bytes.start_with?(AR_MAGIC)
+          Input.new(kind: :archive, label: label, reader: ArReader.read(bytes))
+        else
+          Input.new(kind: :object, label: label, reader: ELFReader.read(bytes))
+        end
+      end
+
+      # Selection phase: walk the inputs left to right, returning the ordered list
+      # of participating objects ([{reader:, label:}]). Object inputs join
+      # unconditionally; archives contribute members lazily.
+      def select_inputs
+        @included = []
+        @defined = Set.new    # global/weak names already defined by an included object
+        @undefined = Set.new  # names referenced but not yet defined
+        # When non-nil, #include_object appends every name it newly adds to
+        # @undefined here; the archive walk uses it to widen its candidate set.
+        @new_undefined = nil
+
+        @inputs.each do |input|
+          case input.kind
+          when :object  then include_object(input.reader, input.label)
+          when :archive then pull_from_archive(input)
+          end
+        end
+        @included
+      end
+
+      # Adds an object to the participating set and folds its global/weak symbols
+      # into the running defined/undefined name sets, so a later archive pass
+      # knows what still needs resolving.
+      def include_object(reader, label)
+        @included << { reader: reader, label: label }
+        reader.symbols.each do |sym|
+          next unless sym.bind == :global || sym.bind == :weak
+          next if sym.name.to_s.empty?
+
+          if sym.defined?
+            @defined << sym.name
+            @undefined.delete(sym.name)
+          elsif sym.undefined? && !@defined.include?(sym.name)
+            @new_undefined << sym.name if @new_undefined && !@undefined.include?(sym.name)
+            @undefined << sym.name
+          end
+        end
+      end
+
+      # Pulls members from one archive to a fixpoint: repeatedly take any not-yet
+      # included member whose exports satisfy a currently-undefined symbol, since
+      # loading a member can surface new undefined references that pull further
+      # members from this SAME archive. Members are scanned in archive order for a
+      # deterministic result. Once this call returns the archive is done — a later
+      # input cannot reach back into it (single-pass semantics).
+      #
+      # The naive form of that fixpoint rescans EVERY member after each pull, so
+      # an archive whose members are ordered against their dependency direction
+      # takes one member per pass: O(M^2) member visits, which a hostile archive
+      # controls outright by choosing the member order. Instead an exporter index
+      # (symbol name -> the members defining it) narrows each pass to the members
+      # that could possibly match: a member can only be pulled through a name
+      # that is in @undefined, and every such name either was there when the
+      # archive was reached or was added by a member pulled from it, so seeding
+      # the candidate set from the former and widening it on the latter yields a
+      # SUPERSET of the pullable members — nothing is ever missed.
+      #
+      # The pull ORDER must not change, and that is the delicate part: it fixes
+      # the order of the merged sections and of the output symbol table, so N4
+      # (identical inputs produce byte-identical output) rests on it. Two rules
+      # preserve it exactly. Candidates are visited in ascending member index, so
+      # a pass sees them in archive order; and a candidate that appears mid-pass
+      # is spliced into the remaining queue when it sits after the current
+      # position, because the all-members scan would have reached it later in the
+      # same pass. Members that are not candidates are exactly the ones the
+      # all-members scan visits and rejects, so skipping them changes nothing.
+      def pull_from_archive(input)
+        members = input.reader.members
+        exporters = exporter_index(members)
+        taken = Set.new
+        candidates = Set.new
+        @undefined.each { |name| widen(candidates, exporters[name], taken, members) }
+
+        loop do
+          pulled = pull_pass(input, members, exporters, taken, candidates)
+          break unless pulled
+        end
+      end
+
+      # One archive-order pass over the current candidates. Returns whether any
+      # member was pulled, which is the fixpoint's termination test.
+      def pull_pass(input, members, exporters, taken, candidates)
+        pulled = false
+        queue = candidates.to_a.sort
+        qi = 0
+        while qi < queue.length
+          index = queue[qi]
+          qi += 1
+          member = members[index]
+          next if member.special? || taken.include?(member)
+          next unless member_exports(member).any? { |name| @undefined.include?(name) }
+
+          @new_undefined = []
+          include_object(ELFReader.read(member.data), "#{input.label}(#{member.name})")
+          added = @new_undefined
+          @new_undefined = nil
+          taken << member
+          candidates.delete(index)
+          pulled = true
+          added.each do |name|
+            widen(candidates, exporters[name], taken, members) do |cand|
+              # Still ahead of us: the all-members scan would reach it in this
+              # very pass, so the queue has to as well.
+              insert_pending(queue, qi, cand) if cand > index
+            end
+          end
+        end
+        pulled
+      end
+
+      # Adds the members exporting one name to the candidate set, yielding each
+      # index that was not a candidate already (so a caller mid-pass can queue
+      # it). Members already pulled are never re-added.
+      def widen(candidates, indexes, taken, members)
+        return if indexes.nil?
+
+        indexes.each do |index|
+          next if candidates.include?(index) || taken.include?(members[index])
+
+          candidates << index
+          yield index if block_given?
+        end
+      end
+
+      # Splices `value` into the not-yet-visited tail queue[from..], which is
+      # sorted ascending, keeping it sorted.
+      def insert_pending(queue, from, value)
+        lo = from
+        hi = queue.length
+        while lo < hi
+          mid = (lo + hi) / 2
+          if queue[mid] < value
+            lo = mid + 1
+          else
+            hi = mid
+          end
+        end
+        queue.insert(lo, value)
+      end
+
+      # symbol name => ascending array of the indexes of the members exporting
+      # it. Built once per archive off the same memoized #member_exports the
+      # selection uses, so it costs one parse per member — the same parse the
+      # first fixpoint pass would have done anyway.
+      def exporter_index(members)
+        index = {}
+        members.each_with_index do |member, i|
+          next if member.special?
+
+          member_exports(member).each { |name| (index[name] ||= []) << i }
+        end
+        index
+      end
+
+      # The defined global/weak symbol names a member exports, memoized per
+      # member. A member ELFReader cannot parse contributes none.
+      def member_exports(member)
+        @member_exports ||= {}
+        @member_exports[member.object_id] ||= begin
+          reader = ELFReader.read(member.data)
+          reader.symbols.select do |sym|
+            (sym.bind == :global || sym.bind == :weak) && sym.defined? && !sym.name.to_s.empty?
+          end.map(&:name)
+        rescue ObjFile::ELFFormatError
+          []
+        end
+      end
+
+      # Merges the selected objects into one ET_REL image. Kept as an inner class
+      # so the phase-2 state (placement maps, the output symbol table, the
+      # per-object symbol remaps) stays isolated from the selection driver.
+      class Merger
+        include ObjFile
+
+        SHT_NOBITS = PartialLinker::SHT_NOBITS
+        SHN_ABS    = PartialLinker::SHN_ABS
+
+        def initialize(included)
+          @objects = included # [{reader:, label:}]
+          # The merged object keeps the inputs' architecture: the merge is
+          # machine-independent (section concatenation and numeric-type relocation
+          # retargeting), but the output header must carry the right e_machine so
+          # the final linker that reads it back can pick the target's relocation
+          # and crt logic. Empty inputs default to x86_64 through RelocatableWriter.
+          machine = included.first&.dig(:reader)&.machine
+          @writer = machine ? RelocatableWriter.new(machine: machine) : RelocatableWriter.new
+        end
+
+        def run
+          merge_sections
+          resolve_symbols
+          retarget_relocations
+          @writer.to_binary
+        end
+
+        private
+
+        # --- section merging -------------------------------------------------
+
+        # Concatenates same-named content sections in object order. For each
+        # object/section it records the piece's placement offset within the merged
+        # section — the shift every symbol value and relocation offset in that
+        # piece is adjusted by. Output alignment/entsize are the max over inputs;
+        # a NOBITS section (.bss) merges by size only.
+        def merge_sections
+          @order = []                                 # output section names, first-seen order
+          @meta = {}                                  # name => merge metadata
+          @data = {}                                  # name => merged bytes (PROGBITS)
+          @size = Hash.new(0)                         # name => running size
+          @placement = Array.new(@objects.size) { {} } # [oi][in_section.index] => offset
+
+          @objects.each_with_index do |obj, oi|
+            obj[:reader].sections.each do |sec|
+              next unless carried?(sec)
+
+              place_piece(oi, sec)
+            end
+          end
+
+          @out_section = {}
+          @order.each do |name|
+            m = @meta[name]
+            @out_section[name] =
+              if m[:type] == SHT_NOBITS
+                @writer.add_section(name: name, type: m[:type], flags: m[:flags],
+                                    addralign: m[:addralign], entsize: m[:entsize], size: @size[name])
+              else
+                @writer.add_section(name: name, type: m[:type], flags: m[:flags],
+                                    addralign: m[:addralign], entsize: m[:entsize], data: @data[name])
+              end
+          end
+        end
+
+        # A section carried into the output: any named section that is not one of
+        # the regenerated linking-metadata kinds.
+        def carried?(sec)
+          !PartialLinker::SKIPPED_TYPES.include?(sec.type) && !sec.name.to_s.empty?
+        end
+
+        # Places one input section piece into its merged section, aligning the
+        # running length up to the piece's own alignment first and recording where
+        # it landed. The output metadata accumulates the max alignment/entsize and
+        # the union of flags across pieces.
+        def place_piece(oi, sec)
+          name = sec.name
+          unless @meta.key?(name)
+            @order << name
+            @meta[name] = { type: sec.type, flags: sec.flags,
+                            addralign: [sec.addralign, 1].max, entsize: sec.entsize }
+          end
+          m = @meta[name]
+          m[:addralign] = [m[:addralign], sec.addralign, 1].max
+          m[:entsize] = [m[:entsize], sec.entsize].max
+          m[:flags] |= sec.flags
+
+          offset = align(@size[name], [sec.addralign, 1].max)
+          @placement[oi][sec.index] = offset
+
+          if sec.type == SHT_NOBITS
+            @size[name] = offset + sec.size
+          else
+            buf = (@data[name] ||= +"".b)
+            buf << ("\0" * (offset - buf.bytesize)).b if buf.bytesize < offset
+            buf << sec.data
+            @size[name] = buf.bytesize
+          end
+        end
+
+        # --- symbol resolution ----------------------------------------------
+
+        # Builds the output symbol table in a fixed order — section symbols, then
+        # each object's local symbols, then the resolved globals — and records,
+        # per object, the map from an input symbol to its output symbol so
+        # relocations can be re-pointed.
+        def resolve_symbols
+          @remap = Array.new(@objects.size) { {} }        # [oi][in_symbol.index] => out Symbol
+          @section_symbol = {}                            # output section name => out Symbol
+          @order.each do |name|
+            @section_symbol[name] =
+              @writer.add_symbol(name: nil, bind: :local, type: :section, section: @out_section[name])
+          end
+
+          add_local_symbols
+          resolve_globals
+        end
+
+        # Passes every input local symbol through per object, shifting a defined
+        # local's value by its section's placement offset and re-pointing it at the
+        # merged output section. Section symbols are not passed through (the output
+        # has one synthesized per merged section); undefined or dropped-section
+        # locals are skipped.
+        def add_local_symbols
+          @objects.each_with_index do |obj, oi|
+            obj[:reader].symbols.each do |sym|
+              next if sym.index.zero?               # the reserved null symbol
+              next unless sym.bind == :local
+              next if sym.type == :section          # synthesized, not passed through
+
+              @remap[oi][sym.index] = local_output_symbol(oi, sym)
+            end
+          end
+        end
+
+        # The output symbol for one input local, or nil when it cannot be carried
+        # (an undefined local, or one defined in a section that was not merged).
+        def local_output_symbol(oi, sym)
+          if sym.type == :file
+            @writer.add_symbol(name: sym.name, bind: :local, type: :file, shndx: SHN_ABS)
+          elsif sym.absolute?
+            @writer.add_symbol(name: sym.name, bind: :local, type: sym.type, shndx: SHN_ABS,
+                               value: sym.value, size: sym.size)
+          elsif sym.defined? && sym.section && @out_section.key?(sym.section.name)
+            out = @out_section[sym.section.name]
+            @writer.add_symbol(name: sym.name, bind: :local, type: sym.type,
+                               visibility: sym.visibility, section: out,
+                               value: @placement[oi][sym.section.index] + sym.value, size: sym.size)
+          end
+        end
+
+        # Resolves the global/weak symbols to one output entry per name, applying
+        # the definition-precedence rules, then re-points every object's reference
+        # to that single entry.
+        def resolve_globals
+          @globals = {} # name => resolution state (insertion order = first-seen)
+          @objects.each_with_index do |obj, oi|
+            obj[:reader].symbols.each do |sym|
+              next unless sym.bind == :global || sym.bind == :weak
+              next if sym.name.to_s.empty?
+
+              record_global(oi, sym, obj[:label])
+            end
+          end
+
+          out = emit_globals
+          # A relocation may reference a global that this object leaves undefined;
+          # it still points at the one merged entry, so map every occurrence.
+          @objects.each_with_index do |obj, oi|
+            obj[:reader].symbols.each do |sym|
+              next unless sym.bind == :global || sym.bind == :weak
+              next if sym.name.to_s.empty?
+
+              @remap[oi][sym.index] = out[sym.name]
+            end
+          end
+        end
+
+        # Folds one global/weak symbol into the resolution state, enforcing the
+        # precedence rules: a strong definition overrides a weak one, a second
+        # strong definition is a hard error, weak-vs-weak keeps the first, and a
+        # definition always beats an undefined reference. COMMON is unsupported.
+        def record_global(oi, sym, label)
+          if sym.common?
+            raise LinkError, "COMMON symbol '#{sym.name}' is not supported (compile with -fno-common)"
+          end
+
+          state = @globals[sym.name]
+          weak = sym.bind == :weak
+
+          if sym.defined?
+            if state.nil? || !state[:defined]
+              @globals[sym.name] = { defined: true, oi: oi, sym: sym, weak: weak, label: label }
+            elsif state[:weak] && !weak
+              @globals[sym.name] = { defined: true, oi: oi, sym: sym, weak: false, label: label }
+            elsif !state[:weak] && !weak
+              raise LinkError,
+                    "multiple definition of '#{sym.name}' (defined in #{state[:label]} and #{label})"
+            end
+            # strong-over-weak keep, weak-over-strong ignore, weak-over-weak first wins
+          elsif state.nil?
+            @globals[sym.name] = { defined: false, weak: weak }
+          elsif !state[:defined] && !weak
+            state[:weak] = false # a strong reference promotes the pending UND binding
+          end
+        end
+
+        # Emits the resolved globals as output symbols and returns a name -> output
+        # Symbol map. A defined winner is shifted onto its merged section; an
+        # all-undefined name stays UND with the strongest observed binding.
+        def emit_globals
+          out = {}
+          @globals.each do |name, state|
+            out[name] =
+              if state[:defined]
+                sym = state[:sym]
+                section = @out_section[sym.section.name]
+                @writer.add_symbol(name: name, bind: state[:weak] ? :weak : :global,
+                                   type: sym.type, visibility: sym.visibility, section: section,
+                                   value: @placement[state[:oi]][sym.section.index] + sym.value,
+                                   size: sym.size)
+              else
+                @writer.add_symbol(name: name, bind: state[:weak] ? :weak : :global,
+                                   type: :notype, shndx: SHN_UNDEF)
+              end
+          end
+          out
+        end
+
+        # --- relocation retargeting -----------------------------------------
+
+        # Carries every input relocation to the merged output: the offset gains the
+        # target piece's placement, and the symbol reference is re-pointed. A
+        # section-symbol reference moves to the OUTPUT section symbol with its
+        # addend bumped by the referenced piece's placement (the section's bytes
+        # now start there); every other reference keeps its addend. Relocation
+        # types pass through untouched.
+        def retarget_relocations
+          @objects.each_with_index do |obj, oi|
+            obj[:reader].relocation_sections.each do |rs|
+              target = rs.target
+              next unless target && @out_section.key?(target.name)
+
+              base = @placement[oi][target.index]
+              out_target = @out_section[target.name]
+              rs.relocations.each { |reloc| emit_relocation(oi, out_target, base, reloc) }
+            end
+          end
+        end
+
+        def emit_relocation(oi, out_target, base, reloc)
+          sym = reloc.symbol
+          if sym && sym.type == :section && sym.section && @out_section.key?(sym.section.name)
+            out_sym = @section_symbol[sym.section.name]
+            addend = reloc.addend + @placement[oi][sym.section.index]
+          else
+            out_sym = (sym && @remap[oi][sym.index]) || @writer.null_symbol
+            addend = reloc.addend
+          end
+
+          @writer.add_relocation(target: out_target, offset: reloc.offset + base,
+                                 symbol: out_sym, type: reloc.type, addend: addend)
+        end
+
+        def align(value, alignment)
+          (value + alignment - 1) / alignment * alignment
+        end
+      end
+    end
+  end
+end