rubycc 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +56 -0
  3. data/LICENSE.txt +21 -0
  4. data/NOTICE +52 -0
  5. data/README.md +208 -0
  6. data/data/README.md +117 -0
  7. data/data/r10_corpus_scan.json +4082 -0
  8. data/data/r10_manual_classification.json +5342 -0
  9. data/data/r10_verification_m4a.json +1531 -0
  10. data/data/r10_verification_m4b.json +1818 -0
  11. data/data/r10_verification_m4c.json +1489 -0
  12. data/data/r10_verification_m4d.json +318 -0
  13. data/data/r10_verification_rbs.json +192 -0
  14. data/data/verified_gems.json +397 -0
  15. data/exe/rmake +16 -0
  16. data/exe/rubycc +9 -0
  17. data/exe/rubycc-ar +114 -0
  18. data/exe/rubycc-doctor +14 -0
  19. data/exe/rubycc-pkgconf +8 -0
  20. data/include/float.h +87 -0
  21. data/include/iso646.h +18 -0
  22. data/include/libc/alloca.h +18 -0
  23. data/include/libc/arpa/inet.h +61 -0
  24. data/include/libc/assert.h +43 -0
  25. data/include/libc/dirent.h +72 -0
  26. data/include/libc/dlfcn.h +58 -0
  27. data/include/libc/features.h +205 -0
  28. data/include/libc/glibc/aarch64/ctype.h +108 -0
  29. data/include/libc/glibc/aarch64/endian.h +54 -0
  30. data/include/libc/glibc/aarch64/errno.h +154 -0
  31. data/include/libc/glibc/aarch64/fcntl.h +144 -0
  32. data/include/libc/glibc/aarch64/inttypes.h +182 -0
  33. data/include/libc/glibc/aarch64/limits.h +63 -0
  34. data/include/libc/glibc/aarch64/pthread.h +131 -0
  35. data/include/libc/glibc/aarch64/setjmp.h +84 -0
  36. data/include/libc/glibc/aarch64/stdint.h +174 -0
  37. data/include/libc/glibc/aarch64/sys/epoll.h +88 -0
  38. data/include/libc/glibc/aarch64/sys/fcntl.h +19 -0
  39. data/include/libc/glibc/aarch64/sys/select.h +75 -0
  40. data/include/libc/glibc/aarch64/sys/stat.h +136 -0
  41. data/include/libc/glibc/aarch64/sys/syscall.h +167 -0
  42. data/include/libc/glibc/aarch64/sys/time.h +74 -0
  43. data/include/libc/glibc/aarch64/sys/types.h +127 -0
  44. data/include/libc/glibc/aarch64/time.h +113 -0
  45. data/include/libc/glibc/x86_64/ctype.h +108 -0
  46. data/include/libc/glibc/x86_64/endian.h +54 -0
  47. data/include/libc/glibc/x86_64/errno.h +154 -0
  48. data/include/libc/glibc/x86_64/fcntl.h +134 -0
  49. data/include/libc/glibc/x86_64/inttypes.h +182 -0
  50. data/include/libc/glibc/x86_64/limits.h +61 -0
  51. data/include/libc/glibc/x86_64/pthread.h +111 -0
  52. data/include/libc/glibc/x86_64/setjmp.h +82 -0
  53. data/include/libc/glibc/x86_64/stdint.h +163 -0
  54. data/include/libc/glibc/x86_64/sys/epoll.h +91 -0
  55. data/include/libc/glibc/x86_64/sys/fcntl.h +19 -0
  56. data/include/libc/glibc/x86_64/sys/select.h +75 -0
  57. data/include/libc/glibc/x86_64/sys/stat.h +130 -0
  58. data/include/libc/glibc/x86_64/sys/syscall.h +188 -0
  59. data/include/libc/glibc/x86_64/sys/time.h +74 -0
  60. data/include/libc/glibc/x86_64/sys/types.h +123 -0
  61. data/include/libc/glibc/x86_64/time.h +113 -0
  62. data/include/libc/grp.h +50 -0
  63. data/include/libc/langinfo.h +126 -0
  64. data/include/libc/link.h +18 -0
  65. data/include/libc/locale.h +83 -0
  66. data/include/libc/math.h +191 -0
  67. data/include/libc/netinet/in.h +126 -0
  68. data/include/libc/netinet/tcp.h +44 -0
  69. data/include/libc/poll.h +46 -0
  70. data/include/libc/pwd.h +58 -0
  71. data/include/libc/regex.h +53 -0
  72. data/include/libc/sched.h +35 -0
  73. data/include/libc/signal.h +204 -0
  74. data/include/libc/stdio.h +157 -0
  75. data/include/libc/stdlib.h +92 -0
  76. data/include/libc/string.h +74 -0
  77. data/include/libc/strings.h +25 -0
  78. data/include/libc/sys/cdefs.h +126 -0
  79. data/include/libc/sys/inotify.h +111 -0
  80. data/include/libc/sys/ioctl.h +36 -0
  81. data/include/libc/sys/mman.h +65 -0
  82. data/include/libc/sys/param.h +41 -0
  83. data/include/libc/sys/resource.h +109 -0
  84. data/include/libc/sys/socket.h +167 -0
  85. data/include/libc/sys/statfs.h +82 -0
  86. data/include/libc/sys/timerfd.h +55 -0
  87. data/include/libc/sys/uio.h +40 -0
  88. data/include/libc/sys/un.h +25 -0
  89. data/include/libc/sys/utsname.h +35 -0
  90. data/include/libc/sys/wait.h +135 -0
  91. data/include/libc/termios.h +179 -0
  92. data/include/libc/unistd.h +194 -0
  93. data/include/stdalign.h +16 -0
  94. data/include/stdarg.h +31 -0
  95. data/include/stdatomic.h +158 -0
  96. data/include/stdbool.h +15 -0
  97. data/include/stdckdint.h +28 -0
  98. data/include/stddef.h +60 -0
  99. data/include/stdnoreturn.h +18 -0
  100. data/include/x86intrin.h +16 -0
  101. data/lib/rubycc/backend/aarch64.rb +1724 -0
  102. data/lib/rubycc/backend/x86_64.rb +1369 -0
  103. data/lib/rubycc/compile_error.rb +36 -0
  104. data/lib/rubycc/compiler.rb +305 -0
  105. data/lib/rubycc/doctor/builder.rb +151 -0
  106. data/lib/rubycc/doctor/cli.rb +190 -0
  107. data/lib/rubycc/doctor/fetcher.rb +84 -0
  108. data/lib/rubycc/doctor/gemfile.rb +115 -0
  109. data/lib/rubycc/doctor/verified_gems.rb +114 -0
  110. data/lib/rubycc/doctor.rb +16 -0
  111. data/lib/rubycc/driver.rb +463 -0
  112. data/lib/rubycc/front/ast.rb +528 -0
  113. data/lib/rubycc/front/constant_evaluator.rb +631 -0
  114. data/lib/rubycc/front/initializer_resolver.rb +592 -0
  115. data/lib/rubycc/front/lexeme_reader.rb +460 -0
  116. data/lib/rubycc/front/lexer.rb +232 -0
  117. data/lib/rubycc/front/parser.rb +4122 -0
  118. data/lib/rubycc/front/token.rb +49 -0
  119. data/lib/rubycc/ir/call_convention.rb +486 -0
  120. data/lib/rubycc/ir/generator.rb +6036 -0
  121. data/lib/rubycc/ir/ir.rb +417 -0
  122. data/lib/rubycc/link/compat_runtime.rb +89 -0
  123. data/lib/rubycc/link/errors.rb +13 -0
  124. data/lib/rubycc/link/executable_linker.rb +405 -0
  125. data/lib/rubycc/link/library_resolver.rb +437 -0
  126. data/lib/rubycc/link/partial_linker.rb +546 -0
  127. data/lib/rubycc/link/shared_linker.rb +1732 -0
  128. data/lib/rubycc/mkmf_shim.rb +83 -0
  129. data/lib/rubycc/objfile/ar_archive.rb +437 -0
  130. data/lib/rubycc/objfile/elf_reader.rb +646 -0
  131. data/lib/rubycc/objfile/elf_writer.rb +891 -0
  132. data/lib/rubycc/objfile/relocatable_writer.rb +376 -0
  133. data/lib/rubycc/pkgconf/cli.rb +139 -0
  134. data/lib/rubycc/pkgconf/errors.rb +39 -0
  135. data/lib/rubycc/pkgconf/model.rb +28 -0
  136. data/lib/rubycc/pkgconf/parser.rb +109 -0
  137. data/lib/rubycc/pkgconf/pkgconf.rb +15 -0
  138. data/lib/rubycc/pkgconf/resolver.rb +70 -0
  139. data/lib/rubycc/pkgconf/search_path.rb +38 -0
  140. data/lib/rubycc/pkgconf/system_path_filter.rb +131 -0
  141. data/lib/rubycc/preprocess/constant_expression.rb +219 -0
  142. data/lib/rubycc/preprocess/glibc_version.rb +124 -0
  143. data/lib/rubycc/preprocess/pp_token.rb +62 -0
  144. data/lib/rubycc/preprocess/preprocessor.rb +2020 -0
  145. data/lib/rubycc/preprocess/scanner.rb +290 -0
  146. data/lib/rubycc/preprocess/token_converter.rb +157 -0
  147. data/lib/rubycc/rmake/cli.rb +143 -0
  148. data/lib/rubycc/rmake/errors.rb +71 -0
  149. data/lib/rubycc/rmake/executor.rb +818 -0
  150. data/lib/rubycc/rmake/expander.rb +251 -0
  151. data/lib/rubycc/rmake/makefile.rb +352 -0
  152. data/lib/rubycc/rmake/model.rb +137 -0
  153. data/lib/rubycc/rmake/parser.rb +226 -0
  154. data/lib/rubycc/rmake/rmake.rb +15 -0
  155. data/lib/rubycc/type.rb +1236 -0
  156. data/lib/rubycc/version.rb +5 -0
  157. data/lib/rubycc.rb +36 -0
  158. data/lib/rubygems_plugin.rb +102 -0
  159. metadata +219 -0
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Rubycc
4
+ # Base error class (also defined in rubycc.rb; reopened here so this file can
5
+ # be required standalone).
6
+ class Error < StandardError; end
7
+
8
+ # A diagnostic raised for any user-facing compilation failure (lexing,
9
+ # parsing, ...). Carries enough source location information (N3) to render a
10
+ # gcc-style message with a caret pointing at the offending column.
11
+ class CompileError < Error
12
+ attr_reader :description, :filename, :line, :column, :source_line
13
+
14
+ # description:: the short error text, e.g. "expected ';'"
15
+ # filename:: the source file name for the message header
16
+ # line:: 1-based line number
17
+ # column:: 1-based column number
18
+ # source_line:: the full text of the offending source line (no newline)
19
+ def initialize(description, filename:, line:, column:, source_line:)
20
+ @description = description
21
+ @filename = filename
22
+ @line = line
23
+ @column = column
24
+ @source_line = source_line
25
+ super(build_message)
26
+ end
27
+
28
+ private
29
+
30
+ def build_message
31
+ header = "#{filename}:#{line}:#{column}: error: #{description}"
32
+ caret = "#{" " * (column - 1)}^"
33
+ "#{header}\n#{source_line}\n#{caret}"
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,305 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "set"
4
+ require_relative "compile_error"
5
+ require_relative "preprocess/preprocessor"
6
+ require_relative "front/parser"
7
+ require_relative "ir/generator"
8
+ require_relative "backend/x86_64"
9
+ require_relative "backend/aarch64"
10
+ require_relative "objfile/elf_writer"
11
+
12
+ module Rubycc
13
+ # Orchestrates every compilation stage: source -> tokens -> AST -> IR ->
14
+ # machine code -> ELF relocatable object.
15
+ class Compiler
16
+ # The backend dispatch table: a normalized target name selects the code
17
+ # generator that lowers IR to that machine's instructions, the ELF machine
18
+ # description (e_machine value + relocation-type table) the object writer
19
+ # emits under, and the one ABI trait the machine-independent front end has to
20
+ # know about — whether plain `char` is signed. That last entry is what makes
21
+ # the front end target-aware at all: the signedness of plain `char` is
22
+ # implementation-defined (6.2.5p15) and each ABI pins it, signed under the
23
+ # x86-64 System V psABI and unsigned under AAPCS64, so it has to reach type
24
+ # resolution rather than being decided once for the whole compiler. The
25
+ # `arch_macros` entry is the same idea one stage earlier: the CPU-identifying
26
+ # predefined macros a translation unit (and the libc headers it includes)
27
+ # dispatches on. `unnamed_bitfields_align` is a third: the ABIs disagree on
28
+ # whether an unnamed bit-field's type raises its aggregate's alignment, so
29
+ # sizeof and _Alignof of such a struct are target-dependent (see
30
+ # StructType#define). `libc_arch` is a fourth: it names the bundled
31
+ # libc-and-arch header layer the preprocessor puts on its default search path,
32
+ # so a cross compile reads the target's ABI headers (struct stat, nlink_t,
33
+ # WCHAR_* and kin) rather than the host's. Apart from those four every entry
34
+ # shares the orchestration logic below unchanged.
35
+ #
36
+ # Which C library those headers describe is deliberately *not* in this table:
37
+ # the machine and the libc are independent axes (either arch runs either
38
+ # libc), so it is #compile's own `libc` keyword instead.
39
+ TARGETS = {
40
+ "x86_64" => { backend: Backend::X86_64, machine: ObjFile::ELFWriter::X86_64,
41
+ char_signed: true,
42
+ arch_macros: Preprocess::Preprocessor::X86_64_ARCH_MACROS,
43
+ libc_arch: "x86_64",
44
+ unnamed_bitfields_align: false,
45
+ convention: IR::CallConvention::SYSTEM_V_AMD64 },
46
+ "aarch64" => { backend: Backend::AArch64, machine: ObjFile::ELFWriter::AARCH64,
47
+ char_signed: false,
48
+ arch_macros: Preprocess::Preprocessor::AARCH64_ARCH_MACROS,
49
+ libc_arch: "aarch64",
50
+ unnamed_bitfields_align: true,
51
+ convention: IR::CallConvention::AAPCS64 }
52
+ }.freeze
53
+
54
+ # Compiles C source into an ELF64 relocatable object, returned as an
55
+ # ASCII-8BIT String. Raises Rubycc::CompileError on user errors. `target`
56
+ # names the machine to generate code for (see TARGETS); it defaults to
57
+ # x86_64 and an unknown value is a caller error. `libc` names the C library
58
+ # whose ABI the bundled headers are to describe ("glibc" or "musl"); it
59
+ # defaults to the host's own, so an unconfigured compile on a musl host
60
+ # reads the musl branches, and an unknown value is a caller error too (the
61
+ # preprocessor raises it).
62
+ def compile(source, filename:, include_paths: [], pic: false, defines: [], system_includes: true,
63
+ target: "x86_64", libc: Preprocess::Preprocessor.host_libc,
64
+ default_visibility: :default)
65
+ entry = TARGETS.fetch(target) { raise ArgumentError, "unsupported target: #{target.inspect}" }
66
+ # The target's plain-`char` type, threaded through every stage that builds
67
+ # or reasons about one: the preprocessor (which predefines
68
+ # __CHAR_UNSIGNED__ when it is unsigned), the parser (which resolves the
69
+ # `char` type-specifier to it) and the generator (which types a string
70
+ # literal's elements with it).
71
+ plain_char = Type.plain_char(entry[:char_signed])
72
+ tokens = Preprocess::Preprocessor.new(char_unsigned: plain_char.unsigned?,
73
+ arch_macros: entry[:arch_macros],
74
+ libc_arch: entry[:libc_arch],
75
+ libc: libc)
76
+ .run(source, filename: filename,
77
+ include_paths: include_paths, defines: defines,
78
+ system_includes: system_includes)
79
+ program = Front::Parser.new(tokens, plain_char: plain_char,
80
+ unnamed_bitfields_align: entry[:unnamed_bitfields_align],
81
+ builtin_va_list: entry[:convention].va_list_type).parse
82
+ ir_program = IR::Generator.new(plain_char: plain_char,
83
+ convention: entry[:convention]).generate(program, pic: pic)
84
+
85
+ backend = entry[:backend].new
86
+ writer = ObjFile::ELFWriter.new(machine: entry[:machine])
87
+ writer.add_file_symbol(File.basename(filename))
88
+
89
+ # Lay out the translation unit's string pool as .rodata: each interned
90
+ # string in id order, NUL-terminated. `string_offsets[id]` is the byte
91
+ # offset of string `id` within the section.
92
+ rodata = +"".b
93
+ string_offsets = []
94
+ ir_program.strings.each do |bytes|
95
+ string_offsets << rodata.bytesize
96
+ rodata << bytes << "\0".b
97
+ end
98
+
99
+ # Lay out the file-scope variables: initialized ones into .data (their
100
+ # values packed little-endian at the declared width) and zero-initialized
101
+ # ones into .bss (space reserved only). Each is placed at its type's
102
+ # alignment and registered as a global STT_OBJECT symbol; the section
103
+ # alignment is the widest member's.
104
+ data = +"".b
105
+ data_align = 1
106
+ bss_size = 0
107
+ bss_align = 1
108
+ # Symbols a .data pointer slot resolves against (a "&global", a decayed
109
+ # global array, or a function address in a function-pointer global). A
110
+ # reference to a function defined elsewhere must be registered as an
111
+ # undefined symbol, but only once the set of locally defined functions is
112
+ # known (after the text is compiled), so the names are collected here.
113
+ data_symbol_refs = []
114
+ ir_program.globals.each do |global|
115
+ # A `static` global gets an internal-linkage (STB_LOCAL) object symbol,
116
+ # an ordinary one a global (STB_GLOBAL) symbol; both are laid out into
117
+ # .data/.bss identically.
118
+ internal = global.linkage == :internal
119
+ if global.init.nil?
120
+ bss_align = [bss_align, global.align].max
121
+ bss_size = align_up(bss_size, global.align)
122
+ add_object_symbol(writer, internal, global.name, :bss, bss_size, global.size,
123
+ visibility: ir_program.visibility.fetch(global.name, default_visibility))
124
+ bss_size += global.size
125
+ else
126
+ data_align = [data_align, global.align].max
127
+ offset = align_up(data.bytesize, global.align)
128
+ data << ("\0".b * (offset - data.bytesize))
129
+ add_object_symbol(writer, internal, global.name, :data, offset, global.size,
130
+ visibility: ir_program.visibility.fetch(global.name, default_visibility))
131
+ data << global.init.bytes
132
+ # Each pointer slot in the image is patched by a .data relocation,
133
+ # its .text-relative offset (within the global) biased by where the
134
+ # global itself landed in .data.
135
+ global.init.relocations.each do |reloc|
136
+ register_data_relocation(writer, reloc, offset, string_offsets)
137
+ data_symbol_refs << reloc.symbol if reloc.kind == :symbol
138
+ end
139
+ end
140
+ end
141
+ writer.set_data(data, align: data_align) unless data.empty?
142
+ writer.set_bss(bss_size, align: bss_align) if bss_size.positive?
143
+
144
+ text = +"".b
145
+ # A defined function's name is only ever tested for membership below
146
+ # (never iterated in order), so a Set keeps that check O(1) instead of
147
+ # the O(n) linear scan an Array would need per relocation.
148
+ defined_names = Set.new
149
+ relocations = []
150
+ ir_program.functions.each do |ir_func|
151
+ result = backend.compile(ir_func)
152
+ # Align each function to 16 bytes with the target's NOP filler, keeping
153
+ # the output deterministic and every entry point aligned.
154
+ pad_to_alignment(text, 16, entry[:machine].text_padding)
155
+ base = text.bytesize
156
+ text << result.bytes
157
+ result.symbols.each do |sym|
158
+ # A `static` function is a file-local (STB_LOCAL) symbol; an ordinary
159
+ # one is global. Either way it is a defined name a same-object
160
+ # reference resolves against, never an undefined external.
161
+ if ir_func.linkage == :internal
162
+ writer.add_local_func(sym[:name], base + sym[:offset], sym[:size])
163
+ else
164
+ writer.add_global_func(
165
+ sym[:name], base + sym[:offset], sym[:size],
166
+ visibility: ir_program.visibility.fetch(ir_func.name, default_visibility)
167
+ )
168
+ end
169
+ defined_names << sym[:name]
170
+ end
171
+ result.relocations.each do |reloc|
172
+ relocations << reloc.merge(offset: base + reloc[:offset])
173
+ end
174
+ end
175
+
176
+ # A function-pointer global that names a function defined elsewhere leaves
177
+ # an undefined symbol for the linker; one that names a local function or
178
+ # another global is already in the symbol table. Like `defined_names`,
179
+ # this is only ever queried for membership, so it stays a Set.
180
+ known_names = defined_names | ir_program.globals.map(&:name)
181
+ data_symbol_refs.each do |symbol|
182
+ writer.add_undefined_symbol(symbol) unless known_names.include?(symbol)
183
+ end
184
+
185
+ relocations.each do |reloc|
186
+ case reloc[:kind]
187
+ when :call, :func
188
+ # A call, or a taken function address, whose target is not defined in
189
+ # this translation unit becomes an undefined symbol for the linker to
190
+ # resolve (e.g. libc's abs). The two are recorded as distinct kinds
191
+ # because they only coincide on some targets: x86_64 resolves both with
192
+ # the same PC-relative PLT32, while aarch64 needs a `bl`'s CALL26 for
193
+ # one and an address-forming instruction pair for the other.
194
+ writer.add_undefined_symbol(reloc[:symbol]) unless defined_names.include?(reloc[:symbol])
195
+ if reloc[:kind] == :call
196
+ writer.add_text_relocation(offset: reloc[:offset], symbol: reloc[:symbol])
197
+ else
198
+ writer.add_func_relocation(offset: reloc[:offset], symbol: reloc[:symbol])
199
+ end
200
+ when :string
201
+ # A reference from .text into .rodata, resolved against the .rodata
202
+ # section symbol and displaced by the string's byte offset within the
203
+ # pool. The offset is passed unbiased: whatever a target's own field
204
+ # placement demands on top of it belongs to its machine description,
205
+ # not to this machine-independent layer.
206
+ writer.add_rodata_relocation(offset: reloc[:offset],
207
+ addend: string_offsets[reloc[:string_id]])
208
+ when :global
209
+ # A reference from .text addressing a file-scope variable, resolved
210
+ # against that variable's own object symbol. A variable only declared
211
+ # `extern` in this unit (referenced but never defined here) has no
212
+ # local object symbol, so it becomes an undefined symbol for the
213
+ # linker, just like an undefined call target.
214
+ writer.add_undefined_symbol(reloc[:symbol]) unless known_names.include?(reloc[:symbol])
215
+ writer.add_global_relocation(offset: reloc[:offset], symbol: reloc[:symbol])
216
+ when :got
217
+ # A PIC access (-fPIC) through the Global Offset Table: a load of the
218
+ # address of a symbol this unit does not define — an extern file-scope
219
+ # object, or an external function whose address is taken — from that
220
+ # symbol's GOT slot. The symbol becomes an undefined symbol for the
221
+ # linker to bind (unless another part of this unit defines it); which
222
+ # relocation addresses the slot is the machine description's business.
223
+ writer.add_undefined_symbol(reloc[:symbol]) unless known_names.include?(reloc[:symbol])
224
+ writer.add_got_relocation(offset: reloc[:offset], symbol: reloc[:symbol])
225
+ end
226
+ end
227
+
228
+ # A `__attribute__((constructor))` / `((destructor))` function becomes a
229
+ # slot in this object's .init_array / .fini_array pointing at its own text
230
+ # symbol. Registered after the text loop, so every function symbol the
231
+ # slots resolve against is already in the symbol table.
232
+ ir_program.array_entries.each do |entry|
233
+ writer.add_array_entry(kind: entry.kind, priority: entry.priority, symbol: entry.symbol)
234
+ end
235
+
236
+ writer.set_rodata(rodata) unless rodata.empty?
237
+ writer.add_text_section(text)
238
+ writer.to_binary
239
+ end
240
+
241
+ private
242
+
243
+ # Pads `buffer` with repetitions of the target's NOP encoding until its
244
+ # length is a multiple of `alignment`. `filler` may be several bytes wide
245
+ # (aarch64's NOP is a whole 32-bit word), so the gap is filled in whole
246
+ # units; a gap that is not a multiple of the filler width cannot arise
247
+ # because the alignment is a multiple of the instruction size.
248
+ def pad_to_alignment(buffer, alignment, filler)
249
+ remainder = buffer.bytesize % alignment
250
+ return unless remainder.positive?
251
+
252
+ buffer << (filler * ((alignment - remainder) / filler.bytesize))
253
+ end
254
+
255
+ # Rounds `value` up to the next multiple of `alignment`.
256
+ def align_up(value, alignment)
257
+ (value + alignment - 1) / alignment * alignment
258
+ end
259
+
260
+ # Registers a file-scope object's symbol, choosing the internal-linkage
261
+ # (STB_LOCAL) writer entry for a `static` object and the global one
262
+ # otherwise.
263
+ def add_object_symbol(writer, internal, name, section, offset, size, visibility: :default)
264
+ if internal
265
+ writer.add_local_object(name, section, offset, size)
266
+ else
267
+ writer.add_global_object(name, section, offset, size, visibility: visibility)
268
+ end
269
+ end
270
+
271
+ # Registers one global-image relocation with the ELF writer, translating a
272
+ # within-global slot offset into a .data-section offset. A :symbol reloc
273
+ # points at another object's symbol (an absolute 64-bit address, plus the
274
+ # relocation's own byte displacement for a computed "&arr[i]"); a :string
275
+ # reloc points into .rodata, its addend the interned string's byte offset
276
+ # plus that same displacement.
277
+ def register_data_relocation(writer, reloc, global_offset, string_offsets)
278
+ case reloc.kind
279
+ when :symbol
280
+ writer.add_data_relocation(offset: global_offset + reloc.offset,
281
+ symbol: reloc.symbol, addend: reloc.addend)
282
+ when :string
283
+ writer.add_data_rodata_relocation(offset: global_offset + reloc.offset,
284
+ addend: string_offsets[reloc.string_id] + reloc.addend)
285
+ end
286
+ end
287
+
288
+ public
289
+
290
+ # Convenience: read `input_path`, compile it and write the object to
291
+ # `output_path`.
292
+ def self.compile_file(input_path, output_path, include_paths: [], pic: false, defines: [],
293
+ system_includes: true, target: "x86_64",
294
+ libc: Preprocess::Preprocessor.host_libc,
295
+ default_visibility: :default)
296
+ source = File.read(input_path)
297
+ binary = new.compile(source, filename: input_path, include_paths: include_paths,
298
+ pic: pic, defines: defines, system_includes: system_includes,
299
+ target: target, libc: libc,
300
+ default_visibility: default_visibility)
301
+ File.binwrite(output_path, binary)
302
+ output_path
303
+ end
304
+ end
305
+ end
@@ -0,0 +1,151 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "open3"
4
+ require "fileutils"
5
+ require "timeout"
6
+ require "tmpdir"
7
+
8
+ require_relative "fetcher"
9
+
10
+ module Rubycc
11
+ module Doctor
12
+ # The on-the-fly build a gem gets when it is not in the verified database: the
13
+ # same path RubyGems takes for a `gem install`, driven by hand so each stage's
14
+ # outcome can be reported separately.
15
+ #
16
+ # Per gem: fetch the .gem, read its real extensions list; with none it needs
17
+ # no C toolchain (no_ext). For each extension, run its extconf.rb with the
18
+ # mkmf_shim loaded (so mkmf's conftests and the Makefile it writes route to
19
+ # rubycc), then build the generated Makefile with rmake (which always
20
+ # substitutes rubycc for the compiler/linker). On success the freshly built
21
+ # .so is required in a child process to prove its Init function dlopens.
22
+ #
23
+ # A failure is pinned to a stage — extconf, compile, link, require or the
24
+ # per-gem timeout — with a short summary of the tool's stderr.
25
+ class Builder
26
+ # Repository lib/ (this file lives at lib/rubycc/doctor/) and the executables
27
+ # the build shells out to.
28
+ REPO_LIB = File.expand_path("../..", __dir__)
29
+ REPO_ROOT = File.expand_path("../../..", __dir__)
30
+ RMAKE_EXE = File.join(REPO_ROOT, "exe", "rmake")
31
+ MKMF_SHIM_REQUIRE = "-rrubycc/mkmf_shim"
32
+
33
+ # The outcome of building one gem.
34
+ # status : :no_ext | :built | :failed | :unknown
35
+ # stage : nil, or the failing stage (:extconf/:compile/:link/:require/:timeout)
36
+ # reason : short human-readable explanation (stderr summary etc.)
37
+ # sos : produced .so paths (on :built)
38
+ # require_ok : whether the require smoke check passed (:built only; a
39
+ # failed require does not demote the build — the .so was made —
40
+ # but is noted)
41
+ Result = Struct.new(:status, :stage, :reason, :sos, :require_ok, keyword_init: true)
42
+
43
+ # +timeout+ is the per-gem ceiling in seconds. +out+ receives progress lines.
44
+ def initialize(cache_dir:, timeout: 300, out: nil)
45
+ @cache_dir = cache_dir
46
+ @timeout = timeout
47
+ @out = out
48
+ end
49
+
50
+ # Build +entry+ (a Gemfile::Entry). Returns a Result. Never raises: every
51
+ # failure mode is folded into a Result the CLI can print.
52
+ def build(entry)
53
+ fetcher = Fetcher.new(@cache_dir)
54
+ version = entry.version || fetcher.latest_version(entry.name)
55
+ return Result.new(status: :unknown, reason: "could not resolve a version") unless version
56
+
57
+ gem_path =
58
+ begin
59
+ fetcher.download(entry.name, version)
60
+ rescue Fetcher::FetchError => e
61
+ return Result.new(status: :unknown, reason: "fetch failed (offline?): #{e.message}")
62
+ end
63
+
64
+ spec = fetcher.spec(gem_path)
65
+ return Result.new(status: :no_ext) if spec.extensions.empty?
66
+
67
+ Timeout.timeout(@timeout) { build_extensions(entry.name, version, gem_path, spec) }
68
+ rescue Timeout::Error
69
+ Result.new(status: :failed, stage: :timeout, reason: "exceeded #{@timeout}s")
70
+ rescue StandardError => e
71
+ Result.new(status: :failed, stage: :internal, reason: "#{e.class}: #{e.message}")
72
+ end
73
+
74
+ private
75
+
76
+ # Unpack the gem and drive extconf -> rmake -> require for each extension.
77
+ def build_extensions(name, version, gem_path, spec)
78
+ dir = File.join(@cache_dir, "#{name}-#{version}")
79
+ FileUtils.rm_rf(dir)
80
+ FileUtils.mkdir_p(dir)
81
+ Gem::Package.new(gem_path).extract_files(dir)
82
+
83
+ sos = []
84
+ spec.extensions.each do |ext_rel|
85
+ ext_dir = File.join(dir, File.dirname(ext_rel))
86
+ extconf = File.basename(ext_rel)
87
+
88
+ result = build_one(ext_dir, extconf)
89
+ return result unless result.status == :built
90
+
91
+ sos.concat(result.sos)
92
+ end
93
+
94
+ require_ok = sos.all? { |so| require_check(so) }
95
+ Result.new(status: :built, sos: sos, require_ok: require_ok,
96
+ reason: require_ok ? nil : "built, but the require smoke check failed")
97
+ end
98
+
99
+ # extconf -> rmake -> locate .so for a single extension directory.
100
+ def build_one(ext_dir, extconf)
101
+ log("extconf: #{ext_dir}/#{extconf}")
102
+ out, ok = run(RbConfig.ruby, "-I#{REPO_LIB}", MKMF_SHIM_REQUIRE, extconf, chdir: ext_dir)
103
+ unless ok && File.file?(File.join(ext_dir, "Makefile"))
104
+ return Result.new(status: :failed, stage: :extconf, reason: summarize(out))
105
+ end
106
+
107
+ log("rmake: #{ext_dir}")
108
+ out, ok = run(RbConfig.ruby, RMAKE_EXE, chdir: ext_dir)
109
+ unless ok
110
+ stage = Dir.glob(File.join(ext_dir, "**", "*.o")).empty? ? :compile : :link
111
+ return Result.new(status: :failed, stage: stage, reason: summarize(out))
112
+ end
113
+
114
+ sos = Dir.glob(File.join(ext_dir, "**", "*.so"))
115
+ return Result.new(status: :failed, stage: :link, reason: "rmake reported success but produced no .so") if sos.empty?
116
+
117
+ Result.new(status: :built, sos: sos)
118
+ end
119
+
120
+ # Prove the built object dlopens and its Init function runs, in a child
121
+ # process so a crash cannot take the doctor down. Requiring a .so by absolute
122
+ # path invokes Init_<basename>, which is the load we want to confirm.
123
+ def require_check(so)
124
+ _out, ok = run(RbConfig.ruby, "-e", "require #{so.dump}")
125
+ ok
126
+ end
127
+
128
+ # Run a command with the build environment, returning [combined_output, ok?].
129
+ def run(*cmd, chdir: nil)
130
+ env = { "RUBYLIB" => [REPO_LIB, ENV["RUBYLIB"]].compact.join(File::PATH_SEPARATOR) }
131
+ opts = {}
132
+ opts[:chdir] = chdir if chdir
133
+ out, status = Open3.capture2e(env, *cmd, **opts)
134
+ [out, status.success?]
135
+ rescue StandardError => e
136
+ ["#{e.class}: #{e.message}", false]
137
+ end
138
+
139
+ # The last few non-empty lines of a tool's output — enough to name the error
140
+ # without dumping a whole build log into the report.
141
+ def summarize(output)
142
+ lines = output.to_s.lines.map(&:chomp).reject(&:empty?)
143
+ lines.last(8).join("\n")
144
+ end
145
+
146
+ def log(msg)
147
+ @out&.puts(" #{msg}")
148
+ end
149
+ end
150
+ end
151
+ end
@@ -0,0 +1,190 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "optparse"
4
+
5
+ require_relative "../version"
6
+
7
+ require_relative "gemfile"
8
+ require_relative "verified_gems"
9
+ require_relative "builder"
10
+
11
+ module Rubycc
12
+ module Doctor
13
+ # `rubycc doctor` — tells a Ruby application developer whether rubycc can
14
+ # build the C extensions their app depends on.
15
+ #
16
+ # It reads the app's Gemfile.lock (or Gemfile), and for each gem: reports it
17
+ # verified when the shipped build-verified database (data/verified_gems.json)
18
+ # already covers that name+version; otherwise fetches and builds it on the
19
+ # spot through the same extconf -> rmake -> rubycc path a real install takes,
20
+ # reporting exactly where any failure occurred. Gems with no C extension need
21
+ # rubycc at all and are marked as such.
22
+ #
23
+ # The verified check is offline and instant; only unverified gems reach the
24
+ # network. The bottom line is an adoption verdict and an exit code (0 when
25
+ # every C-extension gem is verified or built, 1 otherwise).
26
+ class CLI
27
+ # Per-status display: symbol + label. Ordering of keys is the report legend.
28
+ STATUS = {
29
+ verified: ["✔", "verified"],
30
+ built: ["✔", "built on the spot"],
31
+ no_ext: ["-", "no C extension"],
32
+ failed: ["✘", "FAILED"],
33
+ unknown: ["?", "unknown"],
34
+ local: ["-", "local/vcs source (skipped)"],
35
+ skipped_budget: ["?", "skipped (build budget reached)"]
36
+ }.freeze
37
+
38
+ def self.run(argv, out: $stdout, err: $stderr, verified: nil, builder: nil)
39
+ new(out: out, err: err, verified: verified, builder: builder).run(argv)
40
+ end
41
+
42
+ def initialize(out:, err:, verified: nil, builder: nil)
43
+ @out = out
44
+ @err = err
45
+ @verified = verified
46
+ @builder = builder
47
+ end
48
+
49
+ def run(argv)
50
+ options = parse(argv)
51
+ return 0 if options[:help]
52
+
53
+ result = Gemfile.load(options[:gemfile])
54
+ unless result
55
+ @err.puts("rubycc doctor: no Gemfile.lock or Gemfile found at #{options[:gemfile]}")
56
+ return 2
57
+ end
58
+
59
+ verified = @verified || VerifiedGems.load
60
+ rows = evaluate(result, verified, options)
61
+ report(result, rows)
62
+ exit_code(rows)
63
+ rescue OptionParser::ParseError => e
64
+ @err.puts("rubycc doctor: #{e.message}")
65
+ 2
66
+ end
67
+
68
+ private
69
+
70
+ # A row of the report: the gem, its resolved status and an optional note.
71
+ Row = Struct.new(:name, :version, :status, :note, keyword_init: true)
72
+
73
+ def parse(argv)
74
+ options = { gemfile: File.expand_path("Gemfile"), timeout: 300, max_builds: nil }
75
+ parser = OptionParser.new do |o|
76
+ o.banner = "Usage: rubycc-doctor [options]"
77
+ # OptionParser answers --version itself, but only once it has been given
78
+ # one; without this it prints its "version unknown" fallback, which is a
79
+ # poor answer from a command that does know its version.
80
+ o.version = Rubycc::VERSION
81
+ o.on("--gemfile PATH", "Gemfile path (default ./Gemfile; .lock preferred)") { |v| options[:gemfile] = File.expand_path(v) }
82
+ o.on("--timeout SECONDS", Integer, "Per-gem build timeout (default 300)") { |v| options[:timeout] = v }
83
+ o.on("--max-builds N", Integer, "Cap the number of on-the-spot builds") { |v| options[:max_builds] = v }
84
+ o.on("--data PATH", "verified_gems.json path (default: bundled)") { |v| options[:data] = File.expand_path(v) }
85
+ o.on("-h", "--help", "Show this help") { @out.puts(o); options[:help] = true }
86
+ end
87
+ parser.parse(argv)
88
+ @verified ||= VerifiedGems.load(options[:data]) if options[:data]
89
+ options
90
+ end
91
+
92
+ # Walk every gem and resolve its status, honouring the build budget.
93
+ def evaluate(result, verified, options)
94
+ builder = @builder || Builder.new(cache_dir: build_cache, timeout: options[:timeout], out: @out)
95
+ builds_done = 0
96
+
97
+ result.entries.map do |entry|
98
+ if entry.source != :gem
99
+ next Row.new(name: entry.name, version: entry.version, status: :local)
100
+ end
101
+
102
+ # A gem may be verified in more than one environment, so name every
103
+ # environment that actually covers this version (and the newest date
104
+ # among them) instead of a single entry-level pair. A non-empty result
105
+ # is exactly what #match calls verified.
106
+ hits = verified.matching_verifications(entry.name, entry.version)
107
+ unless hits.empty?
108
+ note = "verified #{hits.map(&:verified_at).max} (#{hits.map(&:environment).join('; ')})"
109
+ next Row.new(name: entry.name, version: entry.version, status: :verified, note: note)
110
+ end
111
+
112
+ if options[:max_builds] && builds_done >= options[:max_builds]
113
+ next Row.new(name: entry.name, version: entry.version, status: :skipped_budget)
114
+ end
115
+
116
+ @out.puts(" building #{entry.name} #{entry.version}...") if @out
117
+ bresult = builder.build(entry)
118
+ builds_done += 1 unless bresult.status == :no_ext
119
+ Row.new(name: entry.name, version: entry.version, status: bresult.status, note: note_for(bresult))
120
+ end
121
+ end
122
+
123
+ # The note column for a build result.
124
+ def note_for(bresult)
125
+ case bresult.status
126
+ when :failed then "#{bresult.stage}: #{bresult.reason}"
127
+ when :built then bresult.require_ok ? "require OK" : bresult.reason
128
+ when :unknown then bresult.reason
129
+ end
130
+ end
131
+
132
+ def report(result, rows)
133
+ if result.origin == :gemfile
134
+ @out.puts("Note: no Gemfile.lock found; parsed #{result.path} for direct gems only " \
135
+ "(versions may be missing, dependency closure not resolved).")
136
+ @out.puts
137
+ end
138
+
139
+ width = rows.map { |r| "#{r.name} #{r.version}".length }.max || 0
140
+ rows.each do |row|
141
+ symbol, label = STATUS[row.status]
142
+ line = format(" %s %-#{width}s %s", symbol, "#{row.name} #{row.version}".strip, label)
143
+ line += " (#{row.note})" if row.note && !row.note.empty?
144
+ @out.puts(line)
145
+ end
146
+
147
+ summary(rows)
148
+ end
149
+
150
+ # The adoption verdict: over the gems that actually have C extensions, are
151
+ # they all handled (verified or built)?
152
+ def summary(rows)
153
+ cext = rows.reject { |r| %i[no_ext local].include?(r.status) }
154
+ failed = cext.count { |r| r.status == :failed }
155
+ # A gem skipped for the build budget was never checked, so it is
156
+ # undetermined for adoption purposes, exactly like an offline "unknown".
157
+ unknown = cext.count { |r| %i[unknown skipped_budget].include?(r.status) }
158
+ ok = cext.count { |r| %i[verified built].include?(r.status) }
159
+
160
+ @out.puts
161
+ @out.puts("C-extension gems: #{cext.size} (verified/built #{ok}, failed #{failed}, unknown #{unknown})")
162
+ @out.puts(
163
+ if cext.empty?
164
+ "Verdict: no C-extension gems to build — rubycc is not required."
165
+ elsif failed.zero? && unknown.zero?
166
+ "Verdict: ADOPTABLE — every C-extension gem is verified or builds with rubycc."
167
+ else
168
+ parts = []
169
+ parts << "#{failed} to address" if failed.positive?
170
+ parts << "#{unknown} undetermined" if unknown.positive?
171
+ "Verdict: NEEDS ATTENTION — #{parts.join(', ')}."
172
+ end
173
+ )
174
+ end
175
+
176
+ # Exit 0 only when every C-extension gem is verified or built.
177
+ def exit_code(rows)
178
+ cext = rows.reject { |r| %i[no_ext local].include?(r.status) }
179
+ cext.all? { |r| %i[verified built].include?(r.status) } ? 0 : 1
180
+ end
181
+
182
+ def build_cache
183
+ require "tmpdir"
184
+ dir = File.join(Dir.tmpdir, "rubycc-doctor")
185
+ FileUtils.mkdir_p(dir)
186
+ dir
187
+ end
188
+ end
189
+ end
190
+ end