ocran 1.4.6 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.txt +68 -0
  3. data/README.md +295 -10
  4. data/ext/extconf.rb +5 -1
  5. data/lib/ocran/aot_toolchain.rb +154 -0
  6. data/lib/ocran/command_output.rb +8 -1
  7. data/lib/ocran/compatibility_scan.rb +414 -0
  8. data/lib/ocran/cosmo_toolchain.rb +29 -7
  9. data/lib/ocran/dir_builder.rb +88 -38
  10. data/lib/ocran/direction.rb +190 -30
  11. data/lib/ocran/ed_resource.rb +113 -0
  12. data/lib/ocran/ed_resource_builder.rb +199 -0
  13. data/lib/ocran/file_path_set.rb +9 -5
  14. data/lib/ocran/gem_spec_queryable.rb +7 -3
  15. data/lib/ocran/host_config_helper.rb +11 -1
  16. data/lib/ocran/inno_setup_script_builder.rb +24 -14
  17. data/lib/ocran/launcher_batch_builder.rb +8 -11
  18. data/lib/ocran/npm_package.rb +106 -0
  19. data/lib/ocran/option.rb +425 -35
  20. data/lib/ocran/pe_imports.rb +96 -0
  21. data/lib/ocran/pe_resources.rb +226 -0
  22. data/lib/ocran/refine_pathname.rb +29 -32
  23. data/lib/ocran/roundhouse_builder.rb +316 -0
  24. data/lib/ocran/runner.rb +43 -11
  25. data/lib/ocran/runtime_sdk.rb +252 -0
  26. data/lib/ocran/spinel_builder.rb +183 -0
  27. data/lib/ocran/spinel_compatibility.rb +149 -0
  28. data/lib/ocran/static_runtime/boot.rb +443 -0
  29. data/lib/ocran/static_runtime/build_static_ext.rb +86 -0
  30. data/lib/ocran/static_runtime_builder.rb +814 -0
  31. data/lib/ocran/stub_builder.rb +70 -25
  32. data/lib/ocran/version.rb +1 -1
  33. data/lib/ocran/wasm_builder.rb +570 -0
  34. data/lib/ocran/wasm_compatibility.rb +198 -0
  35. data/lib/ocran/windows_command_escaping.rb +21 -0
  36. data/lib/ocran/zip_payload_builder.rb +18 -9
  37. data/lib/ocran/zip_writer.rb +48 -9
  38. data/src/Makefile +18 -2
  39. data/src/Makefile.mswin +65 -0
  40. data/src/inst_dir.c +34 -6
  41. data/src/path_utils.c +144 -0
  42. data/src/script_info.c +5 -0
  43. data/src/stub.c +77 -19
  44. data/src/stub.rc +1 -1
  45. data/src/system_utils.c +276 -155
  46. data/src/system_utils.h +46 -3
  47. data/src/system_utils_posix.c +243 -148
  48. data/src/unpack.c +9 -0
  49. metadata +22 -4
@@ -0,0 +1,316 @@
1
+ # frozen_string_literal: true
2
+ require "fileutils"
3
+ require "open3"
4
+ require "pathname"
5
+ require "tmpdir"
6
+ require_relative "command_output" unless defined?(Ocran::CommandOutput)
7
+ require_relative "aot_toolchain"
8
+
9
+ module Ocran
10
+ # Builds --roundhouse output: a Rails application compiled to a native
11
+ # server binary by Roundhouse (https://github.com/rubys/roundhouse) and
12
+ # Spinel (https://github.com/matz/spinel).
13
+ #
14
+ # Roundhouse lowers the app to the subset of Ruby Spinel compiles and
15
+ # writes it out as a spin project; its `make assets` builds static/ and
16
+ # `spin build` compiles the rest. The binary
17
+ # reads its static assets, configuration and SQLite database from beside
18
+ # it, so the result is a directory: the binary plus those files.
19
+ #
20
+ # Whether an app gets through depends on how much of it Roundhouse covers
21
+ # yet. When it does not - or when the tools are missing - the user gets
22
+ # Roundhouse's own analysis of the app (`roundhouse check`), whose gem
23
+ # census says which of the app's gems it does not model.
24
+ class RoundhouseBuilder
25
+ include CommandOutput
26
+
27
+ # What the binary reads at run time, copied from the generated project.
28
+ RUNTIME_DIRS = %w[static public db config].freeze
29
+
30
+ MAX_OUTPUT_LINES = 60
31
+ MAX_CHECK_ERRORS = 15
32
+
33
+ # Keeps the child tools out of whatever bundle OCRAN was started under:
34
+ # Roundhouse's spin project may shell out to Ruby (asset steps, tests).
35
+ BUNDLER_FREE_ENV = {
36
+ "RUBYOPT" => nil,
37
+ "BUNDLER_SETUP" => nil,
38
+ "BUNDLE_GEMFILE" => nil,
39
+ "BUNDLE_LOCKFILE" => nil
40
+ }.freeze
41
+
42
+ def initialize(option)
43
+ @option = option
44
+ @app = option.roundhouse_app
45
+ @output = option.roundhouse_output
46
+ end
47
+
48
+ def build
49
+ tools = AotToolchain.find_all(%i[roundhouse spinel spin])
50
+ missing = tools.select { |_, path| path.nil? }.keys
51
+ report_missing_tools(tools, missing) unless missing.empty?
52
+
53
+ tools.each do |name, path|
54
+ verbose "Using #{name} at #{path} (#{AotToolchain.version(path) || "version unknown"})"
55
+ end
56
+
57
+ work = Dir.mktmpdir("ocran-roundhouse")
58
+ project = File.join(work, "spinel")
59
+ @env = child_env(tools)
60
+
61
+ say "Transpiling #{display(@app)} with Roundhouse"
62
+ command = [tools[:roundhouse], "--target", "spinel", *@option.roundhouse_options, "-o", project, @app.to_s]
63
+ output, status = run(command)
64
+ transpile_failed(tools, output, status, work) unless status.success?
65
+
66
+ build_assets(project, work)
67
+
68
+ say "Compiling with Spinel (spin build); the first build of a large app can take a few minutes"
69
+ output, status = run([tools[:spin], "build"], chdir: project, env: library_path_env(project))
70
+ compile_failed(tools, output, status, work) unless status.success?
71
+
72
+ install(project)
73
+ FileUtils.remove_entry(work)
74
+ end
75
+
76
+ private
77
+
78
+ def run(command, chdir: Dir.pwd, env: {})
79
+ verbose command.join(" ")
80
+ Open3.capture2e(@env.merge(env), *command, chdir: chdir)
81
+ rescue SystemCallError => e
82
+ [e.message, nil]
83
+ end
84
+
85
+ # The generated project builds its static assets with `make assets`:
86
+ # Turbo and Stimulus copied out of their gems, the app's own JavaScript
87
+ # and stylesheets, and Tailwind when the app uses it (through npx).
88
+ # Bundler installs only the Gemfile's assets group for it, so no
89
+ # C-extension gem is compiled for a step that only copies files.
90
+ # Without static/ the binary still serves pages, with every stylesheet
91
+ # and script a 404, so a failure here stops the build.
92
+ def build_assets(project, work)
93
+ makefile = File.join(project, "Makefile")
94
+ return unless File.file?(makefile) && File.foreach(makefile).any? { |l| l.start_with?("assets:") }
95
+
96
+ retry_command = "cd #{File.join(work, "spinel")} && BUNDLE_ONLY=assets bundle install && BUNDLE_ONLY=assets make assets"
97
+ make = AotToolchain.search_path("make", @env)
98
+ bundle = AotToolchain.search_path("bundle", @env)
99
+ unless make && bundle
100
+ error "Building the static assets needs `make` and `bundle`, which #{make ? "`bundle` was" : bundle ? "`make` was" : "were"} not found."
101
+ STDERR.puts " The generated project is kept; with both installed: #{retry_command}"
102
+ raise "the static assets could not be built"
103
+ end
104
+
105
+ say "Building static assets (make assets)"
106
+ env = { "BUNDLE_ONLY" => "assets" }
107
+ [[bundle, "install"], [make, "assets"]].each do |command|
108
+ output, status = run(command, chdir: project, env: env)
109
+ next if status&.success?
110
+
111
+ print_failure("Building the static assets failed (#{command.map { |c| File.basename(c) }.join(" ")})", output, status)
112
+ STDERR.puts "Things to check:"
113
+ STDERR.puts " - an app that builds Tailwind needs Node.js and npm for `npx @tailwindcss/cli`"
114
+ STDERR.puts " - the generated project is kept; to retry: #{retry_command}"
115
+ raise "the static assets could not be built"
116
+ end
117
+ end
118
+
119
+ # spin links the allocator spin.toml names by bare name (-ljemalloc), and
120
+ # the linker searches only its default directories, which on Apple
121
+ # Silicon do not include Homebrew's /opt/homebrew/lib. pkg-config knows
122
+ # where the library is, so its -L directories go on LIBRARY_PATH (which
123
+ # gcc and clang both read) for the build.
124
+ def library_path_env(project)
125
+ manifest = File.join(project, "spin.toml")
126
+ allocator = File.file?(manifest) && File.read(manifest)[/^\s*allocator\s*=\s*"([^"]+)"/, 1]
127
+ return {} unless allocator && allocator != "system"
128
+
129
+ pkg_config = AotToolchain.search_path("pkg-config", @env)
130
+ return {} unless pkg_config
131
+
132
+ output, status = Open3.capture2(pkg_config, "--libs-only-L", allocator, err: File::NULL)
133
+ return {} unless status.success?
134
+
135
+ dirs = output.split.select { |flag| flag.start_with?("-L") }.map { |flag| flag.delete_prefix("-L") }
136
+ return {} if dirs.empty?
137
+
138
+ verbose "Adding #{dirs.join(", ")} (pkg-config #{allocator}) to LIBRARY_PATH"
139
+ { "LIBRARY_PATH" => (dirs + [ENV.fetch("LIBRARY_PATH", "")]).reject(&:empty?).join(File::PATH_SEPARATOR) }
140
+ rescue SystemCallError
141
+ {}
142
+ end
143
+
144
+ # spin calls spinel by name, so a spinel found outside PATH (through
145
+ # SPINEL or a conventional location) is put on the child's PATH.
146
+ def child_env(tools)
147
+ dirs = tools.values.compact.map { |path| File.dirname(path) }.uniq
148
+ BUNDLER_FREE_ENV.merge("PATH" => (dirs + [ENV.fetch("PATH", "")]).join(File::PATH_SEPARATOR))
149
+ end
150
+
151
+ def report_missing_tools(tools, missing)
152
+ commands = missing.map { |name| "`#{AotToolchain::TOOLS[name].command}`" }
153
+ list = commands.size > 1 ? "#{commands[0..-2].join(", ")} and #{commands[-1]}" : commands.first
154
+ error "--roundhouse needs #{list}, which #{missing.size == 1 ? "was" : "were"} not found."
155
+ STDERR.puts
156
+ STDERR.puts AotToolchain.install_instructions(missing)
157
+ STDERR.puts
158
+
159
+ if tools[:roundhouse]
160
+ @env = child_env(tools)
161
+ print_check(tools[:roundhouse], "Meanwhile, Roundhouse's analysis of #{display(@app)}:")
162
+ else
163
+ STDERR.puts "Once Roundhouse is installed, `roundhouse check --continue #{display(@app)}` lists the gems"
164
+ STDERR.puts "and constructs of the app it does not cover yet; OCRAN shows it when a build fails."
165
+ end
166
+ raise "the Roundhouse toolchain is not installed"
167
+ end
168
+
169
+ def transpile_failed(tools, output, status, work)
170
+ print_failure("Roundhouse could not transpile #{display(@app)}", output, status)
171
+ print_check(tools[:roundhouse], "Roundhouse's analysis of the app:")
172
+ STDERR.puts
173
+ STDERR.puts "Things to try:"
174
+ STDERR.puts " - what Roundhouse covers: #{AotToolchain::ROUNDHOUSE_URL}/blob/main/docs/guide/rails-coverage.md"
175
+ STDERR.puts " - --roundhouse-opt --survey --roundhouse-opt --allow-unsupported transpiles an app that is"
176
+ STDERR.puts " not fully covered yet, with a stub at each unsupported site"
177
+ STDERR.puts " - the generated project so far is in #{work}"
178
+ raise "Roundhouse could not transpile #{display(@app)}"
179
+ end
180
+
181
+ def compile_failed(tools, output, status, work)
182
+ print_failure("Spinel could not compile the transpiled app (spin build)", output, status)
183
+
184
+ STDERR.puts "Things to check:"
185
+ versions = tools.map { |name, path| "#{name}: #{AotToolchain.version(path) || "version unknown"}" }
186
+ STDERR.puts " - versions in use (#{versions.join("; ")}); each Roundhouse snapshot names the Spinel"
187
+ STDERR.puts " release it was tested against in #{AotToolchain::ROUNDHOUSE_URL}/blob/main/RELEASES.md,"
188
+ STDERR.puts " and a mismatch is the most common cause of a failing build"
189
+ missing_libs.each do |lib, package|
190
+ STDERR.puts " - the #{lib} development headers seem to be missing (#{package})"
191
+ end
192
+ unlinked_libs(output).each do |lib|
193
+ STDERR.puts " - the linker could not find lib#{lib}: install it (Debian/Ubuntu: lib#{lib}-dev; macOS: brew install #{lib});"
194
+ STDERR.puts " if it is installed outside the linker's default directories, set LIBRARY_PATH to its lib directory"
195
+ end
196
+ STDERR.puts " - the generated project is kept in #{File.join(work, "spinel")}; run `spin build` there to retry"
197
+ STDERR.puts " - a compiler error in the generated code is a bug in Roundhouse or Spinel: please report it"
198
+ raise "Spinel could not compile the transpiled app"
199
+ end
200
+
201
+ def print_failure(heading, output, status)
202
+ lines = output.to_s.lines
203
+ exit_status = status ? "exit status #{status.exitstatus.inspect}" : "could not be started"
204
+ STDERR.puts "#{heading} (#{exit_status}):"
205
+ STDERR.puts " ... (#{lines.size - MAX_OUTPUT_LINES} earlier lines omitted)" if lines.size > MAX_OUTPUT_LINES
206
+ STDERR.puts lines.last(MAX_OUTPUT_LINES).map { |l| " #{l}" }.join
207
+ STDERR.puts
208
+ end
209
+
210
+ # Prints the useful part of `roundhouse check --continue`: the gem
211
+ # census (which gems Roundhouse does not model), the summary, the first
212
+ # errors and the punch list of unsupported constructs. The full report
213
+ # of a large app runs to thousands of lines.
214
+ def print_check(roundhouse, heading)
215
+ say "Analyzing #{display(@app)} with roundhouse check"
216
+ output, status = run([roundhouse, "check", "--continue", @app.to_s])
217
+ return STDERR.puts("(roundhouse check could not be run: #{output.to_s.strip})") unless status
218
+
219
+ lines = output.lines.map(&:chomp)
220
+ summary = lines.grep(/\Aroundhouse-check:/)
221
+ errors = lines.grep(/: error\[/)
222
+ survey_start = lines.index { |l| l.include?("Survey:") }
223
+ survey = survey_start ? lines[survey_start..].take_while { |l| !l.start_with?("roundhouse-check:") }.reject(&:empty?) : []
224
+
225
+ STDERR.puts heading
226
+ STDERR.puts
227
+ summary.each { |l| STDERR.puts " #{l}" }
228
+ if summary.any? { |l| l.match?(/\d+ unknown/) && !l.match?(/\b0 unknown/) }
229
+ STDERR.puts " (\"unknown\" gems are ones Roundhouse does not model: code using them cannot be compiled yet)"
230
+ end
231
+ unless errors.empty?
232
+ STDERR.puts
233
+ errors.first(MAX_CHECK_ERRORS).each { |l| STDERR.puts " #{l}" }
234
+ STDERR.puts " ... and #{errors.size - MAX_CHECK_ERRORS} more error(s)" if errors.size > MAX_CHECK_ERRORS
235
+ end
236
+ unless survey.empty?
237
+ STDERR.puts
238
+ survey.first(30).each { |l| STDERR.puts " #{l}" }
239
+ end
240
+ STDERR.puts
241
+ STDERR.puts " Full report: roundhouse check --continue #{display(@app)}"
242
+ end
243
+
244
+ # Libraries the generated project links that pkg-config cannot find,
245
+ # as [library, package hint] pairs. Empty when pkg-config is not there
246
+ # to ask.
247
+ def missing_libs
248
+ return [] unless AotToolchain.search_path("pkg-config")
249
+
250
+ { "sqlite3" => "libsqlite3-dev / brew install sqlite",
251
+ "jemalloc" => "libjemalloc-dev / brew install jemalloc" }.reject do |lib, _|
252
+ Kernel.system("pkg-config", "--exists", lib)
253
+ end.to_a
254
+ end
255
+
256
+ # Libraries the link step could not find, from GNU ld's
257
+ # "cannot find -lfoo" and Apple ld's "library 'foo' not found".
258
+ def unlinked_libs(output)
259
+ output.to_s.scan(/cannot find -l([\w.+-]+)|library '([\w.+-]+)' not found/).flatten.compact.uniq
260
+ end
261
+
262
+ def install(project)
263
+ bin_dir = File.join(project, "build", "bin")
264
+ binaries = Dir.glob(File.join(bin_dir, "*")).select { |f| File.file?(f) && File.executable?(f) }
265
+ binary = binaries.find { |f| File.basename(f, ".*") == @app.basename.to_s } || binaries.first
266
+ raise "spin build succeeded but produced no executable in #{bin_dir}" unless binary
267
+
268
+ @output.mkpath
269
+ target = @output + File.basename(binary)
270
+ FileUtils.cp(binary, target)
271
+ File.chmod(0o755, target)
272
+
273
+ RUNTIME_DIRS.each do |dir|
274
+ source = File.join(project, dir)
275
+ next unless File.directory?(source)
276
+
277
+ FileUtils.rm_rf(@output + dir)
278
+ FileUtils.cp_r(source, @output + dir)
279
+ end
280
+ # storage/ holds the database and uploads, so a rebuild keeps it.
281
+ (@output + "storage").mkpath
282
+
283
+ seed_database
284
+ say "Finished building #{display(@output)}"
285
+ say "Run it with: cd #{display(@output)} && ./#{File.basename(binary)} (serves on port 3000; PORT or -p to change)"
286
+ say "It needs libsqlite3 and libjemalloc at run time (Debian/Ubuntu: libsqlite3-0 libjemalloc2)."
287
+ end
288
+
289
+ # Creates the database from the generated seed on the first build, when
290
+ # the sqlite3 command is there to do it.
291
+ def seed_database
292
+ seed = @output + "db" + "seed.sql"
293
+ database = @output + "storage" + "development.sqlite3"
294
+ return unless seed.file? && !database.exist?
295
+
296
+ sqlite3 = AotToolchain.search_path("sqlite3")
297
+ unless sqlite3
298
+ say "Create the database before the first start: sqlite3 storage/development.sqlite3 < db/seed.sql"
299
+ return
300
+ end
301
+
302
+ verbose "Seeding #{database} from #{seed}"
303
+ unless Kernel.system(sqlite3, database.to_s, in: seed.to_s)
304
+ warning "Could not create the database from #{seed}; run: sqlite3 storage/development.sqlite3 < db/seed.sql"
305
+ end
306
+ end
307
+
308
+ # Relative to the working directory when inside it, else absolute.
309
+ def display(path)
310
+ rel = Pathname(path).relative_path_from(Pathname.pwd).to_s
311
+ rel.start_with?("..") ? path.to_s : rel
312
+ rescue ArgumentError
313
+ path.to_s
314
+ end
315
+ end
316
+ end
data/lib/ocran/runner.rb CHANGED
@@ -6,8 +6,19 @@ module Ocran
6
6
  load File.expand_path("command_output.rb", __dir__)
7
7
  include CommandOutput
8
8
 
9
- def fatal_error(statement)
10
- error statement
9
+ # Errors that describe a problem with the input or the build host - an
10
+ # invalid option, a missing file, a path OCRAN cannot write, a value too
11
+ # large for the executable format - rather than a bug in OCRAN. They
12
+ # are reported by their message; a backtrace would only bury it. Any
13
+ # other exception is a bug and keeps Ruby's full report.
14
+ USER_ERRORS = [RuntimeError, ArgumentError, SystemCallError, IOError].freeze
15
+
16
+ # Reports a USER_ERRORS exception and exits. With --verbose the
17
+ # backtrace is shown too, for when the message is not enough.
18
+ def user_error(e)
19
+ message = e.instance_of?(RuntimeError) ? e.message : "#{e.message} (#{e.class})"
20
+ error message
21
+ STDERR.puts e.backtrace.map { |line| "\tfrom #{line}" } if @option&.verbose? && e.backtrace
11
22
  exit false
12
23
  end
13
24
 
@@ -18,11 +29,9 @@ module Ocran
18
29
  load File.expand_path("option.rb", __dir__)
19
30
  @option = Option.new.tap do |opt|
20
31
  opt.parse(ARGV)
21
- rescue RuntimeError => e
22
- # Capture RuntimeError during parsing and display an appropriate
23
- # error message to the user. This error usually occurs from invalid
24
- # option arguments.
25
- fatal_error e.message
32
+ rescue *USER_ERRORS => e
33
+ # Invalid option arguments, usually; report them as such.
34
+ user_error e
26
35
  else
27
36
  # Update ARGV with the parsed command line arguments to pass to
28
37
  # the user's script. This ensures the script executes based on
@@ -31,6 +40,7 @@ module Ocran
31
40
  end
32
41
 
33
42
  Ocran.option = @option
43
+ @option.warnings.each { |message| warning message }
34
44
 
35
45
  @ignore_modules = ObjectSpace.each_object(Module).to_a
36
46
  end
@@ -206,6 +216,24 @@ module Ocran
206
216
  end
207
217
 
208
218
  def build
219
+ # Native compilation hands the source to an ahead-of-time compiler, and
220
+ # --wasm to a WebAssembly runtime; nothing was loaded, and none of the
221
+ # packaging below applies.
222
+ if @option.spinel? || @option.roundhouse? || @option.wasm_runtime
223
+ Dir.chdir(@pre_env.pwd)
224
+ if @option.wasm_runtime
225
+ require_relative "wasm_builder"
226
+ WasmBuilder.new(@option).build
227
+ elsif @option.roundhouse?
228
+ require_relative "roundhouse_builder"
229
+ RoundhouseBuilder.new(@option).build
230
+ else
231
+ require_relative "spinel_builder"
232
+ SpinelBuilder.new(@option).build
233
+ end
234
+ return
235
+ end
236
+
209
237
  # If the script was run and autoload is enabled, attempt to autoload libraries.
210
238
  if @option.force_autoload?
211
239
  attempt_load_autoload(@ignore_modules)
@@ -226,8 +254,10 @@ module Ocran
226
254
 
227
255
  if @option.use_inno_setup?
228
256
  # Native on Windows; on POSIX allow it when an ISCC command is
229
- # available (e.g. via Wine wrapper scripts or in tests).
230
- if Gem.win_platform? || system("command -v ISCC > /dev/null 2>&1")
257
+ # available (e.g. via Wine wrapper scripts or in tests). Checked
258
+ # here, before the files are collected, so a missing ISCC fails fast.
259
+ require_relative "inno_setup_script_builder"
260
+ if InnoSetupScriptBuilder.iscc_available?
231
261
  direction.build_inno_setup_installer
232
262
  else
233
263
  raise "Inno Setup is only supported on Windows (no ISCC command found in PATH)"
@@ -238,13 +268,15 @@ module Ocran
238
268
  direction.build_output_dir(@option.output_dir)
239
269
  elsif @option.output_zip
240
270
  direction.build_zip(@option.output_zip)
271
+ elsif @option.static_runtime?
272
+ direction.build_static_runtime_exe
241
273
  elsif @option.cosmo_zip?
242
274
  direction.build_cosmo_zip_exe
243
275
  else
244
276
  direction.build_stab_exe
245
277
  end
246
- rescue RuntimeError => e
247
- fatal_error e.message
278
+ rescue *USER_ERRORS => e
279
+ user_error e
248
280
  end
249
281
  end
250
282
  end
@@ -0,0 +1,252 @@
1
+ # frozen_string_literal: true
2
+ require "pathname"
3
+ require "fileutils"
4
+ require "open3"
5
+ require "rbconfig"
6
+ require "tmpdir"
7
+ require "json"
8
+ require_relative "pe_imports"
9
+
10
+ module Ocran
11
+ # The SDK part of a static runtime (<runtime>/sdk, written by
12
+ # runtime/make-sdk.rb): what is needed to link the runtime again with more
13
+ # C extensions statically linked in - its link inputs and link command
14
+ # (link.args), Ruby's headers, the runtime's rbconfig.rb with the
15
+ # <arch>-fake.rb Ruby's cross builds use to describe the target, and the
16
+ # gem extensions already linked in (exts/*.line).
17
+ #
18
+ # Linking needs a compiler for the runtime's target on the build host. For
19
+ # a MinGW runtime (toolchain "gcc" in sdk.json): the one link.args names
20
+ # (e.g. x86_64-w64-mingw32ucrt-gcc on Linux), or OCRAN_RUNTIME_CC (e.g. gcc
21
+ # from RubyInstaller's MSYS2 on Windows). For an mswin runtime ("msvc"):
22
+ # cl and nmake of Visual Studio (Build Tools), from a developer prompt.
23
+ class RuntimeSdk
24
+ BUILD_STATIC_EXT = File.expand_path("static_runtime/build_static_ext.rb", __dir__)
25
+
26
+ # One statically built extension: the feature it provides ("bcrypt_ext"),
27
+ # its Init function, the static library and the libraries it links with.
28
+ Extension = Struct.new(:feature, :init, :archive, :libs) do
29
+ def self.parse(line, base)
30
+ feature, init, archive, libs = line.strip.split(" ", 4)
31
+ raise "bad extension line #{line.inspect}" unless feature&.match?(%r{\A[\w/.-]+\z}) && init&.match?(/\A\w+\z/)
32
+
33
+ new(feature, init, File.expand_path(archive.sub("@SDK@", base.to_s)), libs.to_s.split(","))
34
+ end
35
+ end
36
+
37
+ attr_reader :dir, :arch, :toolchain
38
+
39
+ def initialize(dir, arch:)
40
+ @dir = Pathname(dir)
41
+ @arch = arch
42
+ raise "#{@dir} is not a static runtime SDK (no link.args)" unless (@dir / "link.args").file?
43
+
44
+ @info = (@dir / "sdk.json").file? ? JSON.parse((@dir / "sdk.json").read) : {}
45
+ @toolchain = @info.fetch("toolchain", "gcc")
46
+ end
47
+
48
+ # Libraries available as DLLs instead of linked in (--runtime-dll):
49
+ # name ("sybdb") => { "dll" => ..., "import" => ... }, relative to the SDK.
50
+ def dlls = @info.fetch("dlls", {})
51
+
52
+ # extconf.rb options for gems whose C library the SDK provides, so that
53
+ # extconf.rb uses it instead of downloading and building its own.
54
+ GEM_EXTCONF_ARGS = {
55
+ "nokogiri" => ["--use-system-libraries"],
56
+ "tiny_tds" => ["--with-freetds-dir=@SDK_DEPS@"],
57
+ }.freeze
58
+
59
+ def extconf_args(gem_name)
60
+ (GEM_EXTCONF_ARGS[gem_name] || []).map { |a| a.gsub("@SDK_DEPS@", (@dir / "deps").to_s) }
61
+ end
62
+
63
+ def msvc? = @toolchain == "msvc"
64
+
65
+ # True when the application archive cannot be appended to the
66
+ # executable but is linked in as the section __OCRAN,__zip (macOS, where
67
+ # appended data would break the code signature).
68
+ def section_payload? = @info["payload"] == "section"
69
+
70
+ def link_args = (@dir / "link.args").read.lines(chomp: true).map { |a| a.gsub("@SDK@", @dir.to_s) }
71
+
72
+ # The SDK's header directories, ahead of whatever an extconf.rb adds:
73
+ # the SDK's libraries are the ones linked, so their headers must win
74
+ # over a system copy (nokogiri adds the libxml2 headers of Apple's SDK,
75
+ # an older libxml2, on macOS).
76
+ def sdk_include_flags
77
+ deps = @dir / "deps" / "include"
78
+ [deps, deps / "libxml2"].select(&:directory?).map { |dir| "-I#{dir}" }
79
+ end
80
+
81
+ # The C compiler driver of the target.
82
+ def cc
83
+ return "cl" if msvc?
84
+
85
+ env = ENV["OCRAN_RUNTIME_CC"]
86
+ env && !env.empty? ? env : link_args.first
87
+ end
88
+
89
+ # The gem extensions already linked into the runtime.
90
+ def base_extensions
91
+ Dir[(@dir / "exts/*.line").to_s].sort.map { |f| Extension.parse(File.read(f), @dir) }
92
+ end
93
+
94
+ # Fails early, with a hint, when the build host has no compiler for the
95
+ # runtime's target.
96
+ def check_toolchain!
97
+ return check_msvc! if msvc?
98
+
99
+ out, status = Open3.capture2e(cc, "-dumpmachine")
100
+ expected = @info["dumpmachine"]
101
+ return if status.success? && (expected.nil? || out.include?(expected))
102
+
103
+ raise "#{cc} #{status.success? ? "does not target #{expected} (#{out.strip})" : "did not run"}; " \
104
+ "set OCRAN_RUNTIME_CC to #{toolchain_hint}"
105
+ rescue SystemCallError
106
+ raise "#{cc} not found; set OCRAN_RUNTIME_CC to #{toolchain_hint}"
107
+ end
108
+
109
+ def toolchain_hint
110
+ target_os = @info.fetch("target_os", "mingw32")
111
+ if target_os.start_with?("darwin")
112
+ return "clang from Xcode or the Command Line Tools (xcode-select --install)"
113
+ end
114
+ if target_os.start_with?("mingw")
115
+ "a MinGW-w64 UCRT C compiler (on Linux: gcc-mingw-w64-ucrt64, on Windows: RubyInstaller's " \
116
+ "MSYS2, e.g. ucrt64/bin/gcc.exe)"
117
+ else
118
+ "a musl C compiler (musl-gcc from musl-tools)"
119
+ end
120
+ end
121
+
122
+ def check_msvc!
123
+ %w[cl nmake].each do |tool|
124
+ Open3.capture2e(tool, "-?")
125
+ rescue SystemCallError
126
+ raise "#{tool} not found: building C extensions for an mswin runtime needs Visual Studio " \
127
+ "(Build Tools); run ocran from a \"x64 Native Tools\" developer prompt"
128
+ end
129
+ end
130
+
131
+ # Compiles the C extension in +ext_dir+ (a gem's ext/<name>/, with its
132
+ # extconf.rb) to a static library under +work_dir+. +gem_path+: more
133
+ # gem directories for the gems extconf.rb itself uses.
134
+ def build_extension(ext_dir, work_dir, *extconf_args, gem_path: [])
135
+ pkgconfig = File.join(work_dir.to_s + ".pkgconfig")
136
+ FileUtils.mkdir_p(pkgconfig)
137
+ Dir[(@dir / "deps/lib/pkgconfig/*.pc").to_s].each do |pc|
138
+ File.write(File.join(pkgconfig, File.basename(pc)), File.read(pc).gsub("@SDK_DEPS@", (@dir / "deps").to_s))
139
+ end
140
+ # The SDK's rbconfig.rb is found first on the load path; a MinGW SDK
141
+ # also has Ruby's <arch>-fake.rb, which makes any Ruby pose as the
142
+ # runtime's (an mswin build host runs the runtime's own revision).
143
+ fake = Dir[(@dir / "*-fake.rb").to_s].first
144
+ env = { "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil, "BUNDLER_SETUP" => nil,
145
+ "OCRAN_RUNTIME_CC" => msvc? ? nil : ENV["OCRAN_RUNTIME_CC"] || (cc == link_args.first ? nil : cc),
146
+ "OCRAN_RUNTIME_LIBS" => @info.fetch("libs", []).join(" "),
147
+ "OCRAN_RUNTIME_CPPFLAGS" => (sdk_include_flags + @info.fetch("cppflags", [])).join(" "),
148
+ "PKG_CONFIG_LIBDIR" => pkgconfig, "PKG_CONFIG_PATH" => "",
149
+ # Build-time gems extconf.rb uses (nokogiri: mini_portile2):
150
+ # the build host's, which the target's RbConfig would hide.
151
+ "GEM_HOME" => Gem.dir, "GEM_PATH" => (gem_path + Gem.path).join(File::PATH_SEPARATOR) }
152
+ cmd = [RbConfig.ruby, "--disable=gems", "-I", @dir.to_s, *(fake ? ["-r", fake] : []), BUILD_STATIC_EXT,
153
+ (@dir / "include").to_s, (@dir / "include" / @arch).to_s, ext_dir.to_s, work_dir.to_s, *extconf_args]
154
+ out, status = Open3.capture2e(env, *cmd)
155
+ unless status.success?
156
+ raise "Building the C extension in #{ext_dir} for the static runtime failed:\n#{out.lines.last(25).join}"
157
+ end
158
+
159
+ Extension.parse(out.lines.last, @dir)
160
+ end
161
+
162
+ # Links the runtime with the given extensions registered in addition to
163
+ # those it already has, into +output+. Libraries named in +dlls+ are
164
+ # linked against their DLL instead of statically (see #dll_files).
165
+ # macOS: the application archive is linked in as a section (see
166
+ # #section_payload?); +archive+ is that file.
167
+ def link(extensions, output, dlls: [], archive: nil)
168
+ unknown = dlls - self.dlls.keys
169
+ unless unknown.empty?
170
+ raise "No DLL build of #{unknown.join(", ")} in #{@dir} (available: #{self.dlls.keys.join(", ")})"
171
+ end
172
+ as_dll = dlls.to_h { |name| ["-l#{name}", (@dir / self.dlls[name]["import"]).to_s] }
173
+ all = base_extensions + extensions
174
+ Dir.mktmpdir do |tmp|
175
+ src = File.join(tmp, "ocran_exts.c")
176
+ File.write(src, exts_source(all))
177
+ obj = File.join(tmp, msvc? ? "ocran_exts.obj" : "ocran_exts.o")
178
+ if msvc?
179
+ run("cl", "-nologo", "-MT", "-O2", "-c", src, "-Fo#{obj}")
180
+ else
181
+ run(cc, "-O2", "-c", src, "-o", obj)
182
+ end
183
+
184
+ extra = [obj] + all.flat_map { |ext| [ext.archive, *ext.libs] } + @info.fetch("libs", [])
185
+ args = link_args
186
+ args[0] = cc
187
+ cmd = args.flat_map { |a| a == "@EXTRA@" ? extra : [a.gsub("@OUTPUT@", output.to_s)] }
188
+ cmd = cmd.map { |a| as_dll.fetch(a, a) }
189
+ cmd += ["-o", output.to_s] unless args.any? { |a| a.include?("@OUTPUT@") }
190
+ cmd += ["-Wl,-sectcreate,__OCRAN,__zip,#{archive}"] if archive
191
+ run(*cmd)
192
+ end
193
+ strip = strip_tool
194
+ run(strip, output.to_s) if strip
195
+ output
196
+ end
197
+
198
+ # The strip of the compiler in use: sdk.json names the SDK build host's
199
+ # (nil on macOS, where stripping would break the signature), which
200
+ # another compiler named with OCRAN_RUNTIME_CC may not come with (MSYS2
201
+ # has gcc and strip, not x86_64-w64-mingw32ucrt-strip).
202
+ def strip_tool
203
+ return nil if msvc? || (@info.key?("strip") && @info["strip"].nil?)
204
+
205
+ derived = cc.sub(/gcc(\.exe)?\z/) { "strip#{$1}" } if cc.match?(/gcc(\.exe)?\z/)
206
+ return derived if derived && cc != link_args.first
207
+
208
+ @info["strip"] || derived
209
+ end
210
+
211
+ # The DLL files to ship next to the executable for +dlls+: each named
212
+ # library's DLL and the SDK DLLs it imports in turn (libsybdb-5.dll
213
+ # needs libiconv-2.dll).
214
+ def dll_files(dlls)
215
+ by_file = self.dlls.values.to_h { |d| [File.basename(d["dll"]).downcase, @dir / d["dll"]] }
216
+ queue = dlls.map { |name| @dir / self.dlls.fetch(name)["dll"] }
217
+ found = {}
218
+ until queue.empty?
219
+ dll = queue.shift
220
+ next if found[dll.basename.to_s.downcase]
221
+
222
+ found[dll.basename.to_s.downcase] = dll
223
+ PEImports.dll_names(dll.to_s).each { |name| queue << by_file[name.downcase] if by_file[name.downcase] }
224
+ end
225
+ found.values
226
+ end
227
+
228
+ private
229
+
230
+ def exts_source(extensions)
231
+ <<~C
232
+ /* generated by ocran (Ocran::RuntimeSdk) */
233
+ void ruby_init_ext(const char *name, void (*init)(void));
234
+ #{extensions.map { |e| "extern void #{e.init}(void);" }.join("\n")}
235
+ void ocran_init_extra_exts(void)
236
+ {
237
+ #{extensions.map { |e| " ruby_init_ext(\"#{e.feature}.so\", #{e.init});" }.join("\n")}
238
+ }
239
+ C
240
+ end
241
+
242
+ # Compilers, linkers and strip run with SOURCE_DATE_EPOCH set: binutils
243
+ # then writes that, not the current time, into a PE header it rewrites
244
+ # (strip does), so that the same application links to the same bytes.
245
+ def run(*cmd)
246
+ out, status = Open3.capture2e({ "SOURCE_DATE_EPOCH" => ENV["SOURCE_DATE_EPOCH"] || "1767225600" }, *cmd)
247
+ raise "#{cmd.first} failed:\n#{out.lines.last(25).join}" unless status.success?
248
+
249
+ out
250
+ end
251
+ end
252
+ end