ocran 1.4.6 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.txt +68 -0
- data/README.md +295 -10
- data/ext/extconf.rb +5 -1
- data/lib/ocran/aot_toolchain.rb +154 -0
- data/lib/ocran/command_output.rb +8 -1
- data/lib/ocran/compatibility_scan.rb +414 -0
- data/lib/ocran/cosmo_toolchain.rb +29 -7
- data/lib/ocran/dir_builder.rb +88 -38
- data/lib/ocran/direction.rb +190 -30
- data/lib/ocran/ed_resource.rb +113 -0
- data/lib/ocran/ed_resource_builder.rb +199 -0
- data/lib/ocran/file_path_set.rb +9 -5
- data/lib/ocran/gem_spec_queryable.rb +7 -3
- data/lib/ocran/host_config_helper.rb +11 -1
- data/lib/ocran/inno_setup_script_builder.rb +24 -14
- data/lib/ocran/launcher_batch_builder.rb +8 -11
- data/lib/ocran/npm_package.rb +106 -0
- data/lib/ocran/option.rb +425 -35
- data/lib/ocran/pe_imports.rb +96 -0
- data/lib/ocran/pe_resources.rb +226 -0
- data/lib/ocran/refine_pathname.rb +29 -32
- data/lib/ocran/roundhouse_builder.rb +316 -0
- data/lib/ocran/runner.rb +43 -11
- data/lib/ocran/runtime_sdk.rb +252 -0
- data/lib/ocran/spinel_builder.rb +183 -0
- data/lib/ocran/spinel_compatibility.rb +149 -0
- data/lib/ocran/static_runtime/boot.rb +443 -0
- data/lib/ocran/static_runtime/build_static_ext.rb +86 -0
- data/lib/ocran/static_runtime_builder.rb +814 -0
- data/lib/ocran/stub_builder.rb +70 -25
- data/lib/ocran/version.rb +1 -1
- data/lib/ocran/wasm_builder.rb +570 -0
- data/lib/ocran/wasm_compatibility.rb +198 -0
- data/lib/ocran/windows_command_escaping.rb +21 -0
- data/lib/ocran/zip_payload_builder.rb +18 -9
- data/lib/ocran/zip_writer.rb +48 -9
- data/src/Makefile +18 -2
- data/src/Makefile.mswin +65 -0
- data/src/inst_dir.c +34 -6
- data/src/path_utils.c +144 -0
- data/src/script_info.c +5 -0
- data/src/stub.c +77 -19
- data/src/stub.rc +1 -1
- data/src/system_utils.c +276 -155
- data/src/system_utils.h +46 -3
- data/src/system_utils_posix.c +243 -148
- data/src/unpack.c +9 -0
- metadata +22 -4
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "fileutils"
|
|
3
|
+
require "open3"
|
|
4
|
+
require "pathname"
|
|
5
|
+
require "tmpdir"
|
|
6
|
+
require_relative "command_output" unless defined?(Ocran::CommandOutput)
|
|
7
|
+
require_relative "aot_toolchain"
|
|
8
|
+
|
|
9
|
+
module Ocran
|
|
10
|
+
# Builds --roundhouse output: a Rails application compiled to a native
|
|
11
|
+
# server binary by Roundhouse (https://github.com/rubys/roundhouse) and
|
|
12
|
+
# Spinel (https://github.com/matz/spinel).
|
|
13
|
+
#
|
|
14
|
+
# Roundhouse lowers the app to the subset of Ruby Spinel compiles and
|
|
15
|
+
# writes it out as a spin project; its `make assets` builds static/ and
|
|
16
|
+
# `spin build` compiles the rest. The binary
|
|
17
|
+
# reads its static assets, configuration and SQLite database from beside
|
|
18
|
+
# it, so the result is a directory: the binary plus those files.
|
|
19
|
+
#
|
|
20
|
+
# Whether an app gets through depends on how much of it Roundhouse covers
|
|
21
|
+
# yet. When it does not - or when the tools are missing - the user gets
|
|
22
|
+
# Roundhouse's own analysis of the app (`roundhouse check`), whose gem
|
|
23
|
+
# census says which of the app's gems it does not model.
|
|
24
|
+
class RoundhouseBuilder
|
|
25
|
+
include CommandOutput
|
|
26
|
+
|
|
27
|
+
# What the binary reads at run time, copied from the generated project.
|
|
28
|
+
RUNTIME_DIRS = %w[static public db config].freeze
|
|
29
|
+
|
|
30
|
+
MAX_OUTPUT_LINES = 60
|
|
31
|
+
MAX_CHECK_ERRORS = 15
|
|
32
|
+
|
|
33
|
+
# Keeps the child tools out of whatever bundle OCRAN was started under:
|
|
34
|
+
# Roundhouse's spin project may shell out to Ruby (asset steps, tests).
|
|
35
|
+
BUNDLER_FREE_ENV = {
|
|
36
|
+
"RUBYOPT" => nil,
|
|
37
|
+
"BUNDLER_SETUP" => nil,
|
|
38
|
+
"BUNDLE_GEMFILE" => nil,
|
|
39
|
+
"BUNDLE_LOCKFILE" => nil
|
|
40
|
+
}.freeze
|
|
41
|
+
|
|
42
|
+
def initialize(option)
|
|
43
|
+
@option = option
|
|
44
|
+
@app = option.roundhouse_app
|
|
45
|
+
@output = option.roundhouse_output
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def build
|
|
49
|
+
tools = AotToolchain.find_all(%i[roundhouse spinel spin])
|
|
50
|
+
missing = tools.select { |_, path| path.nil? }.keys
|
|
51
|
+
report_missing_tools(tools, missing) unless missing.empty?
|
|
52
|
+
|
|
53
|
+
tools.each do |name, path|
|
|
54
|
+
verbose "Using #{name} at #{path} (#{AotToolchain.version(path) || "version unknown"})"
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
work = Dir.mktmpdir("ocran-roundhouse")
|
|
58
|
+
project = File.join(work, "spinel")
|
|
59
|
+
@env = child_env(tools)
|
|
60
|
+
|
|
61
|
+
say "Transpiling #{display(@app)} with Roundhouse"
|
|
62
|
+
command = [tools[:roundhouse], "--target", "spinel", *@option.roundhouse_options, "-o", project, @app.to_s]
|
|
63
|
+
output, status = run(command)
|
|
64
|
+
transpile_failed(tools, output, status, work) unless status.success?
|
|
65
|
+
|
|
66
|
+
build_assets(project, work)
|
|
67
|
+
|
|
68
|
+
say "Compiling with Spinel (spin build); the first build of a large app can take a few minutes"
|
|
69
|
+
output, status = run([tools[:spin], "build"], chdir: project, env: library_path_env(project))
|
|
70
|
+
compile_failed(tools, output, status, work) unless status.success?
|
|
71
|
+
|
|
72
|
+
install(project)
|
|
73
|
+
FileUtils.remove_entry(work)
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
private
|
|
77
|
+
|
|
78
|
+
def run(command, chdir: Dir.pwd, env: {})
|
|
79
|
+
verbose command.join(" ")
|
|
80
|
+
Open3.capture2e(@env.merge(env), *command, chdir: chdir)
|
|
81
|
+
rescue SystemCallError => e
|
|
82
|
+
[e.message, nil]
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# The generated project builds its static assets with `make assets`:
|
|
86
|
+
# Turbo and Stimulus copied out of their gems, the app's own JavaScript
|
|
87
|
+
# and stylesheets, and Tailwind when the app uses it (through npx).
|
|
88
|
+
# Bundler installs only the Gemfile's assets group for it, so no
|
|
89
|
+
# C-extension gem is compiled for a step that only copies files.
|
|
90
|
+
# Without static/ the binary still serves pages, with every stylesheet
|
|
91
|
+
# and script a 404, so a failure here stops the build.
|
|
92
|
+
def build_assets(project, work)
|
|
93
|
+
makefile = File.join(project, "Makefile")
|
|
94
|
+
return unless File.file?(makefile) && File.foreach(makefile).any? { |l| l.start_with?("assets:") }
|
|
95
|
+
|
|
96
|
+
retry_command = "cd #{File.join(work, "spinel")} && BUNDLE_ONLY=assets bundle install && BUNDLE_ONLY=assets make assets"
|
|
97
|
+
make = AotToolchain.search_path("make", @env)
|
|
98
|
+
bundle = AotToolchain.search_path("bundle", @env)
|
|
99
|
+
unless make && bundle
|
|
100
|
+
error "Building the static assets needs `make` and `bundle`, which #{make ? "`bundle` was" : bundle ? "`make` was" : "were"} not found."
|
|
101
|
+
STDERR.puts " The generated project is kept; with both installed: #{retry_command}"
|
|
102
|
+
raise "the static assets could not be built"
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
say "Building static assets (make assets)"
|
|
106
|
+
env = { "BUNDLE_ONLY" => "assets" }
|
|
107
|
+
[[bundle, "install"], [make, "assets"]].each do |command|
|
|
108
|
+
output, status = run(command, chdir: project, env: env)
|
|
109
|
+
next if status&.success?
|
|
110
|
+
|
|
111
|
+
print_failure("Building the static assets failed (#{command.map { |c| File.basename(c) }.join(" ")})", output, status)
|
|
112
|
+
STDERR.puts "Things to check:"
|
|
113
|
+
STDERR.puts " - an app that builds Tailwind needs Node.js and npm for `npx @tailwindcss/cli`"
|
|
114
|
+
STDERR.puts " - the generated project is kept; to retry: #{retry_command}"
|
|
115
|
+
raise "the static assets could not be built"
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# spin links the allocator spin.toml names by bare name (-ljemalloc), and
|
|
120
|
+
# the linker searches only its default directories, which on Apple
|
|
121
|
+
# Silicon do not include Homebrew's /opt/homebrew/lib. pkg-config knows
|
|
122
|
+
# where the library is, so its -L directories go on LIBRARY_PATH (which
|
|
123
|
+
# gcc and clang both read) for the build.
|
|
124
|
+
def library_path_env(project)
|
|
125
|
+
manifest = File.join(project, "spin.toml")
|
|
126
|
+
allocator = File.file?(manifest) && File.read(manifest)[/^\s*allocator\s*=\s*"([^"]+)"/, 1]
|
|
127
|
+
return {} unless allocator && allocator != "system"
|
|
128
|
+
|
|
129
|
+
pkg_config = AotToolchain.search_path("pkg-config", @env)
|
|
130
|
+
return {} unless pkg_config
|
|
131
|
+
|
|
132
|
+
output, status = Open3.capture2(pkg_config, "--libs-only-L", allocator, err: File::NULL)
|
|
133
|
+
return {} unless status.success?
|
|
134
|
+
|
|
135
|
+
dirs = output.split.select { |flag| flag.start_with?("-L") }.map { |flag| flag.delete_prefix("-L") }
|
|
136
|
+
return {} if dirs.empty?
|
|
137
|
+
|
|
138
|
+
verbose "Adding #{dirs.join(", ")} (pkg-config #{allocator}) to LIBRARY_PATH"
|
|
139
|
+
{ "LIBRARY_PATH" => (dirs + [ENV.fetch("LIBRARY_PATH", "")]).reject(&:empty?).join(File::PATH_SEPARATOR) }
|
|
140
|
+
rescue SystemCallError
|
|
141
|
+
{}
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
# spin calls spinel by name, so a spinel found outside PATH (through
|
|
145
|
+
# SPINEL or a conventional location) is put on the child's PATH.
|
|
146
|
+
def child_env(tools)
|
|
147
|
+
dirs = tools.values.compact.map { |path| File.dirname(path) }.uniq
|
|
148
|
+
BUNDLER_FREE_ENV.merge("PATH" => (dirs + [ENV.fetch("PATH", "")]).join(File::PATH_SEPARATOR))
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
def report_missing_tools(tools, missing)
|
|
152
|
+
commands = missing.map { |name| "`#{AotToolchain::TOOLS[name].command}`" }
|
|
153
|
+
list = commands.size > 1 ? "#{commands[0..-2].join(", ")} and #{commands[-1]}" : commands.first
|
|
154
|
+
error "--roundhouse needs #{list}, which #{missing.size == 1 ? "was" : "were"} not found."
|
|
155
|
+
STDERR.puts
|
|
156
|
+
STDERR.puts AotToolchain.install_instructions(missing)
|
|
157
|
+
STDERR.puts
|
|
158
|
+
|
|
159
|
+
if tools[:roundhouse]
|
|
160
|
+
@env = child_env(tools)
|
|
161
|
+
print_check(tools[:roundhouse], "Meanwhile, Roundhouse's analysis of #{display(@app)}:")
|
|
162
|
+
else
|
|
163
|
+
STDERR.puts "Once Roundhouse is installed, `roundhouse check --continue #{display(@app)}` lists the gems"
|
|
164
|
+
STDERR.puts "and constructs of the app it does not cover yet; OCRAN shows it when a build fails."
|
|
165
|
+
end
|
|
166
|
+
raise "the Roundhouse toolchain is not installed"
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def transpile_failed(tools, output, status, work)
|
|
170
|
+
print_failure("Roundhouse could not transpile #{display(@app)}", output, status)
|
|
171
|
+
print_check(tools[:roundhouse], "Roundhouse's analysis of the app:")
|
|
172
|
+
STDERR.puts
|
|
173
|
+
STDERR.puts "Things to try:"
|
|
174
|
+
STDERR.puts " - what Roundhouse covers: #{AotToolchain::ROUNDHOUSE_URL}/blob/main/docs/guide/rails-coverage.md"
|
|
175
|
+
STDERR.puts " - --roundhouse-opt --survey --roundhouse-opt --allow-unsupported transpiles an app that is"
|
|
176
|
+
STDERR.puts " not fully covered yet, with a stub at each unsupported site"
|
|
177
|
+
STDERR.puts " - the generated project so far is in #{work}"
|
|
178
|
+
raise "Roundhouse could not transpile #{display(@app)}"
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def compile_failed(tools, output, status, work)
|
|
182
|
+
print_failure("Spinel could not compile the transpiled app (spin build)", output, status)
|
|
183
|
+
|
|
184
|
+
STDERR.puts "Things to check:"
|
|
185
|
+
versions = tools.map { |name, path| "#{name}: #{AotToolchain.version(path) || "version unknown"}" }
|
|
186
|
+
STDERR.puts " - versions in use (#{versions.join("; ")}); each Roundhouse snapshot names the Spinel"
|
|
187
|
+
STDERR.puts " release it was tested against in #{AotToolchain::ROUNDHOUSE_URL}/blob/main/RELEASES.md,"
|
|
188
|
+
STDERR.puts " and a mismatch is the most common cause of a failing build"
|
|
189
|
+
missing_libs.each do |lib, package|
|
|
190
|
+
STDERR.puts " - the #{lib} development headers seem to be missing (#{package})"
|
|
191
|
+
end
|
|
192
|
+
unlinked_libs(output).each do |lib|
|
|
193
|
+
STDERR.puts " - the linker could not find lib#{lib}: install it (Debian/Ubuntu: lib#{lib}-dev; macOS: brew install #{lib});"
|
|
194
|
+
STDERR.puts " if it is installed outside the linker's default directories, set LIBRARY_PATH to its lib directory"
|
|
195
|
+
end
|
|
196
|
+
STDERR.puts " - the generated project is kept in #{File.join(work, "spinel")}; run `spin build` there to retry"
|
|
197
|
+
STDERR.puts " - a compiler error in the generated code is a bug in Roundhouse or Spinel: please report it"
|
|
198
|
+
raise "Spinel could not compile the transpiled app"
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def print_failure(heading, output, status)
|
|
202
|
+
lines = output.to_s.lines
|
|
203
|
+
exit_status = status ? "exit status #{status.exitstatus.inspect}" : "could not be started"
|
|
204
|
+
STDERR.puts "#{heading} (#{exit_status}):"
|
|
205
|
+
STDERR.puts " ... (#{lines.size - MAX_OUTPUT_LINES} earlier lines omitted)" if lines.size > MAX_OUTPUT_LINES
|
|
206
|
+
STDERR.puts lines.last(MAX_OUTPUT_LINES).map { |l| " #{l}" }.join
|
|
207
|
+
STDERR.puts
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# Prints the useful part of `roundhouse check --continue`: the gem
|
|
211
|
+
# census (which gems Roundhouse does not model), the summary, the first
|
|
212
|
+
# errors and the punch list of unsupported constructs. The full report
|
|
213
|
+
# of a large app runs to thousands of lines.
|
|
214
|
+
def print_check(roundhouse, heading)
|
|
215
|
+
say "Analyzing #{display(@app)} with roundhouse check"
|
|
216
|
+
output, status = run([roundhouse, "check", "--continue", @app.to_s])
|
|
217
|
+
return STDERR.puts("(roundhouse check could not be run: #{output.to_s.strip})") unless status
|
|
218
|
+
|
|
219
|
+
lines = output.lines.map(&:chomp)
|
|
220
|
+
summary = lines.grep(/\Aroundhouse-check:/)
|
|
221
|
+
errors = lines.grep(/: error\[/)
|
|
222
|
+
survey_start = lines.index { |l| l.include?("Survey:") }
|
|
223
|
+
survey = survey_start ? lines[survey_start..].take_while { |l| !l.start_with?("roundhouse-check:") }.reject(&:empty?) : []
|
|
224
|
+
|
|
225
|
+
STDERR.puts heading
|
|
226
|
+
STDERR.puts
|
|
227
|
+
summary.each { |l| STDERR.puts " #{l}" }
|
|
228
|
+
if summary.any? { |l| l.match?(/\d+ unknown/) && !l.match?(/\b0 unknown/) }
|
|
229
|
+
STDERR.puts " (\"unknown\" gems are ones Roundhouse does not model: code using them cannot be compiled yet)"
|
|
230
|
+
end
|
|
231
|
+
unless errors.empty?
|
|
232
|
+
STDERR.puts
|
|
233
|
+
errors.first(MAX_CHECK_ERRORS).each { |l| STDERR.puts " #{l}" }
|
|
234
|
+
STDERR.puts " ... and #{errors.size - MAX_CHECK_ERRORS} more error(s)" if errors.size > MAX_CHECK_ERRORS
|
|
235
|
+
end
|
|
236
|
+
unless survey.empty?
|
|
237
|
+
STDERR.puts
|
|
238
|
+
survey.first(30).each { |l| STDERR.puts " #{l}" }
|
|
239
|
+
end
|
|
240
|
+
STDERR.puts
|
|
241
|
+
STDERR.puts " Full report: roundhouse check --continue #{display(@app)}"
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# Libraries the generated project links that pkg-config cannot find,
|
|
245
|
+
# as [library, package hint] pairs. Empty when pkg-config is not there
|
|
246
|
+
# to ask.
|
|
247
|
+
def missing_libs
|
|
248
|
+
return [] unless AotToolchain.search_path("pkg-config")
|
|
249
|
+
|
|
250
|
+
{ "sqlite3" => "libsqlite3-dev / brew install sqlite",
|
|
251
|
+
"jemalloc" => "libjemalloc-dev / brew install jemalloc" }.reject do |lib, _|
|
|
252
|
+
Kernel.system("pkg-config", "--exists", lib)
|
|
253
|
+
end.to_a
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
# Libraries the link step could not find, from GNU ld's
|
|
257
|
+
# "cannot find -lfoo" and Apple ld's "library 'foo' not found".
|
|
258
|
+
def unlinked_libs(output)
|
|
259
|
+
output.to_s.scan(/cannot find -l([\w.+-]+)|library '([\w.+-]+)' not found/).flatten.compact.uniq
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
def install(project)
|
|
263
|
+
bin_dir = File.join(project, "build", "bin")
|
|
264
|
+
binaries = Dir.glob(File.join(bin_dir, "*")).select { |f| File.file?(f) && File.executable?(f) }
|
|
265
|
+
binary = binaries.find { |f| File.basename(f, ".*") == @app.basename.to_s } || binaries.first
|
|
266
|
+
raise "spin build succeeded but produced no executable in #{bin_dir}" unless binary
|
|
267
|
+
|
|
268
|
+
@output.mkpath
|
|
269
|
+
target = @output + File.basename(binary)
|
|
270
|
+
FileUtils.cp(binary, target)
|
|
271
|
+
File.chmod(0o755, target)
|
|
272
|
+
|
|
273
|
+
RUNTIME_DIRS.each do |dir|
|
|
274
|
+
source = File.join(project, dir)
|
|
275
|
+
next unless File.directory?(source)
|
|
276
|
+
|
|
277
|
+
FileUtils.rm_rf(@output + dir)
|
|
278
|
+
FileUtils.cp_r(source, @output + dir)
|
|
279
|
+
end
|
|
280
|
+
# storage/ holds the database and uploads, so a rebuild keeps it.
|
|
281
|
+
(@output + "storage").mkpath
|
|
282
|
+
|
|
283
|
+
seed_database
|
|
284
|
+
say "Finished building #{display(@output)}"
|
|
285
|
+
say "Run it with: cd #{display(@output)} && ./#{File.basename(binary)} (serves on port 3000; PORT or -p to change)"
|
|
286
|
+
say "It needs libsqlite3 and libjemalloc at run time (Debian/Ubuntu: libsqlite3-0 libjemalloc2)."
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
# Creates the database from the generated seed on the first build, when
|
|
290
|
+
# the sqlite3 command is there to do it.
|
|
291
|
+
def seed_database
|
|
292
|
+
seed = @output + "db" + "seed.sql"
|
|
293
|
+
database = @output + "storage" + "development.sqlite3"
|
|
294
|
+
return unless seed.file? && !database.exist?
|
|
295
|
+
|
|
296
|
+
sqlite3 = AotToolchain.search_path("sqlite3")
|
|
297
|
+
unless sqlite3
|
|
298
|
+
say "Create the database before the first start: sqlite3 storage/development.sqlite3 < db/seed.sql"
|
|
299
|
+
return
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
verbose "Seeding #{database} from #{seed}"
|
|
303
|
+
unless Kernel.system(sqlite3, database.to_s, in: seed.to_s)
|
|
304
|
+
warning "Could not create the database from #{seed}; run: sqlite3 storage/development.sqlite3 < db/seed.sql"
|
|
305
|
+
end
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# Relative to the working directory when inside it, else absolute.
|
|
309
|
+
def display(path)
|
|
310
|
+
rel = Pathname(path).relative_path_from(Pathname.pwd).to_s
|
|
311
|
+
rel.start_with?("..") ? path.to_s : rel
|
|
312
|
+
rescue ArgumentError
|
|
313
|
+
path.to_s
|
|
314
|
+
end
|
|
315
|
+
end
|
|
316
|
+
end
|
data/lib/ocran/runner.rb
CHANGED
|
@@ -6,8 +6,19 @@ module Ocran
|
|
|
6
6
|
load File.expand_path("command_output.rb", __dir__)
|
|
7
7
|
include CommandOutput
|
|
8
8
|
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
# Errors that describe a problem with the input or the build host - an
|
|
10
|
+
# invalid option, a missing file, a path OCRAN cannot write, a value too
|
|
11
|
+
# large for the executable format - rather than a bug in OCRAN. They
|
|
12
|
+
# are reported by their message; a backtrace would only bury it. Any
|
|
13
|
+
# other exception is a bug and keeps Ruby's full report.
|
|
14
|
+
USER_ERRORS = [RuntimeError, ArgumentError, SystemCallError, IOError].freeze
|
|
15
|
+
|
|
16
|
+
# Reports a USER_ERRORS exception and exits. With --verbose the
|
|
17
|
+
# backtrace is shown too, for when the message is not enough.
|
|
18
|
+
def user_error(e)
|
|
19
|
+
message = e.instance_of?(RuntimeError) ? e.message : "#{e.message} (#{e.class})"
|
|
20
|
+
error message
|
|
21
|
+
STDERR.puts e.backtrace.map { |line| "\tfrom #{line}" } if @option&.verbose? && e.backtrace
|
|
11
22
|
exit false
|
|
12
23
|
end
|
|
13
24
|
|
|
@@ -18,11 +29,9 @@ module Ocran
|
|
|
18
29
|
load File.expand_path("option.rb", __dir__)
|
|
19
30
|
@option = Option.new.tap do |opt|
|
|
20
31
|
opt.parse(ARGV)
|
|
21
|
-
rescue
|
|
22
|
-
#
|
|
23
|
-
|
|
24
|
-
# option arguments.
|
|
25
|
-
fatal_error e.message
|
|
32
|
+
rescue *USER_ERRORS => e
|
|
33
|
+
# Invalid option arguments, usually; report them as such.
|
|
34
|
+
user_error e
|
|
26
35
|
else
|
|
27
36
|
# Update ARGV with the parsed command line arguments to pass to
|
|
28
37
|
# the user's script. This ensures the script executes based on
|
|
@@ -31,6 +40,7 @@ module Ocran
|
|
|
31
40
|
end
|
|
32
41
|
|
|
33
42
|
Ocran.option = @option
|
|
43
|
+
@option.warnings.each { |message| warning message }
|
|
34
44
|
|
|
35
45
|
@ignore_modules = ObjectSpace.each_object(Module).to_a
|
|
36
46
|
end
|
|
@@ -206,6 +216,24 @@ module Ocran
|
|
|
206
216
|
end
|
|
207
217
|
|
|
208
218
|
def build
|
|
219
|
+
# Native compilation hands the source to an ahead-of-time compiler, and
|
|
220
|
+
# --wasm to a WebAssembly runtime; nothing was loaded, and none of the
|
|
221
|
+
# packaging below applies.
|
|
222
|
+
if @option.spinel? || @option.roundhouse? || @option.wasm_runtime
|
|
223
|
+
Dir.chdir(@pre_env.pwd)
|
|
224
|
+
if @option.wasm_runtime
|
|
225
|
+
require_relative "wasm_builder"
|
|
226
|
+
WasmBuilder.new(@option).build
|
|
227
|
+
elsif @option.roundhouse?
|
|
228
|
+
require_relative "roundhouse_builder"
|
|
229
|
+
RoundhouseBuilder.new(@option).build
|
|
230
|
+
else
|
|
231
|
+
require_relative "spinel_builder"
|
|
232
|
+
SpinelBuilder.new(@option).build
|
|
233
|
+
end
|
|
234
|
+
return
|
|
235
|
+
end
|
|
236
|
+
|
|
209
237
|
# If the script was run and autoload is enabled, attempt to autoload libraries.
|
|
210
238
|
if @option.force_autoload?
|
|
211
239
|
attempt_load_autoload(@ignore_modules)
|
|
@@ -226,8 +254,10 @@ module Ocran
|
|
|
226
254
|
|
|
227
255
|
if @option.use_inno_setup?
|
|
228
256
|
# Native on Windows; on POSIX allow it when an ISCC command is
|
|
229
|
-
# available (e.g. via Wine wrapper scripts or in tests).
|
|
230
|
-
|
|
257
|
+
# available (e.g. via Wine wrapper scripts or in tests). Checked
|
|
258
|
+
# here, before the files are collected, so a missing ISCC fails fast.
|
|
259
|
+
require_relative "inno_setup_script_builder"
|
|
260
|
+
if InnoSetupScriptBuilder.iscc_available?
|
|
231
261
|
direction.build_inno_setup_installer
|
|
232
262
|
else
|
|
233
263
|
raise "Inno Setup is only supported on Windows (no ISCC command found in PATH)"
|
|
@@ -238,13 +268,15 @@ module Ocran
|
|
|
238
268
|
direction.build_output_dir(@option.output_dir)
|
|
239
269
|
elsif @option.output_zip
|
|
240
270
|
direction.build_zip(@option.output_zip)
|
|
271
|
+
elsif @option.static_runtime?
|
|
272
|
+
direction.build_static_runtime_exe
|
|
241
273
|
elsif @option.cosmo_zip?
|
|
242
274
|
direction.build_cosmo_zip_exe
|
|
243
275
|
else
|
|
244
276
|
direction.build_stab_exe
|
|
245
277
|
end
|
|
246
|
-
rescue
|
|
247
|
-
|
|
278
|
+
rescue *USER_ERRORS => e
|
|
279
|
+
user_error e
|
|
248
280
|
end
|
|
249
281
|
end
|
|
250
282
|
end
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "pathname"
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require "open3"
|
|
5
|
+
require "rbconfig"
|
|
6
|
+
require "tmpdir"
|
|
7
|
+
require "json"
|
|
8
|
+
require_relative "pe_imports"
|
|
9
|
+
|
|
10
|
+
module Ocran
|
|
11
|
+
# The SDK part of a static runtime (<runtime>/sdk, written by
|
|
12
|
+
# runtime/make-sdk.rb): what is needed to link the runtime again with more
|
|
13
|
+
# C extensions statically linked in - its link inputs and link command
|
|
14
|
+
# (link.args), Ruby's headers, the runtime's rbconfig.rb with the
|
|
15
|
+
# <arch>-fake.rb Ruby's cross builds use to describe the target, and the
|
|
16
|
+
# gem extensions already linked in (exts/*.line).
|
|
17
|
+
#
|
|
18
|
+
# Linking needs a compiler for the runtime's target on the build host. For
|
|
19
|
+
# a MinGW runtime (toolchain "gcc" in sdk.json): the one link.args names
|
|
20
|
+
# (e.g. x86_64-w64-mingw32ucrt-gcc on Linux), or OCRAN_RUNTIME_CC (e.g. gcc
|
|
21
|
+
# from RubyInstaller's MSYS2 on Windows). For an mswin runtime ("msvc"):
|
|
22
|
+
# cl and nmake of Visual Studio (Build Tools), from a developer prompt.
|
|
23
|
+
class RuntimeSdk
|
|
24
|
+
BUILD_STATIC_EXT = File.expand_path("static_runtime/build_static_ext.rb", __dir__)
|
|
25
|
+
|
|
26
|
+
# One statically built extension: the feature it provides ("bcrypt_ext"),
|
|
27
|
+
# its Init function, the static library and the libraries it links with.
|
|
28
|
+
Extension = Struct.new(:feature, :init, :archive, :libs) do
|
|
29
|
+
def self.parse(line, base)
|
|
30
|
+
feature, init, archive, libs = line.strip.split(" ", 4)
|
|
31
|
+
raise "bad extension line #{line.inspect}" unless feature&.match?(%r{\A[\w/.-]+\z}) && init&.match?(/\A\w+\z/)
|
|
32
|
+
|
|
33
|
+
new(feature, init, File.expand_path(archive.sub("@SDK@", base.to_s)), libs.to_s.split(","))
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
attr_reader :dir, :arch, :toolchain
|
|
38
|
+
|
|
39
|
+
def initialize(dir, arch:)
|
|
40
|
+
@dir = Pathname(dir)
|
|
41
|
+
@arch = arch
|
|
42
|
+
raise "#{@dir} is not a static runtime SDK (no link.args)" unless (@dir / "link.args").file?
|
|
43
|
+
|
|
44
|
+
@info = (@dir / "sdk.json").file? ? JSON.parse((@dir / "sdk.json").read) : {}
|
|
45
|
+
@toolchain = @info.fetch("toolchain", "gcc")
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Libraries available as DLLs instead of linked in (--runtime-dll):
|
|
49
|
+
# name ("sybdb") => { "dll" => ..., "import" => ... }, relative to the SDK.
|
|
50
|
+
def dlls = @info.fetch("dlls", {})
|
|
51
|
+
|
|
52
|
+
# extconf.rb options for gems whose C library the SDK provides, so that
|
|
53
|
+
# extconf.rb uses it instead of downloading and building its own.
|
|
54
|
+
GEM_EXTCONF_ARGS = {
|
|
55
|
+
"nokogiri" => ["--use-system-libraries"],
|
|
56
|
+
"tiny_tds" => ["--with-freetds-dir=@SDK_DEPS@"],
|
|
57
|
+
}.freeze
|
|
58
|
+
|
|
59
|
+
def extconf_args(gem_name)
|
|
60
|
+
(GEM_EXTCONF_ARGS[gem_name] || []).map { |a| a.gsub("@SDK_DEPS@", (@dir / "deps").to_s) }
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def msvc? = @toolchain == "msvc"
|
|
64
|
+
|
|
65
|
+
# True when the application archive cannot be appended to the
|
|
66
|
+
# executable but is linked in as the section __OCRAN,__zip (macOS, where
|
|
67
|
+
# appended data would break the code signature).
|
|
68
|
+
def section_payload? = @info["payload"] == "section"
|
|
69
|
+
|
|
70
|
+
def link_args = (@dir / "link.args").read.lines(chomp: true).map { |a| a.gsub("@SDK@", @dir.to_s) }
|
|
71
|
+
|
|
72
|
+
# The SDK's header directories, ahead of whatever an extconf.rb adds:
|
|
73
|
+
# the SDK's libraries are the ones linked, so their headers must win
|
|
74
|
+
# over a system copy (nokogiri adds the libxml2 headers of Apple's SDK,
|
|
75
|
+
# an older libxml2, on macOS).
|
|
76
|
+
def sdk_include_flags
|
|
77
|
+
deps = @dir / "deps" / "include"
|
|
78
|
+
[deps, deps / "libxml2"].select(&:directory?).map { |dir| "-I#{dir}" }
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
# The C compiler driver of the target.
|
|
82
|
+
def cc
|
|
83
|
+
return "cl" if msvc?
|
|
84
|
+
|
|
85
|
+
env = ENV["OCRAN_RUNTIME_CC"]
|
|
86
|
+
env && !env.empty? ? env : link_args.first
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# The gem extensions already linked into the runtime.
|
|
90
|
+
def base_extensions
|
|
91
|
+
Dir[(@dir / "exts/*.line").to_s].sort.map { |f| Extension.parse(File.read(f), @dir) }
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Fails early, with a hint, when the build host has no compiler for the
|
|
95
|
+
# runtime's target.
|
|
96
|
+
def check_toolchain!
|
|
97
|
+
return check_msvc! if msvc?
|
|
98
|
+
|
|
99
|
+
out, status = Open3.capture2e(cc, "-dumpmachine")
|
|
100
|
+
expected = @info["dumpmachine"]
|
|
101
|
+
return if status.success? && (expected.nil? || out.include?(expected))
|
|
102
|
+
|
|
103
|
+
raise "#{cc} #{status.success? ? "does not target #{expected} (#{out.strip})" : "did not run"}; " \
|
|
104
|
+
"set OCRAN_RUNTIME_CC to #{toolchain_hint}"
|
|
105
|
+
rescue SystemCallError
|
|
106
|
+
raise "#{cc} not found; set OCRAN_RUNTIME_CC to #{toolchain_hint}"
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def toolchain_hint
|
|
110
|
+
target_os = @info.fetch("target_os", "mingw32")
|
|
111
|
+
if target_os.start_with?("darwin")
|
|
112
|
+
return "clang from Xcode or the Command Line Tools (xcode-select --install)"
|
|
113
|
+
end
|
|
114
|
+
if target_os.start_with?("mingw")
|
|
115
|
+
"a MinGW-w64 UCRT C compiler (on Linux: gcc-mingw-w64-ucrt64, on Windows: RubyInstaller's " \
|
|
116
|
+
"MSYS2, e.g. ucrt64/bin/gcc.exe)"
|
|
117
|
+
else
|
|
118
|
+
"a musl C compiler (musl-gcc from musl-tools)"
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def check_msvc!
|
|
123
|
+
%w[cl nmake].each do |tool|
|
|
124
|
+
Open3.capture2e(tool, "-?")
|
|
125
|
+
rescue SystemCallError
|
|
126
|
+
raise "#{tool} not found: building C extensions for an mswin runtime needs Visual Studio " \
|
|
127
|
+
"(Build Tools); run ocran from a \"x64 Native Tools\" developer prompt"
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# Compiles the C extension in +ext_dir+ (a gem's ext/<name>/, with its
|
|
132
|
+
# extconf.rb) to a static library under +work_dir+. +gem_path+: more
|
|
133
|
+
# gem directories for the gems extconf.rb itself uses.
|
|
134
|
+
def build_extension(ext_dir, work_dir, *extconf_args, gem_path: [])
|
|
135
|
+
pkgconfig = File.join(work_dir.to_s + ".pkgconfig")
|
|
136
|
+
FileUtils.mkdir_p(pkgconfig)
|
|
137
|
+
Dir[(@dir / "deps/lib/pkgconfig/*.pc").to_s].each do |pc|
|
|
138
|
+
File.write(File.join(pkgconfig, File.basename(pc)), File.read(pc).gsub("@SDK_DEPS@", (@dir / "deps").to_s))
|
|
139
|
+
end
|
|
140
|
+
# The SDK's rbconfig.rb is found first on the load path; a MinGW SDK
|
|
141
|
+
# also has Ruby's <arch>-fake.rb, which makes any Ruby pose as the
|
|
142
|
+
# runtime's (an mswin build host runs the runtime's own revision).
|
|
143
|
+
fake = Dir[(@dir / "*-fake.rb").to_s].first
|
|
144
|
+
env = { "RUBYOPT" => nil, "BUNDLE_GEMFILE" => nil, "BUNDLER_SETUP" => nil,
|
|
145
|
+
"OCRAN_RUNTIME_CC" => msvc? ? nil : ENV["OCRAN_RUNTIME_CC"] || (cc == link_args.first ? nil : cc),
|
|
146
|
+
"OCRAN_RUNTIME_LIBS" => @info.fetch("libs", []).join(" "),
|
|
147
|
+
"OCRAN_RUNTIME_CPPFLAGS" => (sdk_include_flags + @info.fetch("cppflags", [])).join(" "),
|
|
148
|
+
"PKG_CONFIG_LIBDIR" => pkgconfig, "PKG_CONFIG_PATH" => "",
|
|
149
|
+
# Build-time gems extconf.rb uses (nokogiri: mini_portile2):
|
|
150
|
+
# the build host's, which the target's RbConfig would hide.
|
|
151
|
+
"GEM_HOME" => Gem.dir, "GEM_PATH" => (gem_path + Gem.path).join(File::PATH_SEPARATOR) }
|
|
152
|
+
cmd = [RbConfig.ruby, "--disable=gems", "-I", @dir.to_s, *(fake ? ["-r", fake] : []), BUILD_STATIC_EXT,
|
|
153
|
+
(@dir / "include").to_s, (@dir / "include" / @arch).to_s, ext_dir.to_s, work_dir.to_s, *extconf_args]
|
|
154
|
+
out, status = Open3.capture2e(env, *cmd)
|
|
155
|
+
unless status.success?
|
|
156
|
+
raise "Building the C extension in #{ext_dir} for the static runtime failed:\n#{out.lines.last(25).join}"
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
Extension.parse(out.lines.last, @dir)
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
# Links the runtime with the given extensions registered in addition to
|
|
163
|
+
# those it already has, into +output+. Libraries named in +dlls+ are
|
|
164
|
+
# linked against their DLL instead of statically (see #dll_files).
|
|
165
|
+
# macOS: the application archive is linked in as a section (see
|
|
166
|
+
# #section_payload?); +archive+ is that file.
|
|
167
|
+
def link(extensions, output, dlls: [], archive: nil)
|
|
168
|
+
unknown = dlls - self.dlls.keys
|
|
169
|
+
unless unknown.empty?
|
|
170
|
+
raise "No DLL build of #{unknown.join(", ")} in #{@dir} (available: #{self.dlls.keys.join(", ")})"
|
|
171
|
+
end
|
|
172
|
+
as_dll = dlls.to_h { |name| ["-l#{name}", (@dir / self.dlls[name]["import"]).to_s] }
|
|
173
|
+
all = base_extensions + extensions
|
|
174
|
+
Dir.mktmpdir do |tmp|
|
|
175
|
+
src = File.join(tmp, "ocran_exts.c")
|
|
176
|
+
File.write(src, exts_source(all))
|
|
177
|
+
obj = File.join(tmp, msvc? ? "ocran_exts.obj" : "ocran_exts.o")
|
|
178
|
+
if msvc?
|
|
179
|
+
run("cl", "-nologo", "-MT", "-O2", "-c", src, "-Fo#{obj}")
|
|
180
|
+
else
|
|
181
|
+
run(cc, "-O2", "-c", src, "-o", obj)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
extra = [obj] + all.flat_map { |ext| [ext.archive, *ext.libs] } + @info.fetch("libs", [])
|
|
185
|
+
args = link_args
|
|
186
|
+
args[0] = cc
|
|
187
|
+
cmd = args.flat_map { |a| a == "@EXTRA@" ? extra : [a.gsub("@OUTPUT@", output.to_s)] }
|
|
188
|
+
cmd = cmd.map { |a| as_dll.fetch(a, a) }
|
|
189
|
+
cmd += ["-o", output.to_s] unless args.any? { |a| a.include?("@OUTPUT@") }
|
|
190
|
+
cmd += ["-Wl,-sectcreate,__OCRAN,__zip,#{archive}"] if archive
|
|
191
|
+
run(*cmd)
|
|
192
|
+
end
|
|
193
|
+
strip = strip_tool
|
|
194
|
+
run(strip, output.to_s) if strip
|
|
195
|
+
output
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# The strip of the compiler in use: sdk.json names the SDK build host's
|
|
199
|
+
# (nil on macOS, where stripping would break the signature), which
|
|
200
|
+
# another compiler named with OCRAN_RUNTIME_CC may not come with (MSYS2
|
|
201
|
+
# has gcc and strip, not x86_64-w64-mingw32ucrt-strip).
|
|
202
|
+
def strip_tool
|
|
203
|
+
return nil if msvc? || (@info.key?("strip") && @info["strip"].nil?)
|
|
204
|
+
|
|
205
|
+
derived = cc.sub(/gcc(\.exe)?\z/) { "strip#{$1}" } if cc.match?(/gcc(\.exe)?\z/)
|
|
206
|
+
return derived if derived && cc != link_args.first
|
|
207
|
+
|
|
208
|
+
@info["strip"] || derived
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# The DLL files to ship next to the executable for +dlls+: each named
|
|
212
|
+
# library's DLL and the SDK DLLs it imports in turn (libsybdb-5.dll
|
|
213
|
+
# needs libiconv-2.dll).
|
|
214
|
+
def dll_files(dlls)
|
|
215
|
+
by_file = self.dlls.values.to_h { |d| [File.basename(d["dll"]).downcase, @dir / d["dll"]] }
|
|
216
|
+
queue = dlls.map { |name| @dir / self.dlls.fetch(name)["dll"] }
|
|
217
|
+
found = {}
|
|
218
|
+
until queue.empty?
|
|
219
|
+
dll = queue.shift
|
|
220
|
+
next if found[dll.basename.to_s.downcase]
|
|
221
|
+
|
|
222
|
+
found[dll.basename.to_s.downcase] = dll
|
|
223
|
+
PEImports.dll_names(dll.to_s).each { |name| queue << by_file[name.downcase] if by_file[name.downcase] }
|
|
224
|
+
end
|
|
225
|
+
found.values
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
private
|
|
229
|
+
|
|
230
|
+
def exts_source(extensions)
|
|
231
|
+
<<~C
|
|
232
|
+
/* generated by ocran (Ocran::RuntimeSdk) */
|
|
233
|
+
void ruby_init_ext(const char *name, void (*init)(void));
|
|
234
|
+
#{extensions.map { |e| "extern void #{e.init}(void);" }.join("\n")}
|
|
235
|
+
void ocran_init_extra_exts(void)
|
|
236
|
+
{
|
|
237
|
+
#{extensions.map { |e| " ruby_init_ext(\"#{e.feature}.so\", #{e.init});" }.join("\n")}
|
|
238
|
+
}
|
|
239
|
+
C
|
|
240
|
+
end
|
|
241
|
+
|
|
242
|
+
# Compilers, linkers and strip run with SOURCE_DATE_EPOCH set: binutils
|
|
243
|
+
# then writes that, not the current time, into a PE header it rewrites
|
|
244
|
+
# (strip does), so that the same application links to the same bytes.
|
|
245
|
+
def run(*cmd)
|
|
246
|
+
out, status = Open3.capture2e({ "SOURCE_DATE_EPOCH" => ENV["SOURCE_DATE_EPOCH"] || "1767225600" }, *cmd)
|
|
247
|
+
raise "#{cmd.first} failed:\n#{out.lines.last(25).join}" unless status.success?
|
|
248
|
+
|
|
249
|
+
out
|
|
250
|
+
end
|
|
251
|
+
end
|
|
252
|
+
end
|