ocran 1.4.6 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (49) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.txt +68 -0
  3. data/README.md +295 -10
  4. data/ext/extconf.rb +5 -1
  5. data/lib/ocran/aot_toolchain.rb +154 -0
  6. data/lib/ocran/command_output.rb +8 -1
  7. data/lib/ocran/compatibility_scan.rb +414 -0
  8. data/lib/ocran/cosmo_toolchain.rb +29 -7
  9. data/lib/ocran/dir_builder.rb +88 -38
  10. data/lib/ocran/direction.rb +190 -30
  11. data/lib/ocran/ed_resource.rb +113 -0
  12. data/lib/ocran/ed_resource_builder.rb +199 -0
  13. data/lib/ocran/file_path_set.rb +9 -5
  14. data/lib/ocran/gem_spec_queryable.rb +7 -3
  15. data/lib/ocran/host_config_helper.rb +11 -1
  16. data/lib/ocran/inno_setup_script_builder.rb +24 -14
  17. data/lib/ocran/launcher_batch_builder.rb +8 -11
  18. data/lib/ocran/npm_package.rb +106 -0
  19. data/lib/ocran/option.rb +425 -35
  20. data/lib/ocran/pe_imports.rb +96 -0
  21. data/lib/ocran/pe_resources.rb +226 -0
  22. data/lib/ocran/refine_pathname.rb +29 -32
  23. data/lib/ocran/roundhouse_builder.rb +316 -0
  24. data/lib/ocran/runner.rb +43 -11
  25. data/lib/ocran/runtime_sdk.rb +252 -0
  26. data/lib/ocran/spinel_builder.rb +183 -0
  27. data/lib/ocran/spinel_compatibility.rb +149 -0
  28. data/lib/ocran/static_runtime/boot.rb +443 -0
  29. data/lib/ocran/static_runtime/build_static_ext.rb +86 -0
  30. data/lib/ocran/static_runtime_builder.rb +814 -0
  31. data/lib/ocran/stub_builder.rb +70 -25
  32. data/lib/ocran/version.rb +1 -1
  33. data/lib/ocran/wasm_builder.rb +570 -0
  34. data/lib/ocran/wasm_compatibility.rb +198 -0
  35. data/lib/ocran/windows_command_escaping.rb +21 -0
  36. data/lib/ocran/zip_payload_builder.rb +18 -9
  37. data/lib/ocran/zip_writer.rb +48 -9
  38. data/src/Makefile +18 -2
  39. data/src/Makefile.mswin +65 -0
  40. data/src/inst_dir.c +34 -6
  41. data/src/path_utils.c +144 -0
  42. data/src/script_info.c +5 -0
  43. data/src/stub.c +77 -19
  44. data/src/stub.rc +1 -1
  45. data/src/system_utils.c +276 -155
  46. data/src/system_utils.h +46 -3
  47. data/src/system_utils_posix.c +243 -148
  48. data/src/unpack.c +9 -0
  49. metadata +22 -4
@@ -0,0 +1,814 @@
1
+ # frozen_string_literal: true
2
+ require "pathname"
3
+ require "fileutils"
4
+ require "json"
5
+ require "digest"
6
+ require_relative "build_constants"
7
+ require_relative "command_output"
8
+ require_relative "zip_writer"
9
+ require_relative "zip_payload_builder"
10
+ require "tmpdir"
11
+
12
+ module Ocran
13
+ # A static runtime for --static-runtime: a statically linked Windows Ruby
14
+ # (ocran-runtime.exe), its own standard library and bundled gems
15
+ # (lib/ruby/), and runtime.json describing both. Built by
16
+ # runtime/build-runtime.sh (MinGW) or runtime/mswin/ (MSVC).
17
+ class StaticRuntime
18
+ attr_reader :dir, :info
19
+
20
+ def self.load(dir)
21
+ dir = Pathname(dir).expand_path
22
+ json = dir / "runtime.json"
23
+ raise "#{dir} is not an ocran static runtime (no runtime.json)" unless json.file?
24
+
25
+ new(dir, JSON.parse(json.read))
26
+ end
27
+
28
+ def initialize(dir, info)
29
+ @dir = Pathname(dir)
30
+ @info = info
31
+ raise "Unsupported runtime.json format #{info["format"].inspect} in #{dir}" unless info["format"] == 1
32
+ end
33
+
34
+ def executable = @dir / @info.fetch("executable")
35
+ def ruby_version = @info.fetch("ruby_version")
36
+ def os = @info.fetch("os", "windows")
37
+ def windows? = os == "windows"
38
+ def ruby_api_version = @info.fetch("ruby_api_version")
39
+ def arch = @info.fetch("arch")
40
+ def load_path = @info.fetch("load_path")
41
+ def gem_dir = @info.fetch("gem_dir")
42
+ def gem_platform = @info.fetch("gem_platform")
43
+ def extension_api_version = @info.fetch("extension_api_version")
44
+ def gems = @info.fetch("gems")
45
+
46
+ # The runtime executable is what every packaged application runs, so it
47
+ # is checked against the hash recorded when the runtime was built.
48
+ def verify!
49
+ exe = executable
50
+ raise "#{exe} not found" unless exe.file?
51
+
52
+ actual = Digest::SHA256.file(exe.to_s).hexdigest
53
+ expected = @info.fetch("executable_sha256")
54
+ return if actual == expected
55
+
56
+ raise "#{exe} does not match its runtime.json (SHA-256 #{actual}, expected #{expected})"
57
+ end
58
+
59
+ # The C extension a .so file of the build host stands for, when the
60
+ # runtime has it linked in: "lib/ruby/4.0.0/x64-mingw-ucrt/json/ext/parser.so"
61
+ # is "json/ext/parser".
62
+ def static_feature_for(path)
63
+ path = path.to_s.sub(/\.(so|bundle|dll)\z/, "")
64
+ @static_features ||= @info.fetch("static_features").sort_by { |f| -f.size }
65
+ @static_features.find { |feature| path == feature || path.end_with?("/#{feature}") }
66
+ end
67
+
68
+ # Absolute path of a file of the runtime's lib/ruby tree, or nil.
69
+ def lib_file(relative)
70
+ path = @dir / relative
71
+ path.file? ? path : nil
72
+ end
73
+
74
+ # The runtime's SDK (sdk/), for linking in more C extensions; nil when
75
+ # the runtime was distributed without one.
76
+ def sdk
77
+ return @sdk if defined?(@sdk)
78
+
79
+ require_relative "runtime_sdk"
80
+ @sdk = (@dir / "sdk" / "link.args").file? ? RuntimeSdk.new(@dir / "sdk", arch: arch) : nil
81
+ end
82
+ end
83
+
84
+ # Builder for --static-runtime: the output is a copy of the static runtime with
85
+ # the application appended as a ZIP archive. Nothing is extracted at run
86
+ # time; the runtime's stage 0 reads ocran/boot.rb from the archive, and
87
+ # boot.rb serves the archive as a read-only directory tree rooted at the
88
+ # executable's own path (C:/app/tool.exe/src/tool.rb).
89
+ #
90
+ # Direction lays an application out the way the extraction modes need
91
+ # it: the build host's interpreter under bin/, its standard library and
92
+ # gems under lib/ruby/, and so on. This builder translates that layout
93
+ # onto the runtime:
94
+ #
95
+ # * the interpreter, DLLs and other executables are not packed - the
96
+ # runtime is the interpreter, and a DLL cannot be loaded from inside
97
+ # the archive;
98
+ # * the standard library is the runtime's own, so it always matches the
99
+ # interpreter; a host file it does not have is not packed;
100
+ # * gems the runtime ships (default and bundled gems) are packed from the
101
+ # runtime, other gems from the host;
102
+ # * C extensions must be linked into the runtime: those it has are
103
+ # skipped, any other one fails the build.
104
+ class StaticRuntimeBuilder
105
+ include BuildConstants
106
+ include CommandOutput
107
+
108
+ # Where boot.rb finds the application root; replaced at run time.
109
+ ROOT = "<ROOT>"
110
+ BOOT_SCRIPT = File.expand_path("static_runtime/boot.rb", __dir__)
111
+ NATIVE_EXTENSIONS = %w[.so .bundle].freeze
112
+ # Files only the build host's interpreter needs.
113
+ HOST_ONLY = %w[.dll .exe .manifest].freeze
114
+ # RubyInstaller's hook into RubyGems and what it loads (including its
115
+ # win32/dll_directory extension): it manages the DLL search paths of an
116
+ # installed Ruby, which a static runtime has none of.
117
+ RUBYINSTALLER_FILES = %r{\Alib/ruby/(site_ruby/[^/]+/(ruby_installer(/|\.rb\z)|rubygems/defaults/operating_system\.rb\z)|[^/]+/[^/]+/win32/dll_directory\.so\z)}
118
+
119
+ attr_reader :data_size, :skipped, :shipped_dlls
120
+
121
+ def initialize(path, runtime:, host_ruby_api_version:, host_arch:, chdir_before: false,
122
+ chdir_exe_dir: false, windowed: false, debug_mode: false, dlls: [], resources: nil)
123
+ @path = Pathname(path)
124
+ @dlls = dlls
125
+ @resources = resources
126
+ @runtime = runtime
127
+ @host_api = host_ruby_api_version
128
+ @host_arch = host_arch
129
+ @chdir = chdir_exe_dir || chdir_before
130
+ @windowed = windowed
131
+ @debug_mode = debug_mode
132
+ @entries = {}
133
+ @env = {}
134
+ @exec = nil
135
+ @runtime_gems = {}
136
+ @unsupported_native = []
137
+ @skipped = Hash.new { |h, k| h[k] = [] }
138
+ @data_size = 0
139
+ @shipped_dlls = []
140
+
141
+ yield(self) if block_given?
142
+
143
+ finalize
144
+ end
145
+
146
+ def mkdir(target)
147
+ name = normalize(target)
148
+ @entries["#{name}/"] ||= nil unless name.empty?
149
+ end
150
+
151
+ def cp(source, target)
152
+ name = normalize(target)
153
+ source = Pathname(source)
154
+
155
+ if name.start_with?("bin/") || name.start_with?("msys64/")
156
+ skip(:host_interpreter, name)
157
+ elsif name.match?(RUBYINSTALLER_FILES)
158
+ skip(:rubyinstaller, name)
159
+ elsif NATIVE_EXTENSIONS.include?(File.extname(name))
160
+ native_extension(source, name)
161
+ elsif HOST_ONLY.include?(File.extname(name).downcase)
162
+ if name.start_with?("src/")
163
+ warning "#{name} is packed, but a DLL or program inside the executable cannot be loaded or run from there"
164
+ add(name, source)
165
+ else
166
+ skip(:host_interpreter, name)
167
+ end
168
+ elsif (gem = gem_of(name)) && @runtime.gems.key?(gem[0])
169
+ runtime_gem(*gem)
170
+ elsif (relative = stdlib_path(name))
171
+ runtime_file = @runtime.lib_file(relative)
172
+ if runtime_file
173
+ add(relative, runtime_file)
174
+ else
175
+ skip(:not_in_runtime_stdlib, name)
176
+ end
177
+ else
178
+ add(name, source)
179
+ end
180
+ end
181
+
182
+ def symlink(link_path, target)
183
+ skip(:symlink, "#{normalize(link_path)} -> #{target}")
184
+ end
185
+
186
+ # Variables only the build host's interpreter layout needs.
187
+ HOST_ENV = %w[LD_LIBRARY_PATH DYLD_LIBRARY_PATH].freeze
188
+
189
+ # Variables naming a file or directory for C code to read (OpenSSL's
190
+ # CA certificates): C code cannot read inside the archive.
191
+ C_PATH_ENV = %w[SSL_CERT_FILE SSL_CERT_DIR].freeze
192
+
193
+ def export(name, value)
194
+ return if HOST_ENV.include?(name.to_s)
195
+ if C_PATH_ENV.include?(name.to_s) && value.to_s.start_with?(EXTRACT_ROOT.to_s)
196
+ verbose "Not setting #{name}: OpenSSL cannot read inside the executable (it uses the Windows certificate store)"
197
+ return
198
+ end
199
+
200
+ @env[name.to_s] = value.to_s
201
+ end
202
+
203
+ # The image is the host interpreter the extraction modes would start;
204
+ # here the runtime is the interpreter.
205
+ def exec(_image, script, *argv)
206
+ raise "Script is already set" if @exec
207
+
208
+ @exec = [normalize(script), argv.map(&:to_s)]
209
+ end
210
+
211
+ private
212
+
213
+ def normalize(target)
214
+ target.to_s.tr("\\", "/").delete_prefix("./").chomp("/")
215
+ end
216
+
217
+ def add(name, source)
218
+ return if @entries.key?(name)
219
+
220
+ @entries[name] = source
221
+ @data_size += File.size(source)
222
+ end
223
+
224
+ def skip(reason, name)
225
+ @skipped[reason] << name
226
+ end
227
+
228
+ def native_extension(source, name)
229
+ if (feature = @runtime.static_feature_for(name))
230
+ skip(:static_extension, "#{name} (#{feature})")
231
+ else
232
+ @unsupported_native << [name, source]
233
+ end
234
+ end
235
+
236
+ # Relative path inside the runtime's lib/ruby tree for a target in the
237
+ # host's standard library directory (lib/ruby/<api>/...), with the
238
+ # host's architecture directory mapped to the runtime's; nil for any
239
+ # other target.
240
+ def stdlib_path(name)
241
+ prefix = "lib/ruby/#{@host_api}/"
242
+ return nil unless name.start_with?(prefix)
243
+
244
+ rest = name.delete_prefix(prefix)
245
+ rest = "#{@runtime.arch}/#{rest.delete_prefix("#{@host_arch}/")}" if rest.start_with?("#{@host_arch}/")
246
+ "lib/ruby/#{@runtime.ruby_api_version}/#{rest}"
247
+ end
248
+
249
+ GEM_PATH_ENTRY = %r{(?:gems|specifications(?:/default)?|extensions/[^/]+/[^/]+)/([^/]+?)(?:\.gemspec)?(?:/(.*))?\z}
250
+
251
+ # [name, full name, path inside the gem directory] of the gem a target
252
+ # belongs to when it lies in a gem path (gems/, specifications/,
253
+ # extensions/), else nil. The path is nil for specs and extensions.
254
+ def gem_of(name)
255
+ match = name.match(%r{(?:\A|/)gems/[^/]+/#{GEM_PATH_ENTRY}}) || name.match(%r{\Agems/#{GEM_PATH_ENTRY}})
256
+ return nil unless match
257
+
258
+ full_name = match[1]
259
+ inner = name.include?("/gems/#{full_name}/") || name.start_with?("gems/gems/#{full_name}/") ? match[2] : nil
260
+ [full_name.sub(/-\d[^-]*(?:-.+)?\z/, ""), full_name, inner] # name-1.2.3[-x64-mingw-ucrt]
261
+ end
262
+
263
+ # A file of a gem the runtime ships itself: packed from the runtime,
264
+ # whose copy matches the C extensions linked into it.
265
+ def runtime_gem(name, host_full_name, inner)
266
+ spec = @runtime.gems.fetch(name)
267
+ runtime_full_name = File.basename(spec, ".gemspec")
268
+ if host_full_name != runtime_full_name && (@version_notes ||= {}).fetch(name, true)
269
+ @version_notes[name] = false
270
+ warning "#{host_full_name} is replaced by the static runtime's #{runtime_full_name}"
271
+ end
272
+ if spec.start_with?("specifications/default/")
273
+ # A default gem's files are part of the standard library.
274
+ pack_runtime_gem(name)
275
+ if inner&.start_with?("lib/")
276
+ relative = "lib/ruby/#{@runtime.ruby_api_version}/#{inner.delete_prefix("lib/")}"
277
+ runtime_file = @runtime.lib_file(relative)
278
+ runtime_file ? add(relative, runtime_file) : skip(:not_in_runtime_stdlib, relative)
279
+ end
280
+ else
281
+ pack_runtime_gem(name)
282
+ end
283
+ end
284
+
285
+ # Packs a gem from the runtime instead of the host's copy: its spec,
286
+ # its files (without native binaries, which are linked into the
287
+ # runtime), and RubyGems' gem.build_complete marker, without which a
288
+ # gem that declares extensions is not activated.
289
+ def pack_runtime_gem(name)
290
+ return if @runtime_gems.key?(name)
291
+
292
+ @runtime_gems[name] = true
293
+ gem_dir = @runtime.gem_dir
294
+ spec = @runtime.gems.fetch(name)
295
+ add("#{gem_dir}/#{spec}", @runtime.dir / gem_dir / spec)
296
+ full_name = File.basename(spec, ".gemspec")
297
+ root = @runtime.dir / gem_dir / "gems" / full_name
298
+ if root.directory?
299
+ root.find do |path|
300
+ next unless path.file?
301
+ next if (NATIVE_EXTENSIONS + HOST_ONLY).include?(path.extname.downcase)
302
+
303
+ add("#{gem_dir}/gems/#{full_name}/#{path.relative_path_from(root).to_s}", path)
304
+ end
305
+ end
306
+ if !spec.start_with?("specifications/default/") && gem_declares_extensions?(@runtime.dir / gem_dir / spec)
307
+ marker = "#{gem_dir}/extensions/#{@runtime.gem_platform}/#{@runtime.extension_api_version}/#{full_name}/gem.build_complete"
308
+ @entries[marker] ||= :empty
309
+ end
310
+ verbose "Packing gem #{full_name} from the static runtime"
311
+
312
+ # Activating a gem activates its runtime dependencies, so their specs
313
+ # have to be there too (psych needs stringio's, for instance), even
314
+ # when the application never loaded them as gems.
315
+ runtime_dependencies(@runtime.dir / gem_dir / spec).each do |dependency|
316
+ pack_runtime_gem(dependency) if @runtime.gems.key?(dependency)
317
+ end
318
+ end
319
+
320
+ def runtime_dependencies(spec_file)
321
+ Gem::Specification.load(spec_file.to_s)&.runtime_dependencies&.map(&:name) || []
322
+ end
323
+
324
+ def gem_declares_extensions?(spec_file)
325
+ File.read(spec_file).match?(/\.extensions\s*=\s*\[\s*"/)
326
+ end
327
+
328
+ # The standard library's Windows-only Ruby files (win32/registry.rb,
329
+ # win32/resolv.rb): loaded by stdlib code only on Windows (resolv.rb
330
+ # needs win32/resolv to find the hosts file), so a build host other than
331
+ # Windows never sees them loaded. They are small; always pack them.
332
+ def add_windows_stdlib
333
+ dir = @runtime.dir / "lib/ruby" / @runtime.ruby_api_version / "win32"
334
+ return unless dir.directory?
335
+
336
+ dir.find do |path|
337
+ next unless path.file? && path.extname == ".rb"
338
+
339
+ add(path.relative_path_from(@runtime.dir).to_s, path)
340
+ end
341
+ end
342
+
343
+ # What only a Windows executable has: a subsystem, resources, DLLs.
344
+ def check_options_for_os
345
+ return if @runtime.windows?
346
+
347
+ runtime = "the #{@runtime.os} runtime #{@runtime.dir}"
348
+ raise "--windows applies to Windows runtimes, not to #{runtime}" if @windowed
349
+ if (@resources || {}).any? { |_, v| v && !(v.respond_to?(:empty?) && v.empty?) }
350
+ raise "--icon and the version-resource options apply to Windows runtimes, not to #{runtime}"
351
+ end
352
+ unless @dlls.empty?
353
+ raise "--runtime-dll: #{runtime} is fully static and cannot load shared libraries"
354
+ end
355
+ end
356
+
357
+ # The license texts of what the runtime links in (LICENSES/ of the
358
+ # runtime directory), so that they travel with every executable:
359
+ # `unzip -l app.exe ocran/licenses/*`.
360
+ def add_licenses
361
+ dir = @runtime.dir / "LICENSES"
362
+ return unless dir.directory?
363
+
364
+ dir.find do |path|
365
+ add("ocran/licenses/#{path.relative_path_from(dir)}", path) if path.file?
366
+ end
367
+ end
368
+
369
+ def finalize
370
+ raise "No script to run was recorded" unless @exec
371
+
372
+ check_options_for_os
373
+ add_windows_stdlib if @runtime.windows?
374
+ add_licenses
375
+ work = Dir.mktmpdir("ocran-runtime")
376
+ begin
377
+ if @runtime.sdk&.section_payload?
378
+ write_linked(work)
379
+ else
380
+ relink = !@unsupported_native.empty? || !@dlls.empty?
381
+ executable = relink ? link_extensions(work) : @runtime.executable
382
+ write(executable)
383
+ end
384
+ ship_dlls
385
+ rescue StandardError
386
+ # A copy of the runtime without its archive would look like a
387
+ # finished executable and fail only when run.
388
+ FileUtils.rm_f(@path.to_s)
389
+ raise
390
+ ensure
391
+ remove_work_dir(work)
392
+ end
393
+ end
394
+
395
+ # MSYS2's tools on Windows can leave a file named NUL in an extension's
396
+ # build directory, which the Win32 API refuses to delete by its plain
397
+ # path; the \\?\ form bypasses device-name parsing. A directory that
398
+ # still cannot be removed is reported, never a reason to fail the build.
399
+ def remove_work_dir(work)
400
+ FileUtils.rm_rf(work)
401
+ return unless File.exist?(work)
402
+
403
+ if Gem.win_platform?
404
+ system("cmd", "/c", "rmdir", "/s", "/q", "\\\\?\\#{File.expand_path(work).tr("/", "\\")}",
405
+ out: File::NULL, err: File::NULL)
406
+ end
407
+ warning "Could not remove the temporary directory #{work}" if File.exist?(work)
408
+ rescue SystemCallError => e
409
+ warning "Could not remove the temporary directory #{work}: #{e.message}"
410
+ end
411
+
412
+ # macOS: the archive is written on its own (offsets counted from its
413
+ # start) and linked into the executable as the section __OCRAN,__zip,
414
+ # together with any extensions; the linker lays the executable out and
415
+ # signs it. Appended data would break the code signature. The
416
+ # extensions are built first: building them changes what is packed
417
+ # (build markers, renamed platform gems, dropped foreign binaries).
418
+ def write_linked(work)
419
+ extensions = build_extensions(work)
420
+ archive = File.join(work, "application.zip")
421
+ File.binwrite(archive, ZipWriter.end_of_central_directory(0, 0, 0))
422
+ ZipWriter.append(archive, archive_entries)
423
+ link_runtime(extensions, work, archive: archive, output: @path.to_s)
424
+ File.chmod(0o755, @path.to_s)
425
+ end
426
+
427
+ # The C extensions of the application's gems that the runtime does not
428
+ # have: built from the gems' sources against the runtime's SDK and
429
+ # linked into a runtime of this application's own. Returns its path.
430
+ def link_extensions(work)
431
+ link_runtime(build_extensions(work), work)
432
+ end
433
+
434
+ # Builds the missing extensions; returns them for #link_runtime.
435
+ def build_extensions(work)
436
+ sdk = @runtime.sdk
437
+ unless sdk
438
+ list = @unsupported_native.map { |name, source| " #{name} (#{source})" }.join("\n")
439
+ raise "These C extensions are not linked into the static runtime #{@runtime.dir}, which has no SDK " \
440
+ "to link them in, and a C extension cannot be loaded from inside the executable:\n#{list}"
441
+ end
442
+ sdk.check_toolchain!
443
+
444
+ @renamed_gems = {}
445
+ by_gem = @unsupported_native.group_by { |_, source| gem_spec_for(source) }
446
+ if (loose = by_gem.delete(nil))
447
+ raise "These C extensions belong to no installed gem, so there is no source to build them " \
448
+ "from for the static runtime:\n#{loose.map { |name, source| " #{name} (#{source})" }.join("\n")}"
449
+ end
450
+
451
+ extensions = by_gem.flat_map do |spec, files|
452
+ if @runtime.gems.key?(spec.name)
453
+ raise "#{spec.full_name} has a C extension the static runtime's own #{spec.name} does not link in " \
454
+ "(#{files.map(&:first).join(", ")})"
455
+ end
456
+ extconfs = spec.extensions
457
+ source_dir = spec.full_gem_path
458
+ gem_path = []
459
+ if extconfs.empty?
460
+ # A precompiled platform gem. Build from the source gem of the same
461
+ # version (a platform gem may leave sources out: nokogiri's lacks
462
+ # its gumbo parser), or else from the sources it carries.
463
+ source_dir = source_gem_dir(spec) || spec.full_gem_path
464
+ gem_path = build_gem_path(source_dir)
465
+ extconfs = Dir.glob("ext/**/extconf.rb", base: source_dir).sort
466
+ if extconfs.empty?
467
+ raise "#{spec.full_name} is a precompiled gem for #{spec.platform} without its C source: install " \
468
+ "the source gem (gem install #{spec.name} --platform ruby)"
469
+ end
470
+ use_as_ruby_platform_gem(spec, work)
471
+ end
472
+ built = extconfs.map do |extconf|
473
+ ext_dir = File.join(source_dir, File.dirname(extconf))
474
+ unless File.file?(File.join(source_dir, extconf))
475
+ raise "#{spec.full_name}: #{extconf} is not installed, so its C extension cannot be built"
476
+ end
477
+ say "Building the C extension #{extconf} of #{spec.full_name} for the static runtime"
478
+ sdk.build_extension(ext_dir, File.join(work, spec.full_name, File.dirname(extconf)),
479
+ "--with-opt-dir=#{sdk.dir / "deps"}", *sdk.extconf_args(spec.name),
480
+ gem_path: gem_path)
481
+ end
482
+ files.each do |name, _|
483
+ stem = name.sub(/\.(so|bundle)\z/, "")
484
+ # A precompiled gem keeps it in a Ruby-version directory
485
+ # (lib/nokogiri/4.0/nokogiri.so for the feature nokogiri/nokogiri).
486
+ next if built.any? { |ext| stem.end_with?("/#{ext.feature}") || File.basename(stem) == File.basename(ext.feature) }
487
+
488
+ raise "Building #{spec.full_name} for the static runtime did not provide #{name} " \
489
+ "(it built #{built.map(&:feature).join(", ")})"
490
+ end
491
+ mark_extensions_built(spec, files)
492
+ built
493
+ end
494
+ end
495
+
496
+ def link_runtime(extensions, work, archive: nil, output: nil)
497
+ sdk = @runtime.sdk
498
+ executable = output || File.join(work, "ocran-runtime.exe")
499
+ say "Linking #{extensions.map(&:feature).join(", ")} into the static runtime" unless extensions.empty?
500
+ say "Linking against the DLL builds of #{@dlls.join(", ")}" unless @dlls.empty?
501
+ say "Linking the application archive into the executable (section __OCRAN,__zip)" if archive
502
+ sdk.link(extensions, executable, dlls: @dlls, archive: archive)
503
+ @dll_files = sdk.dll_files(@dlls)
504
+ Pathname(executable)
505
+ end
506
+
507
+ # The unpacked source gem (platform "ruby") of the version of a
508
+ # precompiled +spec+, fetched from the configured gem sources into
509
+ # ~/.cache/ocran/gems; nil when it cannot be fetched.
510
+ def source_gem_dir(spec)
511
+ cache = File.join(ENV["XDG_CACHE_HOME"] || File.join(Dir.home, ".cache"), "ocran", "gems")
512
+ dir = File.join(cache, "#{spec.name}-#{spec.version}")
513
+ return dir if File.directory?(dir) && File.file?("#{dir}.gemspec")
514
+
515
+ FileUtils.rm_rf(dir) # fetched by an OCRAN that did not keep the spec
516
+
517
+ say "Fetching the source gem of #{spec.name} #{spec.version}: the precompiled #{spec.full_name} is " \
518
+ "built for #{spec.platform}"
519
+ require "rubygems/spec_fetcher"
520
+ require "rubygems/package"
521
+ tuples, = Gem::SpecFetcher.fetcher.spec_for_dependency(Gem::Dependency.new(spec.name, spec.version.to_s))
522
+ remote, source = tuples.find { |s, _| s.platform.to_s == Gem::Platform::RUBY }
523
+ raise "no source gem of #{spec.name} #{spec.version}" unless remote
524
+
525
+ Dir.mktmpdir do |tmp|
526
+ gem_file = source.download(remote, tmp)
527
+ FileUtils.mkdir_p(cache)
528
+ package = Gem::Package.new(gem_file)
529
+ package.extract_files("#{dir}.tmp")
530
+ # Its dependencies may differ from the precompiled gem's (see
531
+ # #build_gem_path).
532
+ File.write("#{dir}.gemspec", package.spec.to_ruby)
533
+ File.rename("#{dir}.tmp", dir)
534
+ end
535
+ dir
536
+ rescue StandardError => e
537
+ warning "Could not fetch the source gem of #{spec.name} #{spec.version} (#{e.message}); " \
538
+ "building from the sources #{spec.full_name} includes"
539
+ FileUtils.rm_rf("#{dir}.tmp") if dir
540
+ nil
541
+ end
542
+
543
+ # Where the extension build of a fetched source gem finds the gems it
544
+ # depends on that the build host lacks: a source gem can depend on more
545
+ # than its precompiled platform gem (nokogiri's extconf.rb needs
546
+ # mini_portile2, which nokogiri-*-x86_64-linux-gnu does not depend on).
547
+ # They are installed, as RubyGems would install them with the source
548
+ # gem, into a directory used only for building extensions. Returns the
549
+ # extra GEM_PATH entries.
550
+ def build_gem_path(source_dir)
551
+ spec_file = "#{source_dir}.gemspec"
552
+ return [] unless File.file?(spec_file)
553
+
554
+ home = File.join(File.dirname(source_dir), "build-gems")
555
+ local = Dir[File.join(home, "specifications", "*.gemspec")].filter_map { |f| Gem::Specification.load(f) }
556
+ missing = Gem::Specification.load(spec_file).runtime_dependencies.reject do |dep|
557
+ Gem::Specification.any? { |s| dep.match?(s) } || local.any? { |s| dep.match?(s) }
558
+ end
559
+ unless missing.empty?
560
+ require "rubygems/dependency_installer"
561
+ missing.each do |dep|
562
+ say "Installing #{dep.name} (#{dep.requirement}) into #{home} for building C extensions"
563
+ Gem::DependencyInstaller.new(install_dir: home, document: [], domain: :remote,
564
+ wrappers: true, env_shebang: true).install(dep.name, dep.requirement)
565
+ end
566
+ end
567
+ File.directory?(home) ? [home] : []
568
+ end
569
+
570
+ # The packed copy of a precompiled platform gem (nokogiri-1.19.4-x86_64-
571
+ # linux-gnu) becomes a platform "ruby" gem (nokogiri-1.19.4): RubyGems
572
+ # on Windows ignores a spec for another platform. Its spec is rewritten
573
+ # and its files are moved to the directory the new name implies.
574
+ def use_as_ruby_platform_gem(spec, work)
575
+ ruby_spec = spec.dup
576
+ ruby_spec.platform = Gem::Platform::RUBY
577
+ return if ruby_spec.full_name == spec.full_name
578
+
579
+ spec_file = File.join(work, "#{ruby_spec.full_name}.gemspec")
580
+ File.write(spec_file, ruby_spec.to_ruby)
581
+ old, new = spec.full_name, ruby_spec.full_name
582
+ @entries = @entries.to_h do |name, source|
583
+ if name.match?(%r{(\A|/)specifications/#{Regexp.escape(old)}\.gemspec\z})
584
+ [name.sub("#{old}.gemspec", "#{new}.gemspec"), Pathname(spec_file)]
585
+ else
586
+ [name.sub(%r{(\A|/)(gems|extensions/[^/]+/[^/]+)/#{Regexp.escape(old)}(/|\z)}) { "#{$1}#{$2}/#{new}#{$3}" }, source]
587
+ end
588
+ end
589
+ @renamed_gems[old] = new
590
+ drop_foreign_binaries(new)
591
+ end
592
+
593
+ NATIVE_MAGIC = ["\x7FELF".b, "MZ".b, "\xCF\xFA\xED\xFE".b, "\xCE\xFA\xED\xFE".b, "\xCA\xFE\xBA\xBE".b].freeze
594
+
595
+ # A precompiled gem carries binaries for its platform - libraries and
596
+ # programs (tiny_tds: ports/x86_64-linux-gnu/lib/libsybdb.so.5,
597
+ # bin/tsql) - that are of no use on Windows; dropped by content, since
598
+ # their names say little.
599
+ def drop_foreign_binaries(full_name)
600
+ @entries.reject! do |name, source|
601
+ next false unless name.match?(%r{(\A|/)gems/#{Regexp.escape(full_name)}/}) && source.is_a?(Pathname)
602
+
603
+ head = File.binread(source.to_s, 4) || ""
604
+ native = NATIVE_MAGIC.any? { |magic| head.start_with?(magic) }
605
+ if native
606
+ skip(:foreign_binary, name)
607
+ @data_size -= File.size(source.to_s)
608
+ end
609
+ native
610
+ end
611
+ end
612
+
613
+ # --runtime-dll: the DLLs go next to the executable, where Windows looks
614
+ # for them first.
615
+ def ship_dlls
616
+ (@dll_files || []).each do |dll|
617
+ target = @path.dirname / dll.basename
618
+ FileUtils.cp(dll.to_s, target.to_s)
619
+ @shipped_dlls << target
620
+ end
621
+ end
622
+
623
+ # The installed gem a C extension file of the build host belongs to.
624
+ def gem_spec_for(source)
625
+ path = File.expand_path(source.to_s)
626
+ specs = Gem.loaded_specs.values + Gem::Specification.to_a
627
+ specs.find do |spec|
628
+ path.start_with?("#{spec.full_gem_path}/") ||
629
+ (!spec.extensions.empty? && path.start_with?("#{spec.extension_dir}/"))
630
+ end
631
+ end
632
+
633
+ # RubyGems activates a gem that declares extensions only when their
634
+ # gem.build_complete marker exists for the platform it runs on - the
635
+ # runtime's, not the build host's.
636
+ def mark_extensions_built(spec, files)
637
+ return if @renamed_gems.key?(spec.full_name) # a platform-"ruby" copy declares no extensions
638
+
639
+ files.each do |name, _|
640
+ base = name[%r{\A(.*?)/?(?:gems|extensions/[^/]+/[^/]+)/#{Regexp.escape(spec.full_name)}/}, 1] or next
641
+ prefix = base.empty? ? "" : "#{base}/"
642
+ @entries["#{prefix}extensions/#{@runtime.gem_platform}/#{@runtime.extension_api_version}/#{spec.full_name}/gem.build_complete"] ||= :empty
643
+ end
644
+ end
645
+
646
+ def write(executable)
647
+ @executable = Pathname(executable)
648
+ FileUtils.cp(@executable.to_s, @path.to_s)
649
+ File.chmod(0o755, @path.to_s)
650
+ set_gui_subsystem if @windowed
651
+ apply_resources
652
+
653
+ File.open(@path.to_s, "ab") do |io|
654
+ # An empty archive at the end of the runtime, so that ZipWriter
655
+ # writes the entries right after it with offsets counted from the
656
+ # start of the file - which is what makes `unzip -l app.exe` work.
657
+ io.write(ZipWriter.end_of_central_directory(0, 0, io.size))
658
+ end
659
+ ZipWriter.append(@path.to_s, archive_entries)
660
+ end
661
+
662
+ def archive_entries
663
+ boot = File.binread(BOOT_SCRIPT)
664
+ config = config_source
665
+ entries = [
666
+ zip_entry("ocran/boot.rb", data: boot),
667
+ zip_entry("ocran/config.rb", data: config),
668
+ ]
669
+ files = @entries.reject { |name, _| name.end_with?("/") }
670
+ # Every directory gets an entry of its own, with the fixed time:
671
+ # ZipWriter would otherwise add the missing ones with the current time.
672
+ dirs = (@entries.keys.select { |name| name.end_with?("/") } +
673
+ (files.keys + ["ocran/"]).flat_map { |name| parent_dirs(name) }).uniq.sort
674
+ dirs.each { |dir| entries << zip_entry(dir) }
675
+ files.sort.each do |name, source|
676
+ entries << zip_entry(name, **(source == :empty ? { data: "" } : { source: source.to_s }))
677
+ end
678
+ entries << zip_entry("ocran/manifest.sha256", data: manifest(boot, config, files))
679
+ end
680
+
681
+ def parent_dirs(name)
682
+ parts = name.split("/")[0...-1]
683
+ parts.each_index.map { |i| "#{parts[0..i].join("/")}/" }
684
+ end
685
+
686
+ # Entries carry a fixed time, so that packing the same inputs twice
687
+ # gives the same bytes (SOURCE_DATE_EPOCH, as other reproducible-build
688
+ # tools use, else 2000-01-01).
689
+ def zip_entry(name, **kw)
690
+ @mtime ||= Time.at(Integer(ENV.fetch("SOURCE_DATE_EPOCH", "946684800"))).utc
691
+ ZipWriter::Entry.new(name: name, mtime: @mtime, **kw)
692
+ end
693
+
694
+ # SHA-256 of the runtime and of every packed file, for auditing:
695
+ # `unzip -p app.exe ocran/manifest.sha256` lists exactly what the
696
+ # executable contains.
697
+ def manifest(boot, config, files)
698
+ runtime =
699
+ if @executable.nil? # macOS: linked together with this archive
700
+ "linked together with this archive, Ruby #{@runtime.ruby_version} #{@runtime.arch}"
701
+ else
702
+ name = @executable == @runtime.executable ? @runtime.executable.basename.to_s : "linked for this application"
703
+ "#{name}, Ruby #{@runtime.ruby_version} #{@runtime.arch} #{Digest::SHA256.file(@executable.to_s).hexdigest}"
704
+ end
705
+ lines = ["# ocran static runtime application manifest (sha256sum format)",
706
+ "# runtime: #{runtime}",
707
+ "#{Digest::SHA256.hexdigest(boot)} ocran/boot.rb",
708
+ "#{Digest::SHA256.hexdigest(config)} ocran/config.rb"]
709
+ files.sort.each do |name, source|
710
+ digest = source == :empty ? Digest::SHA256.hexdigest("") : Digest::SHA256.file(source.to_s).hexdigest
711
+ lines << "#{digest} #{name}"
712
+ end
713
+ lines.join("\n") + "\n"
714
+ end
715
+
716
+ # ocran/config.rb: what boot.rb needs to start the application. Paths
717
+ # that Direction gives relative to the extraction root (|/...) become
718
+ # <ROOT>/..., and lists are joined with the separator of the target
719
+ # (";" on Windows, ":" elsewhere), whatever the build host uses.
720
+ def config_source
721
+ env = @env.dup
722
+ rubylib = split_paths(env.delete("RUBYLIB"))
723
+ rubyopt = env["RUBYOPT"].to_s
724
+ includes, requires, unsupported = ZipPayloadBuilder.parse_rubyopt(rubyopt)
725
+ unsupported.each { |flag| warning "RUBYOPT #{flag} cannot be applied by the static runtime; only -I and -r are" }
726
+
727
+ load_path = rubylib.filter_map { |dir| relative_to_root(dir) }
728
+ .reject { |dir| dir.start_with?("lib/ruby/") } # the runtime's own, below
729
+ load_path += @runtime.load_path
730
+ env = env.to_h do |name, value|
731
+ if %w[GEM_HOME GEM_PATH].include?(name)
732
+ [name, split_paths(value).map { |dir| root_path(dir) }.join(@runtime.windows? ? ";" : ":")]
733
+ else
734
+ [name, root_path(value)]
735
+ end
736
+ end
737
+
738
+ config = {
739
+ main: @exec[0],
740
+ argv: @exec[1].map { |arg| root_path(arg) },
741
+ load_path: load_path.uniq,
742
+ env: env,
743
+ rubyopt_includes: includes.map { |dir| root_path(dir) },
744
+ rubyopt_requires: requires.map { |lib| root_path(lib) },
745
+ chdir_exe_dir: @chdir,
746
+ rubygems: true,
747
+ debug: @debug_mode,
748
+ }
749
+ "# Generated by OCRAN: settings for ocran/boot.rb.\n#{config.inspect}\n"
750
+ end
751
+
752
+ def split_paths(value)
753
+ value.to_s.split(File::PATH_SEPARATOR).reject(&:empty?)
754
+ end
755
+
756
+ # Archive-relative path for an extraction-root path, or nil when it
757
+ # lies outside the application.
758
+ def relative_to_root(path)
759
+ path = path.to_s.tr("\\", "/")
760
+ root = EXTRACT_ROOT.to_s
761
+ return "" if path == root
762
+ return nil unless path.start_with?("#{root}/")
763
+
764
+ path.delete_prefix("#{root}/")
765
+ end
766
+
767
+ def root_path(value)
768
+ value.to_s.gsub(/#{Regexp.escape(EXTRACT_ROOT.to_s)}(?=\/|\z)/, ROOT)
769
+ end
770
+
771
+ # --icon and the version-resource options, written into the copy of the
772
+ # runtime before the archive is appended (pure Ruby, any build host).
773
+ def apply_resources
774
+ r = (@resources || {}).reject { |_, v| v.nil? || (v.respond_to?(:empty?) && v.empty?) }
775
+ return if r.empty?
776
+
777
+ require_relative "pe_resources"
778
+ PeResources.update(@path.to_s) do |pe|
779
+ if r[:icon_path]
780
+ raise "Icon file #{r[:icon_path]} not found" unless File.file?(r[:icon_path])
781
+
782
+ pe.set_icon(File.binread(r[:icon_path]))
783
+ end
784
+ if r[:version_strings] || r[:file_version] || r[:product_version]
785
+ strings = (r[:version_strings] || {}).to_h { |k, v| [k.to_s, v] }
786
+ strings["FileVersion"] = r[:file_version] if r[:file_version]
787
+ strings["ProductVersion"] = r[:product_version] if r[:product_version]
788
+ pe.set_version_info(EdResource::Build.version_info(
789
+ strings: strings,
790
+ file_version: r[:file_version] || strings["FileVersion"] || "0.0.0.0",
791
+ product_version: r[:product_version] || strings["ProductVersion"] || "0.0.0.0"
792
+ ))
793
+ end
794
+ if r[:application_manifest] || r[:execution_level]
795
+ xml = r[:application_manifest] ? File.read(r[:application_manifest], encoding: "UTF-8") : nil
796
+ xml ||= pe.manifest&.dup&.force_encoding(Encoding::UTF_8)
797
+ pe.set_manifest(EdResource::Build.manifest(xml, r[:execution_level]))
798
+ end
799
+ pe.set_strings(r[:resource_strings]) if r[:resource_strings]
800
+ end
801
+ end
802
+
803
+ # --windows: the runtime is a console program; switching the PE
804
+ # subsystem to GUI is all rubyw.exe does differently at the image level.
805
+ def set_gui_subsystem
806
+ File.open(@path.to_s, "r+b") do |io|
807
+ io.seek(0x3c)
808
+ pe = io.read(4).unpack1("V")
809
+ io.seek(pe + 24 + 68) # optional header Subsystem, same in PE32 and PE32+
810
+ io.write([2].pack("v")) # IMAGE_SUBSYSTEM_WINDOWS_GUI
811
+ end
812
+ end
813
+ end
814
+ end