ocran 1.4.4 → 1.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -26,24 +26,115 @@ module Ocran
26
26
 
27
27
  include BuildConstants, CommandOutput, HostConfigHelper
28
28
 
29
+ # Packed name of the interpreter when a cosmopolitan Ruby payload is
30
+ # used (--cosmo-ruby); the source APE is packed under this name.
31
+ COSMO_RUBY_EXE = "ruby.com"
32
+
33
+ # File name extensions of loadable native binaries that a gem may
34
+ # ship or build. None of them can be loaded by the cosmopolitan Ruby
35
+ # payload, which is statically linked and has no dlopen.
36
+ NATIVE_BINARY_EXTENSIONS = %w[.so .bundle .dll].freeze
37
+
38
+ # Native binaries a gem ships or has had built for it, i.e. the files
39
+ # that make it unusable under the cosmopolitan Ruby payload.
40
+ #
41
+ # spec.extensions alone does NOT identify a native gem: a precompiled
42
+ # platform gem (e.g. sqlite3-2.9.5-x86_64-linux-gnu) has an empty
43
+ # extensions array because nothing is compiled at install time, yet it
44
+ # ships a prebuilt sqlite3_native.so inside its lib directory. Both the
45
+ # gem directory and the extension directory (where RubyGems puts the
46
+ # products of a source build) are scanned.
47
+ def self.gem_native_binaries(spec)
48
+ ext_dir =
49
+ begin
50
+ spec.extension_dir
51
+ rescue StandardError
52
+ nil
53
+ end
54
+ pattern = "**/*{#{NATIVE_BINARY_EXTENSIONS.join(",")}}"
55
+ [spec.gem_dir, ext_dir].compact.uniq.flat_map { |dir|
56
+ next [] unless File.directory?(dir)
57
+
58
+ Dir.glob(pattern, base: dir).map { |rel| Pathname(File.join(dir, rel)) }
59
+ }.uniq
60
+ end
61
+
62
+ # The feature names a gem is required by, i.e. what has to answer
63
+ # inside the payload for the payload's own copy to serve in place of
64
+ # the host gem. RubyGems' convention maps a dash in a gem name to a
65
+ # directory separator in its primary feature (io-console provides
66
+ # "io/console"), and both spellings are seen in the wild, so both are
67
+ # offered as candidates.
68
+ def self.cosmo_gem_features(spec)
69
+ [spec.name, spec.name.tr("-", "/")].uniq
70
+ end
71
+
72
+ # Decides how a gem detected on the build host has to be treated when
73
+ # a cosmopolitan Ruby payload is packed (--cosmo-ruby). Returns a pair
74
+ # of a disposition and the gem's native binaries:
75
+ #
76
+ # [:pack, []] pure Ruby gem, pack it as usual
77
+ # [:payload_provides, files] native, but the payload provides the
78
+ # same library itself: skip the host copy
79
+ # and let the payload's own version serve
80
+ # [:incompatible, files] native and not provided by the payload:
81
+ # the build must fail
82
+ #
83
+ # A gem counts as provided by the payload when the payload has a
84
+ # gemspec of that name, OR when the payload can resolve the gem's
85
+ # primary feature (+provides_feature+, normally
86
+ # CosmoToolchain.resolvable_features against the payload). The second
87
+ # test matters because a gemspec is not what makes a library
88
+ # requirable: an extension statically linked into the APE, or a
89
+ # library in its embedded stdlib rather than in /zip/lib/ruby/gems,
90
+ # answers require with no gemspec at all. Keying only on gemspec names
91
+ # reports such a library as incompatible and refuses a build that would
92
+ # have worked - which is what happens to cgi and pathname, both
93
+ # compiled into the interpreter and both regular native gems on a
94
+ # recent host Ruby.
95
+ def self.cosmo_gem_disposition(spec, payload_gem_names, provides_feature = nil)
96
+ native_files = gem_native_binaries(spec)
97
+ return [:pack, native_files] if spec.extensions.empty? && native_files.empty?
98
+
99
+ if payload_gem_names.include?(spec.name) ||
100
+ (provides_feature && provides_feature.call(cosmo_gem_features(spec)))
101
+ [:payload_provides, native_files]
102
+ else
103
+ [:incompatible, native_files]
104
+ end
105
+ end
106
+
107
+ # Human readable reason why a gem counts as native, for build messages.
108
+ def self.cosmo_native_reason(spec, native_files)
109
+ reasons = []
110
+ reasons << "declares native extensions" if spec.extensions.any?
111
+ if native_files.any?
112
+ names = native_files.map { |file| File.basename(file) }.uniq
113
+ reasons << "ships prebuilt binaries (#{names.join(", ")})"
114
+ end
115
+ reasons.join(" and ")
116
+ end
117
+
29
118
  attr_reader :ruby_executable, :rubyopt
30
119
 
31
120
  def initialize(post_env, pre_env, option)
32
121
  @post_env, @pre_env, @option = post_env, pre_env, option
33
- @ruby_executable = @option.windowed? ? rubyw_exe : ruby_exe
122
+ @ruby_executable =
123
+ if @option.cosmo_ruby
124
+ COSMO_RUBY_EXE
125
+ else
126
+ @option.windowed? ? rubyw_exe : ruby_exe
127
+ end
34
128
 
35
129
  # Initializes @rubyopt with the user-intended RUBYOPT environment variable.
36
130
  # This ensures that RUBYOPT matches the user's initial settings before any
37
131
  # modifications that may occur during script execution.
132
+ #
133
+ # -I and -r entries that refer to build-machine paths (including the
134
+ # `-r<abs path>/bundler/setup` that Bundler adds under `bundle exec`
135
+ # since Ruby 3.2) are translated or removed at build time by
136
+ # RubyoptProcessor in #construct.
38
137
  @rubyopt = @option.rubyopt || pre_env.env["RUBYOPT"] || ""
39
-
40
- # Remove any absolute path to bundler/setup from RUBYOPT.
41
- # When building under `bundle exec`, RUBYOPT contains `-r/absolute/path/bundler/setup`.
42
- # That path doesn't exist inside the packed executable's environment, causing Ruby to
43
- # print "RubyGems were not loaded" / "did_you_mean was not loaded" warnings on startup.
44
- # We strip the flag regardless of install prefix because the gem may live in a user gem
45
- # directory that doesn't share a prefix with RbConfig::TOPDIR (e.g. on CI runners).
46
- @rubyopt = @rubyopt.gsub(/-r\S*\/bundler\/setup/, "").strip
47
138
  end
48
139
 
49
140
  # Resolves the common root directory prefix from an array of absolute paths.
@@ -72,7 +163,16 @@ module Ocran
72
163
 
73
164
  def detect_dlls
74
165
  if Gem.win_platform?
75
- require_relative "library_detector"
166
+ begin
167
+ require_relative "library_detector"
168
+ rescue LoadError => e
169
+ # LibraryDetector needs fiddle, a bundled gem since Ruby 3.5. In a
170
+ # Bundler context (e.g. building with --gemfile) requiring it is
171
+ # refused unless the Gemfile lists fiddle, so degrade to no DLL
172
+ # auto-detection instead of aborting the build.
173
+ warning "DLL auto-detection disabled (#{e.message}). Add fiddle to the Gemfile, or use --dll to include DLLs manually."
174
+ return []
175
+ end
76
176
  else
77
177
  require_relative "library_detector_posix"
78
178
  end
@@ -91,7 +191,8 @@ module Ocran
91
191
  end
92
192
  end
93
193
  if defined?(Gem)
94
- specs += Gem.loaded_specs.values
194
+ foreign = foreign_bundle_gem_names
195
+ specs += Gem.loaded_specs.each_value.reject { |spec| foreign.include?(spec.name) }
95
196
  # Now, we also detect gems that are not included in Gem.loaded_specs.
96
197
  # Therefore, we look for any loaded file from a gem path.
97
198
  specs += GemSpecQueryable.detect_gems_from(features, verbose: @option.verbose?)
@@ -101,6 +202,83 @@ module Ocran
101
202
  specs
102
203
  end
103
204
 
205
+ # The gems RubyGems had already activated for a bundle that is not the
206
+ # application's, by the time OCRAN started.
207
+ #
208
+ # `bundle exec` activates every gem of its bundle before the command it
209
+ # runs executes a single line, so Gem.loaded_specs describes the build
210
+ # environment as much as the application. When the two bundles are the
211
+ # same - `bundle exec ocran app.rb` from the application's own directory,
212
+ # the ordinary case - that is exactly right and nothing is dropped. When
213
+ # they differ, packing the build environment's bundle adds tens of
214
+ # megabytes of code the application can never load: the packaged app runs
215
+ # under its own Gemfile, so gems from a foreign bundle are dead weight
216
+ # even when they are packed.
217
+ #
218
+ # Only activation is discounted, not use: anything the dependency run
219
+ # actually loaded is still found through $LOADED_FEATURES by
220
+ # detect_gems_from, and everything the application's Gemfile names is
221
+ # added by the Gemfile scan. This is the same rule that already applies
222
+ # to a build outside Bundler, where --gemfile is what pulls in gems the
223
+ # dependency run does not load.
224
+ def foreign_bundle_gem_names
225
+ @foreign_bundle_gem_names ||=
226
+ if foreign_build_bundle?
227
+ verbose "Ignoring #{@pre_env.activated_gems.size} gems activated by the build environment's bundle " \
228
+ "#{@pre_env.env["BUNDLE_GEMFILE"]}"
229
+ @pre_env.activated_gems.to_set
230
+ else
231
+ Set.new
232
+ end
233
+ end
234
+
235
+ # Whether OCRAN itself was started under a bundle other than the one the
236
+ # application runs under.
237
+ def foreign_build_bundle?
238
+ return false unless @pre_env.bundler_setup_loaded?
239
+
240
+ build_gemfile = @pre_env.env["BUNDLE_GEMFILE"]
241
+ return false if build_gemfile.nil? || build_gemfile.empty?
242
+
243
+ app_gemfile = @option.application_gemfile
244
+ return true if app_gemfile.nil?
245
+
246
+ !same_file?(build_gemfile, app_gemfile)
247
+ end
248
+
249
+ def same_file?(a, b)
250
+ File.identical?(a, b) || File.expand_path(a) == File.expand_path(b)
251
+ end
252
+
253
+ # Packed name of the file BUNDLER_SETUP is pointed at.
254
+ BUNDLER_SETUP_NOOP = Pathname("no_bundler_setup.rb")
255
+
256
+ # Keeps the environment of whoever launches the packaged application
257
+ # from dragging Bundler into it.
258
+ #
259
+ # A packaged application carries its own gems and its own Gemfile; the
260
+ # bundle of the machine it is started from means nothing to it, and
261
+ # anything of that bundle that survives into the process is fatal rather
262
+ # than merely wrong - Bundler aborts with GemNotFound as soon as it
263
+ # cannot materialize gems that were never packed. RUBYOPT is already
264
+ # overwritten wholesale, but that alone stopped being enough: RubyGems
265
+ # now requires the file named by BUNDLER_SETUP at interpreter startup,
266
+ # which is how current Bundler versions set a process up, and
267
+ # BUNDLE_GEMFILE would still send the application's own
268
+ # `require "bundler/setup"` at the wrong Gemfile.
269
+ #
270
+ # BUNDLER_SETUP names a file to require, so it cannot simply be blanked
271
+ # - an empty value is still truthy and RubyGems would raise trying to
272
+ # require it. It is pointed at an empty packed file instead. Bundler
273
+ # does treat empty BUNDLE_GEMFILE and BUNDLE_LOCKFILE as unset, which is
274
+ # what lets the application find the Gemfile packed beside it.
275
+ def neutralize_bundler_env(builder)
276
+ builder.touch(BUNDLER_SETUP_NOOP)
277
+ builder.set_env_path("BUNDLER_SETUP", BUNDLER_SETUP_NOOP)
278
+ builder.export("BUNDLE_GEMFILE", "")
279
+ builder.export("BUNDLE_LOCKFILE", "")
280
+ end
281
+
104
282
  def normalized_features
105
283
  features = @post_env.loaded_features.map { |feature| Pathname(feature) }
106
284
 
@@ -114,10 +292,12 @@ module Ocran
114
292
  # because rubygems.rb uses require_relative to load it.
115
293
  kernel_require_rel = "rubygems/core_ext/kernel_require.rb"
116
294
  unless features.any? { |f| f.to_posix.end_with?(kernel_require_rel) }
117
- # Prefer the location alongside the actually-loaded rubygems.rb, fall back to rubylibdir
118
- rubygems_feature = features.find { |f| f.to_posix.end_with?("/rubygems.rb") }
119
- candidate_dirs = []
120
- candidate_dirs << rubygems_feature.dirname if rubygems_feature
295
+ # Prefer the location alongside the actually-loaded rubygems.rb, fall back to
296
+ # rubylibdir. Consider every feature ending in "/rubygems.rb", because a plain
297
+ # suffix match can also hit unrelated files such as bundler's
298
+ # lib/bundler/source/rubygems.rb (loaded before rubygems.rb under bundle exec);
299
+ # the existence check below skips candidates without the core_ext file.
300
+ candidate_dirs = features.select { |f| f.to_posix.end_with?("/rubygems.rb") }.map(&:dirname)
121
301
  candidate_dirs << Pathname(RbConfig::CONFIG["rubylibdir"])
122
302
  candidate_dirs.each do |base_dir|
123
303
  kernel_require_path = base_dir / kernel_require_rel
@@ -147,17 +327,54 @@ module Ocran
147
327
  end
148
328
  end
149
329
 
330
+ # True when the packed cosmopolitan Ruby can resolve any of the given
331
+ # features itself, i.e. when a host gem providing them does not have
332
+ # to be packed. Answers are memoized per feature: probing costs one
333
+ # run of the payload interpreter, and only native gems the payload has
334
+ # no gemspec for ever get here.
335
+ def cosmo_payload_provides_feature?(features)
336
+ @cosmo_feature_cache ||= {}
337
+ unknown = features.reject { |feature| @cosmo_feature_cache.key?(feature) }
338
+ unless unknown.empty?
339
+ resolved = CosmoToolchain.resolvable_features(@option.cosmo_ruby, unknown)
340
+ unknown.each { |feature| @cosmo_feature_cache[feature] = resolved.include?(feature) }
341
+ end
342
+ features.any? { |feature| @cosmo_feature_cache[feature] }
343
+ end
344
+
150
345
  def construct(builder)
151
346
  # Store the currently loaded files
152
347
  features = normalized_features
153
348
 
349
+ # With --cosmo-ruby, run the payload interpreter once on the build
350
+ # host: this validates that it works, provides its embedded gem
351
+ # directory (needed for GEM_PATH below) and its version for the
352
+ # host-vs-payload skew warning. Dependency detection has already
353
+ # run under the *host* Ruby, so stdlib/gem resolution may differ
354
+ # when the versions diverge.
355
+ if @option.cosmo_ruby
356
+ # Kernel#load, matching Option#load_cosmo_toolchain: the file may
357
+ # already have been loaded that way at option-parse time, and
358
+ # require_relative would then run it a second time.
359
+ load File.expand_path("cosmo_toolchain.rb", __dir__) unless defined? CosmoToolchain
360
+ @cosmo_ruby_info = CosmoToolchain.query_ruby(@option.cosmo_ruby)
361
+ say "Packaging cosmopolitan Ruby #{@cosmo_ruby_info[:version]} (#{@option.cosmo_ruby})"
362
+ if RUBY_VERSION.split(".").take(2) != @cosmo_ruby_info[:version].split(".").take(2)
363
+ warning "Dependency detection ran under the host Ruby #{RUBY_VERSION}, but the packed cosmopolitan Ruby is #{@cosmo_ruby_info[:version]}; stdlib and gem behavior may differ between these versions"
364
+ end
365
+ end
366
+
154
367
  # If net/http was loaded but openssl wasn't (it is only required lazily
155
368
  # at the point of an actual HTTPS connection), require it now inside the
156
369
  # OCRAN build process so that every transitive dependency — openssl.rb,
157
370
  # digest.so, and any other files pulled in by the extension — appears in
158
371
  # $LOADED_FEATURES and gets bundled alongside the application.
372
+ # Skipped with --cosmo-ruby: the payload interpreter carries its own
373
+ # (statically linked) openssl, and the host's files would be excluded
374
+ # from the package anyway.
159
375
  openssl_so = Pathname(RbConfig::CONFIG["archdir"]) / "openssl.so"
160
- if openssl_so.exist? &&
376
+ if !@option.cosmo_ruby &&
377
+ openssl_so.exist? &&
161
378
  features.any? { |f| f.to_posix.end_with?("/net/http.rb") } &&
162
379
  features.none? { |f| f == openssl_so }
163
380
  say "Auto-loading openssl (net/http loaded but openssl not yet required)"
@@ -175,53 +392,94 @@ module Ocran
175
392
 
176
393
  # Add the ruby executable and DLL
177
394
  say "Adding ruby executable #{ruby_executable}"
178
- ruby_source = bindir / ruby_executable
179
- if !Gem.win_platform? && File.binread(ruby_source, 2) == "#!"
180
- # On some distros (e.g. Fedora), bindir/ruby is a dispatcher shell
181
- # script ("rubypick") rather than the interpreter itself, which
182
- # cannot run on a system without Ruby. Pack the currently running
183
- # interpreter binary under the expected name instead.
184
- real_ruby = Pathname("/proc/self/exe")
185
- raise "#{ruby_source} is a wrapper script and the real interpreter could not be determined" unless real_ruby.exist?
186
-
187
- say "#{ruby_source} is a wrapper script; packing #{real_ruby.realpath} instead"
188
- builder.copy_to_bin(real_ruby.realpath, ruby_executable)
395
+ if @option.cosmo_ruby
396
+ # The cosmopolitan Ruby APE is fully self-contained: a static
397
+ # binary with the standard library embedded in its ZIP store
398
+ # (/zip/lib/ruby/...). No libruby, no shared libraries and no
399
+ # LD_LIBRARY_PATH are needed pack the single file and be done.
400
+ #
401
+ # Except in ZIP packaging mode, where the output IS that binary and
402
+ # the application is injected into it: packing a copy of the
403
+ # interpreter into itself would double the size of the executable
404
+ # for nothing.
405
+ if @option.cosmo_zip?
406
+ say "Injecting the application into the ZIP store of #{@option.cosmo_ruby}"
407
+ else
408
+ builder.copy_to_bin(Pathname(@option.cosmo_ruby), ruby_executable)
409
+ end
189
410
  else
190
- builder.copy_to_bin(ruby_source, ruby_executable)
191
- end
192
- if libruby_so
193
- # On POSIX systems, libruby.so is in libdir; on Windows, it's in bindir
194
- libruby_src = Gem.win_platform? ? bindir / libruby_so : libdir / libruby_so
195
- builder.copy_to_bin(libruby_src, libruby_so)
196
-
197
- # On POSIX systems, create symlinks (aliases) for libruby.so
198
- unless Gem.win_platform?
199
- libruby_aliases.each do |libruby_alias|
200
- builder.symlink_in_bin(libruby_so, libruby_alias)
411
+ ruby_source = bindir / ruby_executable
412
+ if !Gem.win_platform? && File.binread(ruby_source, 2) == "#!"
413
+ # On some distros (e.g. Fedora), bindir/ruby is a dispatcher shell
414
+ # script ("rubypick") rather than the interpreter itself, which
415
+ # cannot run on a system without Ruby. Pack the currently running
416
+ # interpreter binary under the expected name instead.
417
+ real_ruby = Pathname("/proc/self/exe")
418
+ raise "#{ruby_source} is a wrapper script and the real interpreter could not be determined" unless real_ruby.exist?
419
+
420
+ say "#{ruby_source} is a wrapper script; packing #{real_ruby.realpath} instead"
421
+ builder.copy_to_bin(real_ruby.realpath, ruby_executable)
422
+ else
423
+ builder.copy_to_bin(ruby_source, ruby_executable)
424
+ end
425
+ if libruby_so
426
+ # On POSIX systems, libruby.so is in libdir; on Windows, it's in bindir
427
+ libruby_src = Gem.win_platform? ? bindir / libruby_so : libdir / libruby_so
428
+ builder.copy_to_bin(libruby_src, libruby_so)
429
+
430
+ # On POSIX systems, create symlinks (aliases) for libruby.so
431
+ unless Gem.win_platform?
432
+ libruby_aliases.each do |libruby_alias|
433
+ builder.symlink_in_bin(libruby_so, libruby_alias)
434
+ end
201
435
  end
202
436
  end
203
- end
204
437
 
205
- # On POSIX systems, set LD_LIBRARY_PATH to find bundled shared libraries
206
- unless Gem.win_platform?
207
- extract_bin = File.join(EXTRACT_ROOT, BINDIR.to_s)
208
- builder.export("LD_LIBRARY_PATH", extract_bin)
209
- if RUBY_PLATFORM.include?("darwin")
210
- builder.export("DYLD_LIBRARY_PATH", extract_bin)
438
+ # On POSIX systems, set LD_LIBRARY_PATH to find bundled shared libraries
439
+ unless Gem.win_platform?
440
+ extract_bin = File.join(EXTRACT_ROOT, BINDIR.to_s)
441
+ builder.export("LD_LIBRARY_PATH", extract_bin)
442
+ if RUBY_PLATFORM.include?("darwin")
443
+ builder.export("DYLD_LIBRARY_PATH", extract_bin)
444
+ end
211
445
  end
212
446
  end
213
447
 
214
448
  # Windows-only: Add detected DLLs
215
449
  if Gem.win_platform? && @option.auto_detect_dlls?
450
+ # The Windows loader resolves the imports of a native extension from
451
+ # the extension's own directory, the application directory of the
452
+ # packed ruby.exe (bin) plus its SxS assembly (ruby_builtin_dlls),
453
+ # and the system directories. PATH is not consulted on hardened
454
+ # systems, and the AddDllDirectory mechanism gems use through
455
+ # ruby_installer/runtime is not available in a packed app. A detected
456
+ # DLL that lives anywhere else - a gem's bundled library (e.g.
457
+ # FreeTDS under tiny_tds' ports/), a devkit's msys64 tree inside the
458
+ # Ruby prefix, or a directory outside the prefix entirely - is packed
459
+ # only at a location the loader never searches, and the application
460
+ # dies with LoadError on machines where nothing masks the gap. So
461
+ # additionally bundle a copy of every such DLL into bin, next to
462
+ # ruby.exe, mirroring what the Linux branch below achieves with
463
+ # LD_LIBRARY_PATH. DLLs from the Windows directory always come from
464
+ # the target system and are never bundled.
465
+ windows_dir = Pathname(ENV["SystemRoot"] || "C:/Windows")
466
+ dlls_in_bin = Set.new
467
+ add_dll_to_bin = proc do |dll|
468
+ next if dll.subpath?(bindir) || !dlls_in_bin.add?(dll.basename.to_s.downcase)
469
+
470
+ say "Adding detected DLL #{dll} to bin"
471
+ builder.copy_to_bin(dll, dll.basename)
472
+ end
473
+
216
474
  detect_dlls.each do |dll|
217
- next unless dll.subpath?(exec_prefix) && dll.extname?(".dll") && dll.basename != libruby_so
475
+ next unless dll.extname?(".dll") && dll.basename != libruby_so
476
+ next if dll.subpath?(windows_dir)
218
477
 
219
- say "Adding detected DLL #{dll}"
220
478
  if dll.subpath?(exec_prefix)
479
+ say "Adding detected DLL #{dll}"
221
480
  builder.duplicate_to_exec_prefix(dll)
222
- else
223
- builder.copy_to_bin(dll, dll.basename)
224
481
  end
482
+ add_dll_to_bin.call(dll)
225
483
  end
226
484
 
227
485
  # Proactively include companion DLLs for loaded native extensions.
@@ -229,7 +487,10 @@ module Ocran
229
487
  # directory (e.g., libssl-3-x64.dll alongside openssl.so) that are
230
488
  # loaded lazily on first use. Scanning .so directories ensures those
231
489
  # DLLs are bundled even when the extension is required but not
232
- # exercised during the OCRAN dependency scan.
490
+ # exercised during the OCRAN dependency scan. They also go into bin:
491
+ # a copy in archdir only helps extensions in archdir itself, while
492
+ # the same extension packed at a gem path (openssl and psych are
493
+ # gems since Ruby 3.x) resolves its imports from bin.
233
494
  features.select { |f| f.extname?(".so") && f.subpath?(exec_prefix) }
234
495
  .map(&:dirname).uniq
235
496
  .each do |dir|
@@ -237,6 +498,7 @@ module Ocran
237
498
  next unless path.file? && path.extname?(".dll")
238
499
  say "Adding companion DLL #{path}"
239
500
  builder.duplicate_to_exec_prefix(path)
501
+ add_dll_to_bin.call(path)
240
502
  end
241
503
  end
242
504
  end
@@ -247,8 +509,8 @@ module Ocran
247
509
  # points LD_LIBRARY_PATH at the packed bin directory). Core glibc
248
510
  # libraries and the loader are never bundled - they must come from the
249
511
  # target system. Ruby native extensions are packed as features, not
250
- # here.
251
- if RUBY_PLATFORM.include?("linux") && @option.auto_detect_dlls?
512
+ # here. Not needed with --cosmo-ruby: the APE payload is static.
513
+ if RUBY_PLATFORM.include?("linux") && @option.auto_detect_dlls? && !@option.cosmo_ruby
252
514
  feature_set = features.to_set
253
515
  feature_realpaths = features.filter_map { |f| f.realpath rescue nil }.to_set
254
516
  # Ruby native extensions live in these directories and are packed as
@@ -319,6 +581,11 @@ module Ocran
319
581
  end
320
582
  end
321
583
 
584
+ # Gem directories whose packing was skipped because the cosmopolitan
585
+ # Ruby payload provides the gem itself; loaded features from these
586
+ # directories must not be packed either.
587
+ cosmo_skipped_gem_dirs = []
588
+
322
589
  # Searches for features that are loaded from gems, then produces a
323
590
  # list of files included in those gems' manifests. Also returns a
324
591
  # list of original features that caused those gems to be included.
@@ -333,13 +600,80 @@ module Ocran
333
600
  next []
334
601
  end
335
602
 
603
+ if @option.cosmo_ruby
604
+ # Default gems of the *host* Ruby are part of its stdlib; the
605
+ # cosmopolitan Ruby ships its own stdlib and default/bundled
606
+ # gems in its embedded ZIP store, so do not pack them (a host
607
+ # 3.x copy would shadow the payload's version).
608
+ if spec.respond_to?(:default_gem?) && spec.default_gem?
609
+ verbose "Skipping default gem #{spec.full_name} (provided by the cosmopolitan Ruby's embedded stdlib)"
610
+ next []
611
+ end
612
+ # Native gems compile (or were precompiled) against a host Ruby
613
+ # ABI and platform; they cannot load under the x86_64-cosmo
614
+ # payload, which is statically linked and cannot dlopen. This
615
+ # covers both source-installed gems (spec.extensions) and
616
+ # precompiled platform gems, which declare no extensions but
617
+ # ship their .so inside the gem directory.
618
+ # When the payload provides the same gem itself (e.g. json,
619
+ # psych are statically linked into the APE), skip the host copy
620
+ # so the payload's own version is used — packing the host .rb
621
+ # files would shadow the payload's and could mismatch the
622
+ # linked-in C extension. Otherwise fail clearly rather than
623
+ # produce a broken executable.
624
+ disposition, native_files = self.class.cosmo_gem_disposition(
625
+ spec, @cosmo_ruby_info[:gem_names], method(:cosmo_payload_provides_feature?)
626
+ )
627
+ if disposition != :pack
628
+ reason = self.class.cosmo_native_reason(spec, native_files)
629
+ if disposition == :payload_provides
630
+ provided =
631
+ if @cosmo_ruby_info[:gem_names].include?(spec.name)
632
+ "its own #{spec.name}"
633
+ else
634
+ "#{spec.name} without a gemspec (linked in, or part of its embedded stdlib)"
635
+ end
636
+ say "Skipping native gem #{spec.full_name} (#{reason}): the cosmopolitan Ruby provides #{provided}"
637
+ cosmo_skipped_gem_dirs << Pathname(spec.gem_dir) if File.directory?(spec.gem_dir)
638
+ ext_dir =
639
+ begin
640
+ spec.extension_dir
641
+ rescue StandardError
642
+ nil
643
+ end
644
+ cosmo_skipped_gem_dirs << Pathname(ext_dir) if ext_dir && File.directory?(ext_dir)
645
+ next []
646
+ end
647
+ raise "Gem #{spec.full_name} is native (#{reason}) and cannot run under the packed cosmopolitan Ruby (x86_64-cosmo, static): exclude the gem or package without --cosmo-ruby"
648
+ end
649
+ end
650
+
336
651
  # Add gemspec files
652
+ local_gem_dir = nil
337
653
  if spec_file.subpath?(exec_prefix)
338
654
  builder.duplicate_to_exec_prefix(spec_file)
339
655
  elsif (gem_path = GemSpecQueryable.find_gem_path(spec_file))
340
656
  builder.duplicate_to_gem_home(spec_file, gem_path)
341
657
  else
342
- raise "Gem spec #{spec_file} does not exist in the Ruby installation. Don't know where to put it."
658
+ # Local development gems (Bundler `gemspec` or `path:` directives)
659
+ # keep their gemspec inside the project tree, outside both the Ruby
660
+ # installation and every gem path, so there is no installed gem
661
+ # layout to mirror. Pack them into GEMDIR as if they were installed
662
+ # there: generate the spec from the in-memory specification (the
663
+ # on-disk gemspec often uses dynamic constructs such as
664
+ # `git ls-files` that would fail in the packed app) and pack the
665
+ # gem's files under gems/<full_name>/ below.
666
+ say "Including local development gem #{spec.full_name} from #{spec_file.dirname}"
667
+ local_gem_dir = Pathname(spec.gem_dir)
668
+ builder.copy_to_gem(generate_gemspec_file(spec), Pathname("specifications") / "#{spec.full_name}.gemspec")
669
+ # RubyGems refuses to activate a gem with extensions unless its
670
+ # gem.build_complete marker exists. The extension files themselves
671
+ # are packed via the loaded features or the extension-dir mirroring
672
+ # below.
673
+ if spec.extensions.any?
674
+ api_version = Gem.respond_to?(:extension_api_version) ? Gem.extension_api_version : Gem.ruby_api_version
675
+ builder.touch(GEMDIR / "extensions" / Gem::Platform.local.to_s / api_version / spec.full_name / "gem.build_complete")
676
+ end
343
677
  end
344
678
 
345
679
  spec_dir = spec_file.dirname
@@ -399,6 +733,20 @@ module Ocran
399
733
  verbose "\t:files (resource_files) count: #{resource_count}"
400
734
 
401
735
  actual_files = spec.find_gem_files(include, features)
736
+
737
+ # Safety net: gems reaching this point are pure Ruby as far as
738
+ # their gem and extension directories go (see the disposition
739
+ # check above), but a file list can still pull in a native binary
740
+ # from elsewhere. It cannot load under the cosmopolitan payload,
741
+ # so exclude it loudly.
742
+ if @option.cosmo_ruby
743
+ native_files = actual_files.select { |f| NATIVE_BINARY_EXTENSIONS.any? { |ext| f.extname?(ext) } }
744
+ if native_files.any?
745
+ warning "Gem #{spec.full_name} contains native binaries that cannot run under the packed cosmopolitan Ruby; excluding: #{native_files.map(&:basename).join(", ")}"
746
+ actual_files -= native_files
747
+ end
748
+ end
749
+
402
750
  say "\t#{actual_files.size} files, #{actual_files.sum(0, &:size)} bytes"
403
751
 
404
752
  # Decide where to put gem files, either the system gem folder, or
@@ -408,6 +756,10 @@ module Ocran
408
756
  builder.duplicate_to_exec_prefix(gemfile)
409
757
  elsif (gem_path = GemSpecQueryable.find_gem_path(gemfile))
410
758
  builder.duplicate_to_gem_home(gemfile, gem_path)
759
+ elsif local_gem_dir && gemfile.subpath?(local_gem_dir)
760
+ # Mirror local development gem files into the packed GEM_HOME
761
+ # under the gem directory matching the generated specification.
762
+ builder.copy_to_gem(gemfile, Pathname("gems") / spec.full_name / gemfile.relative_path_from(local_gem_dir))
411
763
  else
412
764
  raise "Don't know where to put gemfile #{gemfile}"
413
765
  end
@@ -481,7 +833,9 @@ module Ocran
481
833
  end
482
834
 
483
835
  # If requested, add all ruby standard libraries
484
- if @option.add_all_core?
836
+ if @option.add_all_core? && @option.cosmo_ruby
837
+ say "Skipping host core libraries (--add-all-core): the cosmopolitan Ruby embeds its own standard library"
838
+ elsif @option.add_all_core?
485
839
  say "Will include all ruby core libraries"
486
840
  all_core_dir.each do |path|
487
841
  # Match the load path against standard library, site_ruby, and vendor_ruby paths
@@ -497,7 +851,10 @@ module Ocran
497
851
  end
498
852
 
499
853
  # Include encoding support files
500
- if @option.add_all_encoding?
854
+ if @option.cosmo_ruby
855
+ # Encoding extensions are statically linked into the payload.
856
+ say "Encoding support is embedded in the cosmopolitan Ruby"
857
+ elsif @option.add_all_encoding?
501
858
  @post_env.load_path.each do |load_path|
502
859
  load_path = Pathname(@post_env.expand_path(load_path))
503
860
  next unless load_path.subpath?(exec_prefix)
@@ -535,6 +892,23 @@ module Ocran
535
892
  pre_working_directory = Pathname(@pre_env.pwd)
536
893
  working_directory = Pathname(@post_env.pwd)
537
894
  features.each do |feature|
895
+ # With --cosmo-ruby, files of the host Ruby installation must not
896
+ # be packed: the payload interpreter resolves the standard library
897
+ # from its embedded ZIP store, and a packed host-version copy (or
898
+ # a host-ABI native extension) would be wrong for it.
899
+ if @option.cosmo_ruby
900
+ if feature.subpath?(exec_prefix)
901
+ verbose "\tlibfile: #{feature} -> skipped (host Ruby installation; the cosmopolitan Ruby uses its embedded stdlib)"
902
+ next
903
+ elsif cosmo_skipped_gem_dirs.any? { |dir| feature.subpath?(dir) }
904
+ verbose "\tlibfile: #{feature} -> skipped (gem provided by the cosmopolitan Ruby)"
905
+ next
906
+ elsif feature.extname?(".so") || feature.extname?(".bundle")
907
+ warning "Excluding native extension file #{feature}: native extensions cannot run under the packed cosmopolitan Ruby"
908
+ next
909
+ end
910
+ end
911
+
538
912
  load_path = @post_env.find_load_path(feature)
539
913
  if load_path.nil?
540
914
  verbose "\tlibfile: #{feature} -> src (no load path)"
@@ -649,8 +1023,24 @@ module Ocran
649
1023
  end
650
1024
  end
651
1025
 
1026
+ # Translate -I and -r entries in RUBYOPT that refer to build-machine
1027
+ # paths into the packed layout (GitHub issue #20). Ruby cannot parse
1028
+ # quoted arguments inside RUBYOPT and the runtime extraction directory
1029
+ # may contain spaces, so instead of rewriting the paths inside RUBYOPT
1030
+ # the translated entries are applied by a generated launcher script
1031
+ # (see generate_rubyopt_launcher) where they become plain Ruby string
1032
+ # literals.
1033
+ require_relative "rubyopt_processor"
1034
+ rubyopt_result = RubyoptProcessor.new(rubyopt).translate do |path|
1035
+ packed_rubyopt_path(Pathname(path).cleanpath, inst_src_prefix)
1036
+ end
1037
+ rubyopt_result.dropped.each do |entry|
1038
+ say "Removing #{entry} from RUBYOPT (path is not part of the package)"
1039
+ end
1040
+
652
1041
  # Set environment variable
653
- builder.export("RUBYOPT", rubyopt)
1042
+ builder.export("RUBYOPT", rubyopt_result.rubyopt)
1043
+ neutralize_bundler_env(builder)
654
1044
  # Add the load path that are required with the correct path after
655
1045
  # src_prefix was adjusted.
656
1046
  load_path = src_load_path.map { |path| SRCDIR / path.relative_path_from(inst_src_prefix) }.uniq
@@ -660,7 +1050,9 @@ module Ocran
660
1050
  # host, which doesn't exist on other systems (e.g., Docker with no Ruby).
661
1051
  # By adding the extract-dir equivalents of rubylibdir, sitelibdir, etc. to
662
1052
  # RUBYLIB, Ruby can find rubygems and the standard library in the packed tree.
663
- unless Gem.win_platform?
1053
+ # Not with --cosmo-ruby: the host stdlib is not packed at all, and the
1054
+ # payload finds its own stdlib in its embedded ZIP store.
1055
+ unless Gem.win_platform? || @option.cosmo_ruby
664
1056
  # Use the build Ruby's actual default load path in addition to the
665
1057
  # RbConfig directories: some distros compile in extra entries that
666
1058
  # RbConfig does not expose (e.g. Fedora's /usr/share/rubygems, where
@@ -697,14 +1089,99 @@ module Ocran
697
1089
  # when the directory does not exist in the packed layout - so always
698
1090
  # create the packed prefix gem dirs, even when no specs landed there.
699
1091
  prefix_gem_dirs.each { |dir| builder.mkdir(dir) }
1092
+ if @option.cosmo_ruby
1093
+ # When GEM_PATH is set, RubyGems no longer scans its compiled-in
1094
+ # default directory — which for the cosmopolitan Ruby is the /zip
1095
+ # store inside the binary, where its bundled gems live. Keep it
1096
+ # reachable by appending it explicitly.
1097
+ gem_paths << @cosmo_ruby_info[:default_gem_dir]
1098
+ end
700
1099
  builder.set_env_path("GEM_PATH", *gem_paths)
701
1100
 
702
1101
  # Add the opcode to launch the script
703
1102
  installed_ruby_exe = BINDIR / ruby_executable
704
1103
  target_script = builder.resolve_source_path(@option.script, inst_src_prefix)
1104
+ if rubyopt_result.translated?
1105
+ target_script = generate_rubyopt_launcher(builder, target_script, rubyopt_result)
1106
+ end
705
1107
  builder.exec(installed_ruby_exe, target_script, *@option.argv)
706
1108
  end
707
1109
 
1110
+ # Maps an absolute build-machine path to its location (relative to the
1111
+ # extraction root) inside the packed application, mirroring where
1112
+ # #construct places files. Returns nil when the path is not packed.
1113
+ def packed_rubyopt_path(path, src_prefix)
1114
+ require_relative "gem_spec_queryable"
1115
+
1116
+ if path.subpath?(exec_prefix)
1117
+ path.relative_path_from(exec_prefix)
1118
+ elsif (gem_path = GemSpecQueryable.find_gem_path(path))
1119
+ GEMDIR / path.relative_path_from(gem_path)
1120
+ elsif path.subpath?(src_prefix)
1121
+ SRCDIR / path.relative_path_from(src_prefix)
1122
+ end
1123
+ end
1124
+ private :packed_rubyopt_path
1125
+
1126
+ RUBYOPT_LAUNCHER_NAME = "ocran-rubyopt-launcher.rb"
1127
+
1128
+ # Writes a launcher script next to the packed application script that
1129
+ # applies the translated RUBYOPT -I/-r entries (as plain Ruby string
1130
+ # literals resolved against the extraction directory at runtime) and
1131
+ # then loads the original script. Returns the packed path of the
1132
+ # launcher, which becomes the script the stub executes.
1133
+ def generate_rubyopt_launcher(builder, target_script, result)
1134
+ say "Generating launcher script for RUBYOPT entries with translated paths"
1135
+ launcher_target = target_script.dirname / RUBYOPT_LAUNCHER_NAME
1136
+
1137
+ # Relative path from the launcher's directory up to the extraction root.
1138
+ depth = target_script.dirname.each_filename.count { |name| name != "." }
1139
+ root_rel = depth.zero? ? "." : ([".."] * depth).join("/")
1140
+
1141
+ lines = []
1142
+ lines << "# frozen_string_literal: true"
1143
+ lines << "# Generated by OCRAN. Applies -I/-r options that were given in RUBYOPT"
1144
+ lines << "# with build-machine paths, translated to the extraction directory."
1145
+ lines << "# They cannot remain in RUBYOPT because Ruby does not support quoting"
1146
+ lines << "# there and the extraction path may contain spaces."
1147
+ lines << "ocran_root = File.expand_path(#{root_rel.dump}, __dir__)"
1148
+ lines << "$0 = File.expand_path(#{target_script.basename.to_s.dump}, __dir__)"
1149
+ result.load_paths.reverse_each do |dir|
1150
+ lines << "$LOAD_PATH.unshift(File.expand_path(#{dir.to_posix.dump}, ocran_root))"
1151
+ end
1152
+ result.requires.each do |feature|
1153
+ lines << "require File.expand_path(#{feature.to_posix.dump}, ocran_root)"
1154
+ end
1155
+ lines << "load $0"
1156
+
1157
+ require "tempfile"
1158
+ launcher_file = Tempfile.new(["ocran-rubyopt-launcher", ".rb"])
1159
+ launcher_file.write(lines.join("\n") + "\n")
1160
+ launcher_file.close
1161
+ verbose File.read(launcher_file.path)
1162
+ # Keep a reference so the temporary file survives until the build
1163
+ # finishes; the Inno Setup builder reads source files only when the
1164
+ # installer is compiled, after #construct has returned.
1165
+ @rubyopt_launcher_file = launcher_file
1166
+
1167
+ builder.cp(launcher_file.path, launcher_target)
1168
+ launcher_target
1169
+ end
1170
+ private :generate_rubyopt_launcher
1171
+ # Writes the in-memory gem specification to a temporary file and returns
1172
+ # the file's path, for packing gemspecs that cannot be copied verbatim
1173
+ # from disk (e.g. local development gems). The Tempfile object is
1174
+ # retained because some builders (e.g. InnoSetupScriptBuilder) read
1175
+ # their source files only after construction has completed.
1176
+ def generate_gemspec_file(spec)
1177
+ require "tempfile"
1178
+ file = Tempfile.new(["#{spec.full_name}-", ".gemspec"])
1179
+ file.write(spec.to_ruby)
1180
+ file.close
1181
+ (@generated_gemspec_files ||= []) << file
1182
+ file.path
1183
+ end
1184
+
708
1185
  def to_proc
709
1186
  method(:construct).to_proc
710
1187
  end
@@ -762,6 +1239,23 @@ module Ocran
762
1239
  say "Finished building installer file"
763
1240
  end
764
1241
 
1242
+ # Returns the path to the stub built from source with cosmocc when
1243
+ # --cosmo was given, or nil to use the pre-built stub shipped with
1244
+ # the gem. The build result is memoized (and CosmoToolchain caches
1245
+ # compiled stubs across runs), so multiple stubs per build (e.g.
1246
+ # wrapper executables) compile at most once.
1247
+ def cosmo_stub_path
1248
+ return nil unless @option.cosmo_cc
1249
+
1250
+ @cosmo_stub_path ||= begin
1251
+ load File.expand_path("cosmo_toolchain.rb", __dir__) unless defined? CosmoToolchain
1252
+ say "Building launcher stub from source with cosmocc (#{@option.cosmo_cc})"
1253
+ path = CosmoToolchain.build_stub(@option.cosmo_cc)
1254
+ say "Using APE stub #{path}"
1255
+ path
1256
+ end
1257
+ end
1258
+
765
1259
  # Builds the small RUN_IN_EXE_DIR wrapper stub that starts the deployed
766
1260
  # application directly from the directory the wrapper resides in.
767
1261
  def build_wrapper_exe(wrapper_path)
@@ -769,10 +1263,12 @@ module Ocran
769
1263
  say "Build wrapper executable #{wrapper_path.basename}"
770
1264
  StubBuilder.new(wrapper_path,
771
1265
  chdir_before: @option.chdir_before?,
1266
+ chdir_to_exe_dir: @option.chdir_exe_dir?,
772
1267
  debug_mode: @option.enable_debug_mode?,
773
1268
  gui_mode: @option.windowed?,
774
1269
  icon_path: @option.icon_filename,
775
- run_in_exe_dir: true) do |stub|
1270
+ run_in_exe_dir: true,
1271
+ stub_path: cosmo_stub_path) do |stub|
776
1272
  yield(stub)
777
1273
  end
778
1274
  end
@@ -830,11 +1326,13 @@ module Ocran
830
1326
 
831
1327
  StubBuilder.new(executable_path,
832
1328
  chdir_before: @option.chdir_before?,
1329
+ chdir_to_exe_dir: @option.chdir_exe_dir?,
833
1330
  debug_extract: @option.enable_debug_extract?,
834
1331
  debug_mode: @option.enable_debug_mode?,
835
1332
  enable_compression: @option.enable_compression?,
836
1333
  gui_mode: false,
837
1334
  icon_path: nil,
1335
+ stub_path: cosmo_stub_path,
838
1336
  &to_proc) => builder
839
1337
 
840
1338
  if @option.icon_filename
@@ -870,6 +1368,40 @@ module Ocran
870
1368
  say "Finished building #{bundle_path} (#{builder.data_size} bytes decompressed)"
871
1369
  end
872
1370
 
1371
+ # Builds the executable by copying the cosmopolitan Ruby and injecting
1372
+ # the application into its ZIP store (--cosmo-ruby with an interpreter
1373
+ # that runs an embedded /zip/main.rb). No compiler runs, no launcher
1374
+ # stub is involved, and the resulting binary unpacks nothing when it
1375
+ # starts.
1376
+ def build_cosmo_zip_exe
1377
+ require_relative "zip_payload_builder"
1378
+
1379
+ output = @option.output_executable
1380
+ ZipPayloadBuilder.new(output,
1381
+ cosmo_ruby: @option.cosmo_ruby,
1382
+ chdir_before: @option.chdir_before?,
1383
+ debug_mode: @option.enable_debug_mode?,
1384
+ &to_proc) => builder
1385
+
1386
+ builder.ignored_symlinks.each do |link_path, target|
1387
+ verbose "Skipping symlink #{link_path} -> #{target} (ZIP members cannot be symlinks)"
1388
+ end
1389
+
1390
+ if @option.icon_filename
1391
+ warning "--icon has no effect in this mode: the executable is a copy of the cosmopolitan Ruby, whose resources OCRAN does not rewrite"
1392
+ end
1393
+ if @option.enable_debug_extract?
1394
+ warning "--debug-extract has no effect in this mode: nothing is extracted, the application is read from the executable's own ZIP store"
1395
+ end
1396
+
1397
+ _, _, unsupported = ZipPayloadBuilder.parse_rubyopt(rubyopt)
1398
+ unless unsupported.empty?
1399
+ warning "RUBYOPT #{unsupported.join(" ")} cannot be applied when the application is packed into the interpreter's ZIP store (the interpreter is already running); only -I and -r are replayed"
1400
+ end
1401
+
1402
+ say "Finished building #{output} (#{output.size} bytes, #{builder.data_size} bytes of application data)"
1403
+ end
1404
+
873
1405
  def build_stab_exe
874
1406
  require_relative "stub_builder"
875
1407
 
@@ -879,11 +1411,13 @@ module Ocran
879
1411
 
880
1412
  StubBuilder.new(@option.output_executable,
881
1413
  chdir_before: @option.chdir_before?,
1414
+ chdir_to_exe_dir: @option.chdir_exe_dir?,
882
1415
  debug_extract: @option.enable_debug_extract?,
883
1416
  debug_mode: @option.enable_debug_mode?,
884
1417
  enable_compression: @option.enable_compression?,
885
1418
  gui_mode: @option.windowed?,
886
1419
  icon_path: @option.icon_filename,
1420
+ stub_path: cosmo_stub_path,
887
1421
  &to_proc) => builder
888
1422
  say "Finished building #{@option.output_executable} (#{@option.output_executable.size} bytes)"
889
1423
  say "After decompression, the data will expand to #{builder.data_size} bytes."