scint 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/FEATURES.md +4 -0
- data/README.md +142 -198
- data/VERSION +1 -1
- data/lib/scint/cache/layout.rb +66 -5
- data/lib/scint/cache/manifest.rb +120 -0
- data/lib/scint/cache/prewarm.rb +445 -33
- data/lib/scint/cache/validity.rb +134 -0
- data/lib/scint/cli/cache.rb +34 -6
- data/lib/scint/cli/exec.rb +1 -1
- data/lib/scint/cli/install.rb +611 -292
- data/lib/scint/fs.rb +175 -28
- data/lib/scint/gem/package.rb +6 -2
- data/lib/scint/gemfile/parser.rb +13 -6
- data/lib/scint/index/client.rb +13 -2
- data/lib/scint/installer/extension_builder.rb +63 -43
- data/lib/scint/installer/linker.rb +43 -2
- data/lib/scint/installer/planner.rb +24 -28
- data/lib/scint/installer/preparer.rb +167 -37
- data/lib/scint/installer/promoter.rb +97 -0
- data/lib/scint/linker.sh +137 -0
- data/lib/scint/spec_utils.rb +79 -0
- data/lib/scint.rb +12 -4
- metadata +5 -1
data/lib/scint/fs.rb
CHANGED
|
@@ -21,28 +21,75 @@ module Scint
|
|
|
21
21
|
@mkdir_mutex.synchronize { @mkdir_cache[path] = true }
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
+
# Detect best file copy strategy once per process.
|
|
25
|
+
# Returns :reflink, :hardlink, or :copy.
|
|
26
|
+
@copy_strategy = nil
|
|
27
|
+
@copy_strategy_mutex = Thread::Mutex.new
|
|
28
|
+
|
|
29
|
+
def detect_copy_strategy(src_dir, dst_dir)
|
|
30
|
+
@copy_strategy_mutex.synchronize do
|
|
31
|
+
return @copy_strategy if @copy_strategy
|
|
32
|
+
|
|
33
|
+
@copy_strategy = _probe_copy_strategy(src_dir, dst_dir)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def _probe_copy_strategy(src_dir, dst_dir)
|
|
38
|
+
# Create a temp file in src to test against dst
|
|
39
|
+
probe_src = File.join(src_dir, ".scint_probe_#{$$}")
|
|
40
|
+
probe_dst = File.join(dst_dir, ".scint_probe_#{$$}")
|
|
41
|
+
begin
|
|
42
|
+
File.write(probe_src, "x")
|
|
43
|
+
mkdir_p(dst_dir)
|
|
44
|
+
|
|
45
|
+
if Platform.macos?
|
|
46
|
+
if system("cp", "-c", probe_src, probe_dst, [:out, :err] => File::NULL)
|
|
47
|
+
return :reflink
|
|
48
|
+
end
|
|
49
|
+
File.delete(probe_dst) if File.exist?(probe_dst)
|
|
50
|
+
elsif Platform.linux?
|
|
51
|
+
if system("cp", "--reflink=always", probe_src, probe_dst, [:out, :err] => File::NULL)
|
|
52
|
+
return :reflink
|
|
53
|
+
end
|
|
54
|
+
File.delete(probe_dst) if File.exist?(probe_dst)
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
begin
|
|
58
|
+
File.link(probe_src, probe_dst)
|
|
59
|
+
return :hardlink
|
|
60
|
+
rescue SystemCallError
|
|
61
|
+
File.delete(probe_dst) if File.exist?(probe_dst)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
:copy
|
|
65
|
+
ensure
|
|
66
|
+
File.delete(probe_src) if File.exist?(probe_src)
|
|
67
|
+
File.delete(probe_dst) if File.exist?(probe_dst)
|
|
68
|
+
end
|
|
69
|
+
rescue StandardError
|
|
70
|
+
:copy
|
|
71
|
+
end
|
|
72
|
+
|
|
24
73
|
# APFS clonefile (CoW copy). Falls back to hardlink, then regular copy.
|
|
25
74
|
def clonefile(src, dst)
|
|
26
75
|
src = src.to_s
|
|
27
76
|
dst = dst.to_s
|
|
28
77
|
mkdir_p(File.dirname(dst))
|
|
29
78
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
rescue SystemCallError
|
|
45
|
-
# cross-device or unsupported
|
|
79
|
+
case detect_copy_strategy(File.dirname(src), File.dirname(dst))
|
|
80
|
+
when :reflink
|
|
81
|
+
if Platform.macos?
|
|
82
|
+
return if system("cp", "-c", src, dst, [:out, :err] => File::NULL)
|
|
83
|
+
elsif Platform.linux?
|
|
84
|
+
return if system("cp", "--reflink=always", src, dst, [:out, :err] => File::NULL)
|
|
85
|
+
end
|
|
86
|
+
when :hardlink
|
|
87
|
+
begin
|
|
88
|
+
File.link(src, dst)
|
|
89
|
+
return
|
|
90
|
+
rescue SystemCallError
|
|
91
|
+
# fall through to copy
|
|
92
|
+
end
|
|
46
93
|
end
|
|
47
94
|
|
|
48
95
|
# Final fallback: regular copy
|
|
@@ -58,21 +105,117 @@ module Scint
|
|
|
58
105
|
raise Errno::ENOENT, src_dir unless Dir.exist?(src_dir)
|
|
59
106
|
mkdir_p(dst_dir)
|
|
60
107
|
|
|
61
|
-
|
|
62
|
-
if Platform.macos?
|
|
63
|
-
src_contents = File.join(src_dir, ".")
|
|
64
|
-
return if system("cp", "-cR", src_contents, dst_dir, [:out, :err] => File::NULL)
|
|
65
|
-
end
|
|
108
|
+
strategy = detect_copy_strategy(src_dir, dst_dir)
|
|
66
109
|
|
|
67
|
-
|
|
68
|
-
if Platform.linux?
|
|
110
|
+
if strategy == :reflink
|
|
69
111
|
src_contents = File.join(src_dir, ".")
|
|
70
|
-
|
|
112
|
+
if Platform.macos?
|
|
113
|
+
return if system("cp", "-cR", src_contents, dst_dir, [:out, :err] => File::NULL)
|
|
114
|
+
elsif Platform.linux?
|
|
115
|
+
return if system("cp", "--reflink=always", "-R", src_contents, dst_dir, [:out, :err] => File::NULL)
|
|
116
|
+
end
|
|
71
117
|
end
|
|
72
118
|
|
|
73
119
|
hardlink_tree(src_dir, dst_dir)
|
|
74
120
|
end
|
|
75
121
|
|
|
122
|
+
# Materialize a tree using a manifest to avoid directory scans.
|
|
123
|
+
# Manifest entries must be hashes with "path" and "type" keys.
|
|
124
|
+
# Uses the fastest available file copy strategy (reflink > hardlink > copy).
|
|
125
|
+
def materialize_from_manifest(src_dir, dst_dir, entries)
|
|
126
|
+
src_dir = src_dir.to_s
|
|
127
|
+
dst_dir = dst_dir.to_s
|
|
128
|
+
entries = Array(entries)
|
|
129
|
+
raise Errno::ENOENT, src_dir unless Dir.exist?(src_dir)
|
|
130
|
+
mkdir_p(dst_dir)
|
|
131
|
+
|
|
132
|
+
strategy = detect_copy_strategy(src_dir, dst_dir)
|
|
133
|
+
|
|
134
|
+
entries.each do |entry|
|
|
135
|
+
rel = entry["path"].to_s
|
|
136
|
+
next if rel.empty? || rel.start_with?("/") || rel.include?("..")
|
|
137
|
+
|
|
138
|
+
src_path = File.join(src_dir, rel)
|
|
139
|
+
dst_path = File.join(dst_dir, rel)
|
|
140
|
+
|
|
141
|
+
case entry["type"]
|
|
142
|
+
when "dir"
|
|
143
|
+
mkdir_p(dst_path)
|
|
144
|
+
when "symlink"
|
|
145
|
+
mkdir_p(File.dirname(dst_path))
|
|
146
|
+
next if File.exist?(dst_path) || File.symlink?(dst_path)
|
|
147
|
+
|
|
148
|
+
target = File.readlink(src_path)
|
|
149
|
+
begin
|
|
150
|
+
File.symlink(target, dst_path)
|
|
151
|
+
rescue Errno::EEXIST
|
|
152
|
+
next
|
|
153
|
+
end
|
|
154
|
+
else
|
|
155
|
+
mkdir_p(File.dirname(dst_path))
|
|
156
|
+
next if File.exist?(dst_path)
|
|
157
|
+
|
|
158
|
+
begin
|
|
159
|
+
_link_or_copy(src_path, dst_path, strategy)
|
|
160
|
+
rescue Errno::EEXIST
|
|
161
|
+
next
|
|
162
|
+
rescue SystemCallError
|
|
163
|
+
next if File.exist?(dst_path)
|
|
164
|
+
raise
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# Fast file link/copy using a pre-detected strategy (no per-file probing).
|
|
171
|
+
def _link_or_copy(src, dst, strategy)
|
|
172
|
+
case strategy
|
|
173
|
+
when :reflink
|
|
174
|
+
if Platform.macos?
|
|
175
|
+
return if system("cp", "-c", src, dst, [:out, :err] => File::NULL)
|
|
176
|
+
elsif Platform.linux?
|
|
177
|
+
return if system("cp", "--reflink=always", src, dst, [:out, :err] => File::NULL)
|
|
178
|
+
end
|
|
179
|
+
# reflink failed for this file, try hardlink
|
|
180
|
+
begin
|
|
181
|
+
File.link(src, dst)
|
|
182
|
+
return
|
|
183
|
+
rescue SystemCallError; end
|
|
184
|
+
FileUtils.cp(src, dst)
|
|
185
|
+
when :hardlink
|
|
186
|
+
begin
|
|
187
|
+
File.link(src, dst)
|
|
188
|
+
return
|
|
189
|
+
rescue SystemCallError; end
|
|
190
|
+
FileUtils.cp(src, dst)
|
|
191
|
+
else
|
|
192
|
+
FileUtils.cp(src, dst)
|
|
193
|
+
end
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
LINKER_SCRIPT = File.expand_path("linker.sh", __dir__).freeze
|
|
197
|
+
|
|
198
|
+
# Bulk-link cached gem directories into dst_parent.
|
|
199
|
+
# Opens one helper process (linker.sh) and writes gem basenames to its
|
|
200
|
+
# stdin. The helper probes the fastest FS strategy once then applies it
|
|
201
|
+
# to every gem.
|
|
202
|
+
def bulk_link_gems(src_parent, dst_parent, gem_names)
|
|
203
|
+
src_parent = src_parent.to_s
|
|
204
|
+
dst_parent = dst_parent.to_s
|
|
205
|
+
gem_names = Array(gem_names)
|
|
206
|
+
return 0 if gem_names.empty?
|
|
207
|
+
|
|
208
|
+
mkdir_p(dst_parent)
|
|
209
|
+
|
|
210
|
+
IO.popen(["/bin/bash", LINKER_SCRIPT], "w") do |io|
|
|
211
|
+
io.puts src_parent
|
|
212
|
+
io.puts dst_parent
|
|
213
|
+
gem_names.each { |name| io.puts name }
|
|
214
|
+
end
|
|
215
|
+
|
|
216
|
+
gem_names.size
|
|
217
|
+
end
|
|
218
|
+
|
|
76
219
|
# Clone many source directories into one destination parent directory.
|
|
77
220
|
# This is significantly faster than one process per gem on large warm
|
|
78
221
|
# installs because it batches cp invocations while preserving CoW/reflink.
|
|
@@ -86,6 +229,8 @@ module Scint
|
|
|
86
229
|
return 0 if sources.empty?
|
|
87
230
|
|
|
88
231
|
copied = 0
|
|
232
|
+
strategy = sources.first ? detect_copy_strategy(sources.first, dst_parent) : :copy
|
|
233
|
+
|
|
89
234
|
sources.each_slice([chunk_size.to_i, 1].max) do |slice|
|
|
90
235
|
pending = slice.reject do |src|
|
|
91
236
|
Dir.exist?(File.join(dst_parent, File.basename(src)))
|
|
@@ -93,10 +238,12 @@ module Scint
|
|
|
93
238
|
next if pending.empty?
|
|
94
239
|
|
|
95
240
|
ok = false
|
|
96
|
-
if
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
241
|
+
if strategy == :reflink
|
|
242
|
+
if Platform.macos?
|
|
243
|
+
ok = system("cp", "-cR", *pending, dst_parent, [:out, :err] => File::NULL)
|
|
244
|
+
elsif Platform.linux?
|
|
245
|
+
ok = system("cp", "--reflink=always", "-R", *pending, dst_parent, [:out, :err] => File::NULL)
|
|
246
|
+
end
|
|
100
247
|
end
|
|
101
248
|
|
|
102
249
|
unless ok
|
data/lib/scint/gem/package.rb
CHANGED
|
@@ -17,7 +17,9 @@ module Scint
|
|
|
17
17
|
tar.each do |entry|
|
|
18
18
|
if entry.full_name == "metadata.gz"
|
|
19
19
|
gz = Zlib::GzipReader.new(StringIO.new(entry.read))
|
|
20
|
-
|
|
20
|
+
yaml = gz.read
|
|
21
|
+
yaml.force_encoding("UTF-8") if yaml.encoding == Encoding::US_ASCII
|
|
22
|
+
return ::Gem::Specification.from_yaml(yaml)
|
|
21
23
|
end
|
|
22
24
|
end
|
|
23
25
|
end
|
|
@@ -38,7 +40,9 @@ module Scint
|
|
|
38
40
|
case entry.full_name
|
|
39
41
|
when "metadata.gz"
|
|
40
42
|
gz = Zlib::GzipReader.new(StringIO.new(entry.read))
|
|
41
|
-
|
|
43
|
+
yaml = gz.read
|
|
44
|
+
yaml.force_encoding("UTF-8") if yaml.encoding == Encoding::US_ASCII
|
|
45
|
+
gemspec = ::Gem::Specification.from_yaml(yaml)
|
|
42
46
|
when "data.tar.gz"
|
|
43
47
|
# Write data.tar.gz to a temp file for extraction
|
|
44
48
|
tmp = File.join(dest_dir, ".data.tar.gz.tmp")
|
data/lib/scint/gemfile/parser.rb
CHANGED
|
@@ -6,7 +6,7 @@ require_relative "../source/path"
|
|
|
6
6
|
module Scint
|
|
7
7
|
module Gemfile
|
|
8
8
|
# Result of parsing a Gemfile.
|
|
9
|
-
ParseResult = Struct.new(:dependencies, :sources, :ruby_version, :platforms, keyword_init: true)
|
|
9
|
+
ParseResult = Struct.new(:dependencies, :sources, :ruby_version, :platforms, :optional_groups, keyword_init: true)
|
|
10
10
|
|
|
11
11
|
# Evaluates a Gemfile using instance_eval, just like stock bundler.
|
|
12
12
|
# Supports the full Gemfile DSL: source, gem, group, platforms, git_source,
|
|
@@ -20,14 +20,16 @@ module Scint
|
|
|
20
20
|
sources: parser.parsed_sources.uniq,
|
|
21
21
|
ruby_version: parser.parsed_ruby_version,
|
|
22
22
|
platforms: parser.parsed_platforms,
|
|
23
|
+
optional_groups: parser.parsed_optional_groups,
|
|
23
24
|
)
|
|
24
25
|
end
|
|
25
26
|
|
|
26
27
|
# Accessors that don't collide with DSL method names
|
|
27
|
-
def parsed_dependencies;
|
|
28
|
-
def parsed_sources;
|
|
29
|
-
def parsed_ruby_version;
|
|
30
|
-
def parsed_platforms;
|
|
28
|
+
def parsed_dependencies; @dependencies; end
|
|
29
|
+
def parsed_sources; @sources; end
|
|
30
|
+
def parsed_ruby_version; @ruby_version; end
|
|
31
|
+
def parsed_platforms; @declared_platforms; end
|
|
32
|
+
def parsed_optional_groups; @optional_groups; end
|
|
31
33
|
|
|
32
34
|
def initialize(gemfile_path)
|
|
33
35
|
@gemfile_path = File.expand_path(gemfile_path)
|
|
@@ -39,6 +41,7 @@ module Scint
|
|
|
39
41
|
@current_source_options = {}
|
|
40
42
|
@ruby_version = nil
|
|
41
43
|
@declared_platforms = []
|
|
44
|
+
@optional_groups = []
|
|
42
45
|
|
|
43
46
|
add_default_git_sources
|
|
44
47
|
end
|
|
@@ -167,7 +170,11 @@ module Scint
|
|
|
167
170
|
|
|
168
171
|
def group(*names, **opts, &blk)
|
|
169
172
|
old_groups = @current_groups.dup
|
|
170
|
-
|
|
173
|
+
group_syms = names.map(&:to_sym)
|
|
174
|
+
@current_groups.concat(group_syms)
|
|
175
|
+
if opts[:optional]
|
|
176
|
+
group_syms.each { |g| @optional_groups << g unless @optional_groups.include?(g) }
|
|
177
|
+
end
|
|
171
178
|
yield
|
|
172
179
|
ensure
|
|
173
180
|
@current_groups = old_groups
|
data/lib/scint/index/client.rb
CHANGED
|
@@ -143,8 +143,19 @@ module Scint
|
|
|
143
143
|
private
|
|
144
144
|
|
|
145
145
|
def default_cache_dir
|
|
146
|
-
|
|
147
|
-
|
|
146
|
+
root =
|
|
147
|
+
if Scint.respond_to?(:cache_root)
|
|
148
|
+
Scint.cache_root
|
|
149
|
+
else
|
|
150
|
+
explicit = ENV["SCINT_CACHE"]
|
|
151
|
+
if explicit && !explicit.empty?
|
|
152
|
+
File.expand_path(explicit)
|
|
153
|
+
else
|
|
154
|
+
xdg = ENV["XDG_CACHE_HOME"] || File.join(Dir.home, ".cache")
|
|
155
|
+
File.join(xdg, "scint")
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
File.join(root, "index", Cache.slug_for(@uri))
|
|
148
159
|
end
|
|
149
160
|
|
|
150
161
|
# Fetch a top-level endpoint (names or versions).
|
|
@@ -11,24 +11,22 @@ module Scint
|
|
|
11
11
|
module ExtensionBuilder
|
|
12
12
|
module_function
|
|
13
13
|
|
|
14
|
+
BUILD_MARKER = ".scint.build_complete"
|
|
15
|
+
|
|
14
16
|
# Build native extensions for a prepared gem.
|
|
15
17
|
# prepared_gem: PreparedGem struct
|
|
16
18
|
# bundle_path: .bundle/ root
|
|
17
19
|
# abi_key: e.g. "ruby-3.3.0-arm64-darwin24" (defaults to Platform.abi_key)
|
|
18
20
|
def build(prepared_gem, bundle_path, cache_layout, abi_key: Platform.abi_key, compile_slots: 1, output_tail: nil)
|
|
19
21
|
spec = prepared_gem.spec
|
|
20
|
-
ruby_dir = Platform.ruby_install_dir(bundle_path)
|
|
21
22
|
build_ruby_dir = cache_layout.install_ruby_dir
|
|
23
|
+
source_ruby_dir = Platform.ruby_install_dir(bundle_path)
|
|
24
|
+
src_dir = prepared_gem.extracted_path
|
|
22
25
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
if Dir.exist?(cached_ext) && File.exist?(File.join(cached_ext, "gem.build_complete"))
|
|
26
|
-
link_extensions(cached_ext, ruby_dir, spec, abi_key)
|
|
27
|
-
return true
|
|
28
|
-
end
|
|
26
|
+
marker = build_marker_path(src_dir)
|
|
27
|
+
return true if File.exist?(marker)
|
|
29
28
|
|
|
30
|
-
# Build in a temp dir, then
|
|
31
|
-
src_dir = prepared_gem.extracted_path
|
|
29
|
+
# Build in a temp dir, then sync artifacts into the source tree.
|
|
32
30
|
FS.with_tempdir("scint-ext") do |tmpdir|
|
|
33
31
|
# Stage the full gem source tree in an isolated workspace.
|
|
34
32
|
# Many extconf scripts use paths like ../../vendor relative to ext/,
|
|
@@ -50,37 +48,37 @@ module Scint
|
|
|
50
48
|
# source-tree specific.
|
|
51
49
|
ext_build_dir = File.join(build_root, idx.to_s)
|
|
52
50
|
FS.mkdir_p(ext_build_dir)
|
|
53
|
-
compile_extension(
|
|
51
|
+
compile_extension(
|
|
52
|
+
ext_dir,
|
|
53
|
+
ext_build_dir,
|
|
54
|
+
install_dir,
|
|
55
|
+
staged_src_dir,
|
|
56
|
+
spec,
|
|
57
|
+
build_ruby_dir,
|
|
58
|
+
compile_slots,
|
|
59
|
+
output_tail,
|
|
60
|
+
source_ruby_dir,
|
|
61
|
+
)
|
|
54
62
|
end
|
|
55
63
|
|
|
56
|
-
|
|
57
|
-
File.write(File.join(install_dir, "gem.build_complete"), "")
|
|
58
|
-
|
|
59
|
-
# Cache globally
|
|
60
|
-
FS.mkdir_p(File.dirname(cached_ext))
|
|
61
|
-
FS.atomic_move(install_dir, cached_ext)
|
|
64
|
+
sync_extensions_into_gem(install_dir, src_dir)
|
|
62
65
|
end
|
|
63
66
|
|
|
64
|
-
|
|
67
|
+
File.write(marker, "")
|
|
65
68
|
true
|
|
66
69
|
end
|
|
67
70
|
|
|
68
71
|
# True when a completed global extension build exists for this spec + ABI.
|
|
69
72
|
def cached_build_available?(spec, cache_layout, abi_key: Platform.abi_key)
|
|
70
|
-
|
|
71
|
-
|
|
73
|
+
cached_dir = cache_layout.cached_path(spec, abi_key)
|
|
74
|
+
File.exist?(build_marker_path(cached_dir))
|
|
72
75
|
end
|
|
73
76
|
|
|
74
|
-
# Link already-compiled extensions from
|
|
75
|
-
# Returns true when cache
|
|
76
|
-
def link_cached_build(prepared_gem,
|
|
77
|
+
# Link already-compiled extensions from the cached gem tree.
|
|
78
|
+
# Returns true when cache marker is present, false otherwise.
|
|
79
|
+
def link_cached_build(prepared_gem, _bundle_path, cache_layout, abi_key: Platform.abi_key)
|
|
77
80
|
spec = prepared_gem.spec
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
ruby_dir = Platform.ruby_install_dir(bundle_path)
|
|
81
|
-
cached_ext = cache_layout.ext_path(spec, abi_key)
|
|
82
|
-
link_extensions(cached_ext, ruby_dir, spec, abi_key)
|
|
83
|
-
true
|
|
81
|
+
cached_build_available?(spec, cache_layout, abi_key: abi_key)
|
|
84
82
|
end
|
|
85
83
|
|
|
86
84
|
# True when a gem has native extension sources that need compiling.
|
|
@@ -157,9 +155,9 @@ module Scint
|
|
|
157
155
|
dirs.uniq
|
|
158
156
|
end
|
|
159
157
|
|
|
160
|
-
def compile_extension(ext_dir, build_dir, install_dir, gem_dir, spec, build_ruby_dir, compile_slots, output_tail = nil)
|
|
158
|
+
def compile_extension(ext_dir, build_dir, install_dir, gem_dir, spec, build_ruby_dir, compile_slots, output_tail = nil, source_ruby_dir = nil)
|
|
161
159
|
make_jobs = adaptive_make_jobs(compile_slots)
|
|
162
|
-
env = build_env(gem_dir, build_ruby_dir, make_jobs)
|
|
160
|
+
env = build_env(gem_dir, build_ruby_dir, make_jobs, source_ruby_dir: source_ruby_dir)
|
|
163
161
|
|
|
164
162
|
if File.exist?(File.join(ext_dir, "extconf.rb"))
|
|
165
163
|
compile_extconf(ext_dir, gem_dir, build_dir, install_dir, env, make_jobs, output_tail)
|
|
@@ -235,28 +233,37 @@ module Scint
|
|
|
235
233
|
Platform.gem_arch, Platform.extension_api_version,
|
|
236
234
|
spec_full_name(spec))
|
|
237
235
|
FS.clone_tree(cached_ext, ext_install_dir) unless Dir.exist?(ext_install_dir)
|
|
238
|
-
|
|
236
|
+
gem_dir = File.join(ruby_dir, "gems", spec_full_name(spec))
|
|
237
|
+
sync_extensions_into_gem(cached_ext, gem_dir)
|
|
239
238
|
end
|
|
240
239
|
|
|
241
|
-
|
|
242
|
-
|
|
240
|
+
# Sync compiled extension artifacts into a gem's lib directory.
|
|
241
|
+
# source_dir should contain the compiled artifacts (from build output
|
|
242
|
+
# or a cached gem tree).
|
|
243
|
+
def sync_extensions_into_gem(cached_ext, gem_dir)
|
|
243
244
|
lib_dir = File.join(gem_dir, "lib")
|
|
244
|
-
|
|
245
|
+
FS.mkdir_p(lib_dir)
|
|
245
246
|
|
|
246
|
-
Dir.glob(File.join(
|
|
247
|
-
rel = artifact.delete_prefix("#{
|
|
247
|
+
Dir.glob(File.join(cached_ext, "**", "*.{so,bundle,dll,dylib}")).each do |artifact|
|
|
248
|
+
rel = artifact.delete_prefix("#{cached_ext}/")
|
|
248
249
|
dest = File.join(lib_dir, rel)
|
|
249
250
|
FS.mkdir_p(File.dirname(dest))
|
|
250
251
|
FS.clonefile(artifact, dest)
|
|
251
252
|
end
|
|
252
253
|
end
|
|
253
254
|
|
|
254
|
-
def
|
|
255
|
+
def build_marker_path(gem_dir)
|
|
256
|
+
File.join(gem_dir, BUILD_MARKER)
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
def build_env(gem_dir, build_ruby_dir, make_jobs, source_ruby_dir: nil)
|
|
255
260
|
ruby_bin = File.join(build_ruby_dir, "bin")
|
|
256
261
|
path = [ruby_bin, ENV["PATH"]].compact.reject(&:empty?).join(File::PATH_SEPARATOR)
|
|
262
|
+
inherited_gem_paths = ENV.fetch("GEM_PATH", "").split(File::PATH_SEPARATOR)
|
|
263
|
+
gem_path_entries = [build_ruby_dir, source_ruby_dir, *inherited_gem_paths].compact.reject(&:empty?).uniq
|
|
257
264
|
{
|
|
258
265
|
"GEM_HOME" => build_ruby_dir,
|
|
259
|
-
"GEM_PATH" =>
|
|
266
|
+
"GEM_PATH" => gem_path_entries.join(File::PATH_SEPARATOR),
|
|
260
267
|
"BUNDLE_PATH" => build_ruby_dir,
|
|
261
268
|
"BUNDLE_GEMFILE" => "",
|
|
262
269
|
"MAKEFLAGS" => "-j#{make_jobs}",
|
|
@@ -286,16 +293,17 @@ module Scint
|
|
|
286
293
|
|
|
287
294
|
# Stream output line-by-line so the UX gets live compile progress
|
|
288
295
|
# instead of waiting for the entire subprocess to finish.
|
|
289
|
-
all_output = +""
|
|
296
|
+
all_output = +"".b
|
|
290
297
|
tail_lines = []
|
|
291
298
|
cmd_label = "$ #{cmd.join(" ")}"
|
|
292
299
|
|
|
293
300
|
Open3.popen2e(env, *cmd, **opts) do |stdin, out_err, wait_thr|
|
|
294
301
|
stdin.close
|
|
302
|
+
out_err.set_encoding("ASCII-8BIT")
|
|
295
303
|
|
|
296
304
|
out_err.each_line do |line|
|
|
297
|
-
stripped = line.rstrip
|
|
298
305
|
all_output << line
|
|
306
|
+
stripped = sanitize_output(line).rstrip
|
|
299
307
|
next if stripped.empty?
|
|
300
308
|
|
|
301
309
|
tail_lines << stripped
|
|
@@ -308,7 +316,7 @@ module Scint
|
|
|
308
316
|
|
|
309
317
|
status = wait_thr.value
|
|
310
318
|
unless status.success?
|
|
311
|
-
details = all_output.strip
|
|
319
|
+
details = sanitize_output(all_output).strip
|
|
312
320
|
message = "Command failed (exit #{status.exitstatus}): #{cmd.join(" ")}"
|
|
313
321
|
message = "#{message}\n#{details}" unless details.empty?
|
|
314
322
|
raise ExtensionBuildError, message
|
|
@@ -316,14 +324,26 @@ module Scint
|
|
|
316
324
|
end
|
|
317
325
|
end
|
|
318
326
|
|
|
327
|
+
def sanitize_output(raw)
|
|
328
|
+
return "" if raw.nil? || raw.empty?
|
|
329
|
+
|
|
330
|
+
raw.to_s
|
|
331
|
+
.dup
|
|
332
|
+
.force_encoding(Encoding::BINARY)
|
|
333
|
+
.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "?")
|
|
334
|
+
rescue EncodingError
|
|
335
|
+
raw.to_s.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "?")
|
|
336
|
+
end
|
|
337
|
+
|
|
319
338
|
def spec_full_name(spec)
|
|
320
339
|
SpecUtils.full_name(spec)
|
|
321
340
|
end
|
|
322
341
|
|
|
323
342
|
private_class_method :find_extension_dirs, :compile_extension,
|
|
324
343
|
:compile_extconf, :compile_cmake, :compile_rake,
|
|
325
|
-
:find_rake_executable, :link_extensions,
|
|
326
|
-
:build_env, :run_cmd, :
|
|
344
|
+
:find_rake_executable, :link_extensions,
|
|
345
|
+
:build_env, :run_cmd, :sanitize_output,
|
|
346
|
+
:prebuilt_missing_for_ruby?
|
|
327
347
|
end
|
|
328
348
|
end
|
|
329
349
|
end
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
require_relative "../fs"
|
|
4
4
|
require_relative "../platform"
|
|
5
5
|
require_relative "../spec_utils"
|
|
6
|
+
require_relative "../cache/layout"
|
|
7
|
+
require_relative "../cache/validity"
|
|
6
8
|
require "pathname"
|
|
7
9
|
|
|
8
10
|
module Scint
|
|
@@ -34,7 +36,7 @@ module Scint
|
|
|
34
36
|
# 1. Link gem files into gems/{full_name}/
|
|
35
37
|
gem_dest = File.join(ruby_dir, "gems", full_name)
|
|
36
38
|
unless Dir.exist?(gem_dest)
|
|
37
|
-
|
|
39
|
+
materialize_gem_dir(prepared_gem, gem_dest)
|
|
38
40
|
end
|
|
39
41
|
|
|
40
42
|
# 2. Write gemspec into specifications/
|
|
@@ -60,6 +62,44 @@ module Scint
|
|
|
60
62
|
|
|
61
63
|
# --- private helpers ---
|
|
62
64
|
|
|
65
|
+
def materialize_gem_dir(prepared_gem, gem_dest)
|
|
66
|
+
manifest = cached_manifest_for(prepared_gem)
|
|
67
|
+
if manifest && manifest["files"].is_a?(Array)
|
|
68
|
+
FS.materialize_from_manifest(prepared_gem.extracted_path, gem_dest, manifest["files"])
|
|
69
|
+
else
|
|
70
|
+
FS.clone_tree(prepared_gem.extracted_path, gem_dest)
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def cached_manifest_for(prepared_gem)
|
|
75
|
+
return nil unless prepared_gem.from_cache
|
|
76
|
+
|
|
77
|
+
layout = cache_layout_for(prepared_gem)
|
|
78
|
+
cached_path = layout.cached_path(prepared_gem.spec)
|
|
79
|
+
return nil unless File.expand_path(prepared_gem.extracted_path) == File.expand_path(cached_path)
|
|
80
|
+
|
|
81
|
+
manifest = Cache::Validity.read_manifest(layout.cached_manifest_path(prepared_gem.spec))
|
|
82
|
+
return nil unless manifest
|
|
83
|
+
return nil unless Cache::Validity.manifest_matches?(manifest, prepared_gem.spec, Platform.abi_key, layout)
|
|
84
|
+
|
|
85
|
+
manifest
|
|
86
|
+
rescue StandardError
|
|
87
|
+
nil
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def cache_layout_for(prepared_gem)
|
|
91
|
+
extracted = File.expand_path(prepared_gem.extracted_path)
|
|
92
|
+
abi_dir = File.dirname(extracted)
|
|
93
|
+
cached_dir = File.dirname(abi_dir)
|
|
94
|
+
root = File.dirname(cached_dir)
|
|
95
|
+
|
|
96
|
+
if File.basename(abi_dir) == Platform.abi_key && File.basename(cached_dir) == "cached"
|
|
97
|
+
Cache::Layout.new(root: root)
|
|
98
|
+
else
|
|
99
|
+
Cache::Layout.new
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
63
103
|
def write_gemspec(prepared_gem, ruby_dir, full_name)
|
|
64
104
|
spec_dir = File.join(ruby_dir, "specifications")
|
|
65
105
|
FS.mkdir_p(spec_dir)
|
|
@@ -201,7 +241,8 @@ module Scint
|
|
|
201
241
|
RUBY
|
|
202
242
|
end
|
|
203
243
|
|
|
204
|
-
private_class_method :
|
|
244
|
+
private_class_method :materialize_gem_dir, :cached_manifest_for, :cache_layout_for,
|
|
245
|
+
:write_gemspec, :write_binstubs_impl, :write_ruby_bin_stub,
|
|
205
246
|
:write_bundle_bin_wrapper, :extract_executables,
|
|
206
247
|
:detect_executables_from_files, :augment_executable_metadata, :infer_bindir,
|
|
207
248
|
:minimal_gemspec
|