scint 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. checksums.yaml +7 -0
  2. data/FEATURES.md +13 -0
  3. data/README.md +216 -0
  4. data/bin/bundler-vs-scint +233 -0
  5. data/bin/scint +35 -0
  6. data/bin/scint-io-summary +46 -0
  7. data/bin/scint-syscall-trace +41 -0
  8. data/lib/bundler/setup.rb +5 -0
  9. data/lib/bundler.rb +168 -0
  10. data/lib/scint/cache/layout.rb +131 -0
  11. data/lib/scint/cache/metadata_store.rb +75 -0
  12. data/lib/scint/cache/prewarm.rb +192 -0
  13. data/lib/scint/cli/add.rb +85 -0
  14. data/lib/scint/cli/cache.rb +316 -0
  15. data/lib/scint/cli/exec.rb +150 -0
  16. data/lib/scint/cli/install.rb +1047 -0
  17. data/lib/scint/cli/remove.rb +60 -0
  18. data/lib/scint/cli.rb +77 -0
  19. data/lib/scint/commands/exec.rb +17 -0
  20. data/lib/scint/commands/install.rb +17 -0
  21. data/lib/scint/credentials.rb +153 -0
  22. data/lib/scint/debug/io_trace.rb +218 -0
  23. data/lib/scint/debug/sampler.rb +138 -0
  24. data/lib/scint/downloader/fetcher.rb +113 -0
  25. data/lib/scint/downloader/pool.rb +112 -0
  26. data/lib/scint/errors.rb +63 -0
  27. data/lib/scint/fs.rb +119 -0
  28. data/lib/scint/gem/extractor.rb +86 -0
  29. data/lib/scint/gem/package.rb +62 -0
  30. data/lib/scint/gemfile/dependency.rb +30 -0
  31. data/lib/scint/gemfile/editor.rb +93 -0
  32. data/lib/scint/gemfile/parser.rb +275 -0
  33. data/lib/scint/index/cache.rb +166 -0
  34. data/lib/scint/index/client.rb +301 -0
  35. data/lib/scint/index/parser.rb +142 -0
  36. data/lib/scint/installer/extension_builder.rb +264 -0
  37. data/lib/scint/installer/linker.rb +226 -0
  38. data/lib/scint/installer/planner.rb +140 -0
  39. data/lib/scint/installer/preparer.rb +207 -0
  40. data/lib/scint/lockfile/parser.rb +251 -0
  41. data/lib/scint/lockfile/writer.rb +178 -0
  42. data/lib/scint/platform.rb +71 -0
  43. data/lib/scint/progress.rb +579 -0
  44. data/lib/scint/resolver/provider.rb +230 -0
  45. data/lib/scint/resolver/resolver.rb +249 -0
  46. data/lib/scint/runtime/exec.rb +141 -0
  47. data/lib/scint/runtime/setup.rb +45 -0
  48. data/lib/scint/scheduler.rb +392 -0
  49. data/lib/scint/source/base.rb +46 -0
  50. data/lib/scint/source/git.rb +92 -0
  51. data/lib/scint/source/path.rb +70 -0
  52. data/lib/scint/source/rubygems.rb +79 -0
  53. data/lib/scint/vendor/pub_grub/assignment.rb +20 -0
  54. data/lib/scint/vendor/pub_grub/basic_package_source.rb +169 -0
  55. data/lib/scint/vendor/pub_grub/failure_writer.rb +182 -0
  56. data/lib/scint/vendor/pub_grub/incompatibility.rb +150 -0
  57. data/lib/scint/vendor/pub_grub/package.rb +43 -0
  58. data/lib/scint/vendor/pub_grub/partial_solution.rb +121 -0
  59. data/lib/scint/vendor/pub_grub/rubygems.rb +45 -0
  60. data/lib/scint/vendor/pub_grub/solve_failure.rb +19 -0
  61. data/lib/scint/vendor/pub_grub/static_package_source.rb +61 -0
  62. data/lib/scint/vendor/pub_grub/strategy.rb +42 -0
  63. data/lib/scint/vendor/pub_grub/term.rb +105 -0
  64. data/lib/scint/vendor/pub_grub/version.rb +3 -0
  65. data/lib/scint/vendor/pub_grub/version_constraint.rb +129 -0
  66. data/lib/scint/vendor/pub_grub/version_range.rb +423 -0
  67. data/lib/scint/vendor/pub_grub/version_solver.rb +236 -0
  68. data/lib/scint/vendor/pub_grub/version_union.rb +178 -0
  69. data/lib/scint/vendor/pub_grub.rb +32 -0
  70. data/lib/scint/worker_pool.rb +114 -0
  71. data/lib/scint.rb +87 -0
  72. metadata +116 -0
@@ -0,0 +1,1047 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative "../errors"
4
+ require_relative "../fs"
5
+ require_relative "../platform"
6
+ require_relative "../progress"
7
+ require_relative "../worker_pool"
8
+ require_relative "../scheduler"
9
+ require_relative "../gemfile/dependency"
10
+ require_relative "../gemfile/parser"
11
+ require_relative "../lockfile/parser"
12
+ require_relative "../lockfile/writer"
13
+ require_relative "../source/base"
14
+ require_relative "../source/rubygems"
15
+ require_relative "../source/git"
16
+ require_relative "../source/path"
17
+ require_relative "../index/parser"
18
+ require_relative "../index/cache"
19
+ require_relative "../index/client"
20
+ require_relative "../downloader/fetcher"
21
+ require_relative "../downloader/pool"
22
+ require_relative "../gem/package"
23
+ require_relative "../gem/extractor"
24
+ require_relative "../cache/layout"
25
+ require_relative "../cache/metadata_store"
26
+ require_relative "../installer/planner"
27
+ require_relative "../installer/linker"
28
+ require_relative "../installer/preparer"
29
+ require_relative "../installer/extension_builder"
30
+ require_relative "../vendor/pub_grub"
31
+ require_relative "../resolver/provider"
32
+ require_relative "../resolver/resolver"
33
+ require_relative "../credentials"
34
+ require "open3"
35
+
36
+ module Scint
37
+ module CLI
38
+ class Install
39
+ RUNTIME_LOCK = "scint.lock.marshal"
40
+
41
+ def initialize(argv = [])
42
+ @argv = argv
43
+ @jobs = nil
44
+ @path = nil
45
+ @verbose = false
46
+ @force = false
47
+ parse_options
48
+ end
49
+
50
+ def run
51
+ start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
52
+
53
+ cache = Scint::Cache::Layout.new
54
+ bundle_path = @path || ENV["BUNDLER_PATH"] || ".bundle"
55
+ bundle_path = File.expand_path(bundle_path)
56
+ worker_count = @jobs || [Platform.cpu_count * 2, 50].min
57
+ compile_slots = compile_slots_for(worker_count)
58
+ per_type_limits = install_task_limits(worker_count, compile_slots)
59
+
60
+ # 0. Build credential store from config files (~/.bundle/config, XDG scint/credentials)
61
+ @credentials = Credentials.new
62
+
63
+ # 1. Start the scheduler with 1 worker — scale up dynamically
64
+ scheduler = Scheduler.new(max_workers: worker_count, fail_fast: true, per_type_limits: per_type_limits)
65
+ scheduler.start
66
+
67
+ begin
68
+ # 2. Parse Gemfile
69
+ gemfile = Scint::Gemfile::Parser.parse("Gemfile")
70
+
71
+ # Register credentials from Gemfile sources and dependencies
72
+ @credentials.register_sources(gemfile.sources)
73
+ @credentials.register_dependencies(gemfile.dependencies)
74
+
75
+ # Scale workers based on dependency count
76
+ dep_count = gemfile.dependencies.size
77
+ scheduler.scale_workers(dep_count)
78
+
79
+ # 3. Enqueue index fetches for all sources immediately
80
+ gemfile.sources.each do |source|
81
+ scheduler.enqueue(:fetch_index, source[:uri] || source.to_s,
82
+ -> { fetch_index(source, cache) })
83
+ end
84
+
85
+ # 4. Parse lockfile if it exists
86
+ lockfile = nil
87
+ if File.exist?("Gemfile.lock")
88
+ lockfile = Scint::Lockfile::Parser.parse("Gemfile.lock")
89
+ @credentials.register_lockfile_sources(lockfile.sources)
90
+ end
91
+
92
+ # 5. Enqueue git clones for git sources
93
+ git_sources = gemfile.sources.select { |s| s.is_a?(Source::Git) }
94
+ git_sources.each do |source|
95
+ scheduler.enqueue(:git_clone, source.uri,
96
+ -> { clone_git_source(source, cache) })
97
+ end
98
+
99
+ # 6. Wait for index fetches, then resolve
100
+ scheduler.wait_for(:fetch_index)
101
+ scheduler.wait_for(:git_clone)
102
+
103
+ resolved = resolve(gemfile, lockfile, cache)
104
+ resolved = dedupe_resolved_specs(adjust_meta_gems(resolved))
105
+ force_purge_artifacts(resolved, bundle_path, cache) if @force
106
+
107
+ # 7. Plan: diff resolved vs installed
108
+ plan = Installer::Planner.plan(resolved, bundle_path, cache)
109
+ total_gems = resolved.size
110
+ updated_gems = plan.count { |e| e.action != :skip }
111
+ cached_gems = total_gems - updated_gems
112
+ to_install = plan.reject { |e| e.action == :skip }
113
+
114
+ # Scale up for download/install phase based on actual work count
115
+ scheduler.scale_workers(to_install.size)
116
+
117
+ if to_install.empty?
118
+ elapsed_ms = elapsed_ms_since(start_time)
119
+ warn_missing_bundle_gitignore_entry
120
+ $stdout.puts "\n#{GREEN}#{total_gems}#{RESET} gems installed total#{install_breakdown(cached: cached_gems, updated: updated_gems)}. #{DIM}(#{format_elapsed(elapsed_ms)})#{RESET}"
121
+ return 0
122
+ end
123
+
124
+ # 8. Build a dependency-aware task graph:
125
+ # download -> link_files -> build_ext -> binstub (where applicable).
126
+ compiled_count = enqueue_install_dag(
127
+ scheduler,
128
+ plan,
129
+ cache,
130
+ bundle_path,
131
+ scheduler.progress,
132
+ compile_slots: compile_slots,
133
+ )
134
+
135
+ # 9. Wait for everything
136
+ scheduler.wait_all
137
+ compiled_gems = compiled_count.respond_to?(:call) ? compiled_count.call : compiled_count
138
+ # Stop live progress before printing final summaries/errors so
139
+ # cursor movement does not erase trailing output.
140
+ scheduler.progress.stop if scheduler.respond_to?(:progress)
141
+
142
+ errors = scheduler.errors.dup
143
+ stats = scheduler.stats
144
+ if errors.any?
145
+ $stderr.puts "#{RED}Some gems failed to install:#{RESET}"
146
+ errors.each do |err|
147
+ $stderr.puts " #{BOLD}#{err[:name]}#{RESET}: #{err[:error].message}"
148
+ end
149
+ elsif stats[:failed] > 0
150
+ $stderr.puts "#{YELLOW}Warning: #{stats[:failed]} jobs failed but no error details captured#{RESET}"
151
+ end
152
+
153
+ elapsed_ms = elapsed_ms_since(start_time)
154
+ failed = errors.filter_map { |e| e[:name] }.uniq
155
+ failed_count = failed.size
156
+ failed_count = 1 if failed_count.zero? && stats[:failed] > 0
157
+ installed_total = [total_gems - failed_count, 0].max
158
+ has_failures = errors.any? || stats[:failed] > 0
159
+
160
+ if has_failures
161
+ warn_missing_bundle_gitignore_entry
162
+ $stdout.puts "\n#{RED}Bundle failed!#{RESET} #{installed_total}/#{total_gems} gems installed total#{install_breakdown(cached: cached_gems, updated: updated_gems, compiled: compiled_gems, failed: failed_count)}. #{DIM}(#{format_elapsed(elapsed_ms)})#{RESET}"
163
+ 1
164
+ else
165
+ # 10. Write lockfile + runtime config only for successful installs
166
+ write_lockfile(resolved, gemfile)
167
+ write_runtime_config(resolved, bundle_path)
168
+ warn_missing_bundle_gitignore_entry
169
+ $stdout.puts "\n#{GREEN}#{total_gems}#{RESET} gems installed total#{install_breakdown(cached: cached_gems, updated: updated_gems, compiled: compiled_gems)}. #{DIM}(#{format_elapsed(elapsed_ms)})#{RESET}"
170
+ 0
171
+ end
172
+ ensure
173
+ scheduler.shutdown
174
+ end
175
+ end
176
+
177
+ private
178
+
179
+ # --- Spec adjustment ---
180
+
181
+ # Post-resolution pass: remove bundler (we replace it) and inject scint.
182
+ # This ensures `require "bundler/setup"` loads our shim, and scint
183
+ # appears in the gem list just like bundler does for stock bundler.
184
+ def adjust_meta_gems(resolved)
185
+ resolved = resolved.reject { |s| s.name == "bundler" || s.name == "scint" }
186
+
187
+ scint_spec = ResolvedSpec.new(
188
+ name: "scint",
189
+ version: VERSION,
190
+ platform: "ruby",
191
+ dependencies: [],
192
+ source: "scint (built-in)",
193
+ has_extensions: false,
194
+ remote_uri: nil,
195
+ checksum: nil,
196
+ )
197
+ resolved << scint_spec
198
+
199
+ resolved
200
+ end
201
+
202
+ def dedupe_resolved_specs(resolved)
203
+ seen = {}
204
+ resolved.each do |spec|
205
+ key = "#{spec.name}-#{spec.version}-#{spec.platform}"
206
+ seen[key] ||= spec
207
+ end
208
+ seen.values
209
+ end
210
+
211
+ # Install scint into the bundle by copying our own lib tree.
212
+ # No download needed — we know exactly where we are.
213
+ def install_builtin_gem(entry, bundle_path)
214
+ spec = entry.spec
215
+ ruby_dir = File.join(bundle_path, "ruby", RUBY_VERSION.split(".")[0, 2].join(".") + ".0")
216
+ full_name = spec_full_name(spec)
217
+ scint_root = File.expand_path("../../..", __FILE__)
218
+
219
+ # Copy gem files into gems/scint-x.y.z/lib/
220
+ gem_dest = File.join(ruby_dir, "gems", full_name)
221
+ lib_dest = File.join(gem_dest, "lib")
222
+ unless Dir.exist?(lib_dest)
223
+ FS.mkdir_p(lib_dest)
224
+ FS.hardlink_tree(scint_root, lib_dest)
225
+ end
226
+
227
+ # Write gemspec
228
+ spec_dir = File.join(ruby_dir, "specifications")
229
+ spec_path = File.join(spec_dir, "#{full_name}.gemspec")
230
+ unless File.exist?(spec_path)
231
+ FS.mkdir_p(spec_dir)
232
+ content = <<~RUBY
233
+ Gem::Specification.new do |s|
234
+ s.name = #{spec.name.inspect}
235
+ s.version = #{spec.version.to_s.inspect}
236
+ s.summary = "Fast, parallel gem installer (bundler replacement)"
237
+ s.require_paths = ["lib"]
238
+ end
239
+ RUBY
240
+ FS.atomic_write(spec_path, content)
241
+ end
242
+ end
243
+
244
+ # --- Phase implementations ---
245
+
246
+ def fetch_index(source, cache)
247
+ return unless source.respond_to?(:remotes)
248
+ # Compact index fetch is handled by the index client;
249
+ # we just trigger it here so the data is cached.
250
+ source.remotes.each do |remote|
251
+ cache.ensure_dir(cache.index_path(source))
252
+ end
253
+ end
254
+
255
+ def clone_git_source(source, cache)
256
+ return unless source.respond_to?(:uri)
257
+ git_dir = cache.git_path(source.uri)
258
+ return if Dir.exist?(git_dir)
259
+
260
+ FS.mkdir_p(File.dirname(git_dir))
261
+ system("git", "clone", "--bare", source.uri.to_s, git_dir,
262
+ [:out, :err] => File::NULL)
263
+ end
264
+
265
+ def resolve(gemfile, lockfile, cache)
266
+ # If lockfile is up-to-date, use its specs directly
267
+ if lockfile && lockfile_current?(gemfile, lockfile)
268
+ return lockfile_to_resolved(lockfile)
269
+ end
270
+
271
+ # Collect all unique rubygems source URIs
272
+ default_uri = gemfile.sources.first&.dig(:uri) || "https://rubygems.org"
273
+ all_uris = Set.new([default_uri])
274
+ gemfile.sources.each do |src|
275
+ all_uris << src[:uri] if src[:type] == :rubygems && src[:uri]
276
+ end
277
+
278
+ # Also collect inline source: options from dependencies
279
+ gemfile.dependencies.each do |dep|
280
+ if dep.source_options[:source]
281
+ all_uris << dep.source_options[:source]
282
+ end
283
+ end
284
+
285
+ # Create one Index::Client per unique source URI
286
+ clients = {}
287
+ all_uris.each do |uri|
288
+ clients[uri] = Index::Client.new(uri, credentials: @credentials)
289
+ end
290
+ default_client = clients[default_uri]
291
+
292
+ # Build source_map: gem_name => source_uri for gems with explicit sources
293
+ source_map = {}
294
+ gemfile.dependencies.each do |dep|
295
+ src = dep.source_options[:source]
296
+ source_map[dep.name] = src if src
297
+ end
298
+
299
+ # Build path_gems: gem_name => { version:, dependencies:, source: }
300
+ # for gems with path: or git: sources (skip compact index for these)
301
+ path_gems = {}
302
+ gemfile.dependencies.each do |dep|
303
+ opts = dep.source_options
304
+ next unless opts[:path] || opts[:git]
305
+
306
+ version = "0"
307
+ deps = []
308
+
309
+ # Try to read version and deps from gemspec if it's a path gem
310
+ if opts[:path]
311
+ gemspec = find_gemspec(opts[:path], dep.name)
312
+ if gemspec
313
+ version = gemspec.version.to_s
314
+ deps = gemspec.dependencies
315
+ .select { |d| d.type == :runtime }
316
+ .map { |d| [d.name, d.requirement.to_s] }
317
+ end
318
+ end
319
+
320
+ # For git gems, try lockfile for version
321
+ if opts[:git] && lockfile
322
+ locked_spec = lockfile.specs.find { |s| s[:name] == dep.name }
323
+ version = locked_spec[:version] if locked_spec
324
+ end
325
+
326
+ source_desc = opts[:path] || opts[:git] || "local"
327
+ path_gems[dep.name] = { version: version, dependencies: deps, source: source_desc }
328
+ end
329
+
330
+ locked = {}
331
+ if lockfile
332
+ lockfile.specs.each { |s| locked[s[:name]] = s[:version] }
333
+ end
334
+
335
+ provider = Resolver::Provider.new(
336
+ default_client,
337
+ clients: clients,
338
+ source_map: source_map,
339
+ path_gems: path_gems,
340
+ locked_specs: locked,
341
+ )
342
+ resolver = Resolver::Resolver.new(
343
+ provider: provider,
344
+ dependencies: gemfile.dependencies,
345
+ locked_specs: locked,
346
+ )
347
+ resolver.resolve
348
+ end
349
+
350
+ def find_gemspec(path, gem_name)
351
+ return nil unless Dir.exist?(path)
352
+
353
+ # Look for exact match first, then any gemspec
354
+ candidates = [
355
+ File.join(path, "#{gem_name}.gemspec"),
356
+ *Dir.glob(File.join(path, "*.gemspec")),
357
+ ]
358
+
359
+ candidates.each do |gs|
360
+ next unless File.exist?(gs)
361
+ begin
362
+ spec = Gem::Specification.load(gs)
363
+ return spec if spec
364
+ rescue StandardError
365
+ nil
366
+ end
367
+ end
368
+ nil
369
+ end
370
+
371
+ def lockfile_current?(gemfile, lockfile)
372
+ return false unless lockfile
373
+
374
+ locked_names = Set.new(lockfile.specs.map { |s| s[:name] })
375
+ gemfile.dependencies.all? { |d| locked_names.include?(d.name) }
376
+ end
377
+
378
+ def lockfile_to_resolved(lockfile)
379
+ lockfile.specs.map do |ls|
380
+ source = ls[:source]
381
+ source_value =
382
+ if source.is_a?(Source::Rubygems)
383
+ source.uri.to_s
384
+ else
385
+ source
386
+ end
387
+
388
+ ResolvedSpec.new(
389
+ name: ls[:name],
390
+ version: ls[:version],
391
+ platform: ls[:platform],
392
+ dependencies: ls[:dependencies],
393
+ source: source_value,
394
+ has_extensions: false,
395
+ remote_uri: nil,
396
+ checksum: ls[:checksum],
397
+ )
398
+ end
399
+ end
400
+
401
+ def download_gem(entry, cache)
402
+ spec = entry.spec
403
+ source = spec.source
404
+ if git_source?(source)
405
+ prepare_git_source(entry, cache)
406
+ return
407
+ end
408
+ source_uri = source.to_s
409
+
410
+ # Path gems are not downloaded from a remote
411
+ return if source_uri.start_with?("/") || !source_uri.start_with?("http")
412
+
413
+ full_name = spec_full_name(spec)
414
+ gem_filename = "#{full_name}.gem"
415
+ source_uri = source_uri.chomp("/")
416
+ download_uri = "#{source_uri}/gems/#{gem_filename}"
417
+ dest_path = cache.inbound_path(spec)
418
+
419
+ FS.mkdir_p(File.dirname(dest_path))
420
+
421
+ unless File.exist?(dest_path)
422
+ pool = Downloader::Pool.new(size: 1, credentials: @credentials)
423
+ pool.download(download_uri, dest_path)
424
+ pool.close
425
+ end
426
+ end
427
+
428
+ def extract_gem(entry, cache)
429
+ spec = entry.spec
430
+ source_uri = spec.source.to_s
431
+
432
+ # Git/path gems are already materialized by checkout or local path.
433
+ return if git_source?(spec.source)
434
+ return if source_uri.start_with?("/") || !source_uri.start_with?("http")
435
+
436
+ extracted = cache.extracted_path(spec)
437
+ return if Dir.exist?(extracted)
438
+
439
+ dest_path = cache.inbound_path(spec)
440
+ raise InstallError, "Missing cached gem file for #{spec.name}: #{dest_path}" unless File.exist?(dest_path)
441
+
442
+ FS.mkdir_p(extracted)
443
+ pkg = GemPkg::Package.new
444
+ result = pkg.extract(dest_path, extracted)
445
+ cache_gemspec(spec, result[:gemspec], cache)
446
+ end
447
+
448
+ def git_source?(source)
449
+ return true if source.is_a?(Source::Git)
450
+
451
+ source_str = source.to_s
452
+ source_str.end_with?(".git") || source_str.include?(".git/")
453
+ end
454
+
455
+ def prepare_git_source(entry, cache)
456
+ spec = entry.spec
457
+ source = spec.source
458
+ uri, revision = git_source_ref(source)
459
+
460
+ bare_repo = cache.git_path(uri)
461
+
462
+ # Serialize all git operations per bare repo — git uses index.lock
463
+ # and can't handle concurrent checkouts from the same repo.
464
+ git_mutex_for(bare_repo).synchronize do
465
+ clone_git_repo(uri, bare_repo) unless Dir.exist?(bare_repo)
466
+
467
+ extracted = cache.extracted_path(spec)
468
+ return if Dir.exist?(extracted)
469
+
470
+ tmp = "#{extracted}.#{Process.pid}.#{Thread.current.object_id}.tmp"
471
+ begin
472
+ FileUtils.rm_rf(tmp)
473
+ FS.mkdir_p(tmp)
474
+
475
+ cmd = ["git", "--git-dir", bare_repo, "--work-tree", tmp, "checkout", "-f", revision, "--", "."]
476
+ _out, err, status = Open3.capture3(*cmd)
477
+ unless status.success?
478
+ raise InstallError, "Git checkout failed for #{spec.name} (#{uri}@#{revision}): #{err.to_s.strip}"
479
+ end
480
+
481
+ FS.atomic_move(tmp, extracted)
482
+ ensure
483
+ FileUtils.rm_rf(tmp) if tmp && File.exist?(tmp)
484
+ end
485
+ end
486
+ end
487
+
488
+ def git_source_ref(source)
489
+ if source.is_a?(Source::Git)
490
+ revision = source.revision || source.ref || source.branch || source.tag || "HEAD"
491
+ return [source.uri.to_s, revision.to_s]
492
+ end
493
+
494
+ [source.to_s, "HEAD"]
495
+ end
496
+
497
+ def git_mutex_for(repo_path)
498
+ @git_mutexes_lock ||= Thread::Mutex.new
499
+ @git_mutexes_lock.synchronize do
500
+ @git_mutexes ||= {}
501
+ @git_mutexes[repo_path] ||= Thread::Mutex.new
502
+ end
503
+ end
504
+
505
+ def clone_git_repo(uri, bare_repo)
506
+ FS.mkdir_p(File.dirname(bare_repo))
507
+ _out, err, status = Open3.capture3("git", "clone", "--bare", uri.to_s, bare_repo)
508
+ unless status.success?
509
+ raise InstallError, "Git clone failed for #{uri}: #{err.to_s.strip}"
510
+ end
511
+ end
512
+
513
+ def compile_slots_for(worker_count)
514
+ # Keep one worker lane available for non-compile tasks and cap native
515
+ # compiles at two concurrent jobs.
516
+ max_compile = [2, Platform.cpu_count].min
517
+ available = [worker_count - 1, 1].max
518
+ [max_compile, available].min
519
+ end
520
+
521
+ def install_task_limits(worker_count, compile_slots)
522
+ # Leave headroom for compile and binstub lanes so link/download
523
+ # throughput cannot fully starve them.
524
+ io_cpu_limit = [worker_count - compile_slots - 1, 1].max
525
+ {
526
+ download: io_cpu_limit,
527
+ extract: io_cpu_limit,
528
+ link: io_cpu_limit,
529
+ build_ext: compile_slots,
530
+ binstub: 1,
531
+ }
532
+ end
533
+
534
+ # Enqueue dependency-aware install tasks so compile/binstub can run
535
+ # concurrently with link/download once prerequisites are satisfied.
536
+ def enqueue_install_dag(scheduler, plan, cache, bundle_path, progress = nil, compile_slots: 1)
537
+ link_job_by_key = {}
538
+ link_job_by_name = {}
539
+ build_job_by_key = {}
540
+ build_count = 0
541
+ build_count_lock = Thread::Mutex.new
542
+
543
+ plan.each do |entry|
544
+ case entry.action
545
+ when :skip
546
+ next
547
+ when :builtin
548
+ install_builtin_gem(entry, bundle_path)
549
+ next
550
+ when :download
551
+ key = spec_key(entry.spec)
552
+ download_id = scheduler.enqueue(:download, entry.spec.name,
553
+ -> { download_gem(entry, cache) })
554
+ extract_id = scheduler.enqueue(:extract, entry.spec.name,
555
+ -> { extract_gem(entry, cache) },
556
+ depends_on: [download_id],
557
+ follow_up: lambda { |_job|
558
+ own_link = link_job_by_key[key]
559
+ next unless own_link
560
+
561
+ depends_on = [own_link]
562
+ dep_links = dependency_link_job_ids(entry.spec, link_job_by_name)
563
+ build_depends = (depends_on + dep_links).uniq
564
+
565
+ extracted = extracted_path_for_entry(entry, cache)
566
+ if Installer::ExtensionBuilder.buildable_source_dir?(extracted)
567
+ build_id = scheduler.enqueue(:build_ext, entry.spec.name,
568
+ -> { build_extensions(entry, cache, bundle_path, progress, compile_slots: compile_slots) },
569
+ depends_on: build_depends)
570
+ build_job_by_key[key] = build_id
571
+ depends_on << build_id
572
+ build_count_lock.synchronize { build_count += 1 }
573
+ end
574
+
575
+ scheduler.enqueue(:binstub, entry.spec.name,
576
+ -> { write_binstubs(entry, cache, bundle_path) },
577
+ depends_on: depends_on)
578
+ })
579
+ link_id = scheduler.enqueue(:link, entry.spec.name,
580
+ -> { link_gem_files(entry, cache, bundle_path) },
581
+ depends_on: [extract_id])
582
+ when :link, :build_ext
583
+ link_id = scheduler.enqueue(:link, entry.spec.name,
584
+ -> { link_gem_files(entry, cache, bundle_path) })
585
+ else
586
+ next
587
+ end
588
+
589
+ key = spec_key(entry.spec)
590
+ link_job_by_key[key] = link_id
591
+ link_job_by_name[entry.spec.name] = link_id
592
+ end
593
+
594
+ plan.each do |entry|
595
+ next unless entry.action == :build_ext
596
+
597
+ key = spec_key(entry.spec)
598
+ own_link = link_job_by_key[key]
599
+ next unless own_link
600
+
601
+ dep_links = dependency_link_job_ids(entry.spec, link_job_by_name)
602
+ depends_on = ([own_link] + dep_links).uniq
603
+ build_id = scheduler.enqueue(:build_ext, entry.spec.name,
604
+ -> { build_extensions(entry, cache, bundle_path, progress, compile_slots: compile_slots) },
605
+ depends_on: depends_on)
606
+ build_job_by_key[key] = build_id
607
+ build_count_lock.synchronize { build_count += 1 }
608
+ end
609
+
610
+ plan.each do |entry|
611
+ next if entry.action == :skip || entry.action == :builtin || entry.action == :download
612
+
613
+ key = spec_key(entry.spec)
614
+ own_link = link_job_by_key[key]
615
+ next unless own_link
616
+
617
+ depends_on = [own_link]
618
+ build_id = build_job_by_key[key]
619
+ depends_on << build_id if build_id
620
+ scheduler.enqueue(:binstub, entry.spec.name,
621
+ -> { write_binstubs(entry, cache, bundle_path) },
622
+ depends_on: depends_on)
623
+ end
624
+
625
+ -> { build_count_lock.synchronize { build_count } }
626
+ end
627
+
628
+ def spec_key(spec)
629
+ "#{spec.name}-#{spec.version}-#{spec.platform}"
630
+ end
631
+
632
+ def dependency_link_job_ids(spec, link_job_by_name)
633
+ names = Array(spec.dependencies).filter_map do |dep|
634
+ if dep.is_a?(Hash)
635
+ dep[:name] || dep["name"]
636
+ elsif dep.respond_to?(:name)
637
+ dep.name
638
+ end
639
+ end
640
+ names.filter_map { |name| link_job_by_name[name] }.uniq
641
+ end
642
+
643
+ def enqueue_link_after_download(scheduler, entry, cache, bundle_path)
644
+ scheduler.enqueue(:link, entry.spec.name,
645
+ -> { link_gem_files(entry, cache, bundle_path) })
646
+ end
647
+
648
+ def enqueue_builds(scheduler, entries, cache, bundle_path, compile_slots: 1)
649
+ enqueued = 0
650
+ entries.each do |entry|
651
+ extracted = extracted_path_for_entry(entry, cache)
652
+ next unless Installer::ExtensionBuilder.buildable_source_dir?(extracted)
653
+
654
+ scheduler.enqueue(:build_ext, entry.spec.name,
655
+ -> { build_extensions(entry, cache, bundle_path, nil, compile_slots: compile_slots) })
656
+ enqueued += 1
657
+ end
658
+ enqueued
659
+ end
660
+
661
+ def extracted_path_for_entry(entry, cache)
662
+ source_str = entry.spec.source.to_s
663
+ if source_str.start_with?("/") && Dir.exist?(source_str)
664
+ source_str
665
+ else
666
+ entry.cached_path || cache.extracted_path(entry.spec)
667
+ end
668
+ end
669
+
670
+ def link_gem_files(entry, cache, bundle_path)
671
+ spec = entry.spec
672
+ extracted = extracted_path_for_entry(entry, cache)
673
+
674
+ gemspec = load_gemspec(extracted, spec, cache)
675
+
676
+ prepared = PreparedGem.new(
677
+ spec: spec,
678
+ extracted_path: extracted,
679
+ gemspec: gemspec,
680
+ from_cache: true,
681
+ )
682
+ Installer::Linker.link_files(prepared, bundle_path)
683
+ Installer::Linker.link_files_to_ruby_dir(prepared, cache.install_ruby_dir)
684
+ # If this gem has a cached native build, materialize it during link.
685
+ # This lets reinstalling into a fresh .bundle skip build_ext entirely.
686
+ Installer::ExtensionBuilder.link_cached_build(prepared, bundle_path, cache)
687
+ end
688
+
689
+ def build_extensions(entry, cache, bundle_path, progress = nil, compile_slots: 1)
690
+ extracted = entry.cached_path || cache.extracted_path(entry.spec)
691
+ gemspec = load_gemspec(extracted, entry.spec, cache)
692
+
693
+ sync_build_env_dependencies(entry.spec, bundle_path, cache)
694
+
695
+ prepared = PreparedGem.new(
696
+ spec: entry.spec,
697
+ extracted_path: extracted,
698
+ gemspec: gemspec,
699
+ from_cache: true,
700
+ )
701
+
702
+ Installer::ExtensionBuilder.build(
703
+ prepared,
704
+ bundle_path,
705
+ cache,
706
+ compile_slots: compile_slots,
707
+ output_tail: ->(lines) { progress&.on_build_tail(entry.spec.name, lines) },
708
+ )
709
+ end
710
+
711
+ def sync_build_env_dependencies(spec, bundle_path, cache)
712
+ dep_names = Array(spec.dependencies).filter_map do |dep|
713
+ if dep.is_a?(Hash)
714
+ dep[:name] || dep["name"]
715
+ elsif dep.respond_to?(:name)
716
+ dep.name
717
+ end
718
+ end
719
+ return if dep_names.empty?
720
+
721
+ source_ruby_dir = File.join(bundle_path, "ruby", RUBY_VERSION.split(".")[0, 2].join(".") + ".0")
722
+ target_ruby_dir = cache.install_ruby_dir
723
+
724
+ dep_names.each do |name|
725
+ sync_named_gem_to_build_env(name, source_ruby_dir, target_ruby_dir)
726
+ end
727
+ end
728
+
729
+ def sync_named_gem_to_build_env(name, source_ruby_dir, target_ruby_dir)
730
+ pattern = File.join(source_ruby_dir, "specifications", "#{name}-*.gemspec")
731
+ Dir.glob(pattern).each do |spec_path|
732
+ full_name = File.basename(spec_path, ".gemspec")
733
+ source_gem_dir = File.join(source_ruby_dir, "gems", full_name)
734
+ next unless Dir.exist?(source_gem_dir)
735
+
736
+ target_gem_dir = File.join(target_ruby_dir, "gems", full_name)
737
+ FS.hardlink_tree(source_gem_dir, target_gem_dir) unless Dir.exist?(target_gem_dir)
738
+
739
+ target_spec_dir = File.join(target_ruby_dir, "specifications")
740
+ target_spec_path = File.join(target_spec_dir, "#{full_name}.gemspec")
741
+ next if File.exist?(target_spec_path)
742
+
743
+ FS.mkdir_p(target_spec_dir)
744
+ FS.clonefile(spec_path, target_spec_path)
745
+ end
746
+ end
747
+
748
+ def write_binstubs(entry, cache, bundle_path)
749
+ extracted = extracted_path_for_entry(entry, cache)
750
+ gemspec = load_gemspec(extracted, entry.spec, cache)
751
+ prepared = PreparedGem.new(
752
+ spec: entry.spec,
753
+ extracted_path: extracted,
754
+ gemspec: gemspec,
755
+ from_cache: true,
756
+ )
757
+ Installer::Linker.write_binstubs(prepared, bundle_path)
758
+ end
759
+
760
+ def load_gemspec(extracted_path, spec, cache)
761
+ cached = load_cached_gemspec(spec, cache, extracted_path)
762
+ return cached if cached
763
+
764
+ inbound = cache.inbound_path(spec)
765
+ return nil unless File.exist?(inbound)
766
+
767
+ begin
768
+ metadata = GemPkg::Package.new.read_metadata(inbound)
769
+ cache_gemspec(spec, metadata, cache)
770
+ metadata
771
+ rescue StandardError
772
+ nil
773
+ end
774
+ end
775
+
776
+ def load_cached_gemspec(spec, cache, extracted_path)
777
+ path = cache.spec_cache_path(spec)
778
+ return nil unless File.exist?(path)
779
+
780
+ data = File.binread(path)
781
+ gemspec = if data.start_with?("---")
782
+ Gem::Specification.from_yaml(data)
783
+ else
784
+ begin
785
+ Marshal.load(data)
786
+ rescue StandardError
787
+ Gem::Specification.from_yaml(data)
788
+ end
789
+ end
790
+ return gemspec if cached_gemspec_valid?(gemspec, extracted_path)
791
+
792
+ nil
793
+ rescue StandardError
794
+ nil
795
+ end
796
+
797
+ def cached_gemspec_valid?(gemspec, extracted_path)
798
+ return false unless gemspec.respond_to?(:require_paths)
799
+
800
+ require_paths = Array(gemspec.require_paths).reject(&:empty?)
801
+ return true if require_paths.empty?
802
+
803
+ require_paths.all? do |rp|
804
+ dir = File.join(extracted_path, rp)
805
+ next false unless Dir.exist?(dir)
806
+
807
+ # Heuristic for stale cached metadata seen in some gems:
808
+ # `require_paths=["lib"]` while all entries live under a
809
+ # hyphenated nested directory (e.g. lib/concurrent-ruby).
810
+ if rp == "lib"
811
+ entries = Dir.children(dir)
812
+ top_level_rb = entries.any? do |entry|
813
+ path = File.join(dir, entry)
814
+ File.file?(path) && entry.end_with?(".rb")
815
+ end
816
+ next true if top_level_rb
817
+
818
+ nested_dirs = entries.select { |entry| File.directory?(File.join(dir, entry)) }
819
+ next false if nested_dirs.any? { |entry| entry.include?("-") }
820
+ end
821
+
822
+ true
823
+ end
824
+ end
825
+
826
+ def cache_gemspec(spec, gemspec, cache)
827
+ path = cache.spec_cache_path(spec)
828
+ FS.atomic_write(path, gemspec.to_yaml)
829
+ rescue StandardError
830
+ # Non-fatal: we'll read metadata from .gem next time.
831
+ end
832
+
833
+ # --- Lockfile + runtime config ---
834
+
835
+ def write_lockfile(resolved, gemfile)
836
+ sources = []
837
+
838
+ # Build source objects for path and git gems
839
+ gemfile.dependencies.each do |dep|
840
+ opts = dep.source_options
841
+ if opts[:path]
842
+ sources << Source::Path.new(path: opts[:path], name: dep.name)
843
+ elsif opts[:git]
844
+ sources << Source::Git.new(
845
+ uri: opts[:git],
846
+ branch: opts[:branch],
847
+ tag: opts[:tag],
848
+ ref: opts[:ref],
849
+ )
850
+ end
851
+ end
852
+
853
+ # Build rubygems sources -- collect all unique URIs
854
+ rubygems_uris = gemfile.sources
855
+ .select { |s| s[:type] == :rubygems }
856
+ .map { |s| s[:uri] }
857
+ .uniq
858
+
859
+ # Group URIs that share specs into one Source::Rubygems each.
860
+ # The default source gets all remotes that aren't a separate scoped source.
861
+ scoped_uris = Set.new
862
+ gemfile.dependencies.each do |dep|
863
+ src = dep.source_options[:source]
864
+ scoped_uris << src if src
865
+ end
866
+
867
+ # Each scoped URI gets its own source object
868
+ scoped_uris.each do |uri|
869
+ sources << Source::Rubygems.new(remotes: [uri])
870
+ end
871
+
872
+ # Default rubygems source with remaining remotes
873
+ default_remotes = rubygems_uris.reject { |u| scoped_uris.include?(u) }
874
+ default_remotes = ["https://rubygems.org"] if default_remotes.empty?
875
+ sources << Source::Rubygems.new(remotes: default_remotes)
876
+
877
+ lockfile_data = Lockfile::LockfileData.new(
878
+ specs: resolved,
879
+ dependencies: gemfile.dependencies.map { |d| { name: d.name, version_reqs: d.version_reqs } },
880
+ platforms: [Platform.local_platform.to_s, "ruby"].uniq,
881
+ sources: sources,
882
+ bundler_version: Scint::VERSION,
883
+ ruby_version: nil,
884
+ checksums: nil,
885
+ )
886
+
887
+ content = Lockfile::Writer.write(lockfile_data)
888
+ FS.atomic_write("Gemfile.lock", content)
889
+ end
890
+
891
+ def write_runtime_config(resolved, bundle_path)
892
+ ruby_dir = File.join(bundle_path, "ruby",
893
+ RUBY_VERSION.split(".")[0, 2].join(".") + ".0")
894
+
895
+ data = {}
896
+ resolved.each do |spec|
897
+ full = spec_full_name(spec)
898
+ gem_dir = File.join(ruby_dir, "gems", full)
899
+ spec_file = File.join(ruby_dir, "specifications", "#{full}.gemspec")
900
+ require_paths = read_require_paths(spec_file)
901
+ load_paths = require_paths
902
+ .map { |rp| File.join(gem_dir, rp) }
903
+ .select { |path| Dir.exist?(path) }
904
+
905
+ default_lib = File.join(gem_dir, "lib")
906
+ load_paths << default_lib if load_paths.empty? && Dir.exist?(default_lib)
907
+ load_paths.concat(detect_nested_lib_paths(gem_dir))
908
+ load_paths.uniq!
909
+
910
+ # Add ext load path if extensions exist
911
+ ext_dir = File.join(ruby_dir, "extensions",
912
+ Platform.gem_arch, Platform.extension_api_version, full)
913
+ load_paths << ext_dir if Dir.exist?(ext_dir)
914
+
915
+ data[spec.name] = {
916
+ version: spec.version.to_s,
917
+ load_paths: load_paths,
918
+ }
919
+ end
920
+
921
+ lock_path = File.join(bundle_path, RUNTIME_LOCK)
922
+ FS.atomic_write(lock_path, Marshal.dump(data))
923
+ end
924
+
925
+ def read_require_paths(spec_file)
926
+ return ["lib"] unless File.exist?(spec_file)
927
+
928
+ gemspec = Gem::Specification.load(spec_file)
929
+ paths = Array(gemspec&.require_paths).reject(&:empty?)
930
+ paths.empty? ? ["lib"] : paths
931
+ rescue StandardError
932
+ ["lib"]
933
+ end
934
+
935
+ def detect_nested_lib_paths(gem_dir)
936
+ lib_dir = File.join(gem_dir, "lib")
937
+ return [] unless Dir.exist?(lib_dir)
938
+
939
+ children = Dir.children(lib_dir)
940
+ top_level_rb = children.any? do |entry|
941
+ path = File.join(lib_dir, entry)
942
+ File.file?(path) && entry.end_with?(".rb")
943
+ end
944
+ return [] if top_level_rb
945
+
946
+ children
947
+ .map { |entry| File.join(lib_dir, entry) }
948
+ .select { |path| File.directory?(path) }
949
+ end
950
+
951
+ def spec_full_name(spec)
952
+ base = "#{spec.name}-#{spec.version}"
953
+ plat = spec.respond_to?(:platform) ? spec.platform : nil
954
+ (plat.nil? || plat.to_s == "ruby" || plat.to_s.empty?) ? base : "#{base}-#{plat}"
955
+ end
956
+
957
+ def elapsed_ms_since(start_time)
958
+ elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
959
+ (elapsed * 1000).round
960
+ end
961
+
962
+ def force_purge_artifacts(resolved, bundle_path, cache)
963
+ ruby_dir = File.join(bundle_path, "ruby", RUBY_VERSION.split(".")[0, 2].join(".") + ".0")
964
+ ext_root = File.join(ruby_dir, "extensions", Platform.gem_arch, Platform.extension_api_version)
965
+
966
+ resolved.each do |spec|
967
+ full = cache.full_name(spec)
968
+
969
+ # Global cache artifacts.
970
+ FileUtils.rm_f(cache.inbound_path(spec))
971
+ FileUtils.rm_rf(cache.extracted_path(spec))
972
+ FileUtils.rm_f(cache.spec_cache_path(spec))
973
+ FileUtils.rm_rf(cache.ext_path(spec))
974
+
975
+ # Local bundle artifacts.
976
+ FileUtils.rm_rf(File.join(ruby_dir, "gems", full))
977
+ FileUtils.rm_f(File.join(ruby_dir, "specifications", "#{full}.gemspec"))
978
+ FileUtils.rm_rf(File.join(ext_root, full))
979
+ end
980
+
981
+ # Binstubs are regenerated from gemspec metadata.
982
+ FileUtils.rm_rf(File.join(bundle_path, "bin"))
983
+ FileUtils.rm_rf(File.join(ruby_dir, "bin"))
984
+ FileUtils.rm_f(File.join(bundle_path, RUNTIME_LOCK))
985
+ end
986
+
987
+ def format_elapsed(elapsed_ms)
988
+ return "#{elapsed_ms}ms" if elapsed_ms <= 1000
989
+
990
+ "#{(elapsed_ms / 1000.0).round(2)}s"
991
+ end
992
+
993
+ def warn_missing_bundle_gitignore_entry
994
+ path = ".gitignore"
995
+ return unless File.file?(path)
996
+ return if gitignore_has_bundle_entry?(path)
997
+
998
+ $stderr.puts "#{YELLOW}Warning: .gitignore exists but does not ignore .bundle (add `.bundle/`).#{RESET}"
999
+ end
1000
+
1001
+ def gitignore_has_bundle_entry?(path)
1002
+ File.foreach(path) do |line|
1003
+ entry = line.strip
1004
+ next if entry.empty? || entry.start_with?("#", "!")
1005
+
1006
+ normalized = entry.sub(%r{\A\./}, "")
1007
+ return true if normalized.match?(%r{\A(?:\*\*/)?/?\.bundle(?:/.*)?\z})
1008
+ end
1009
+ false
1010
+ rescue StandardError
1011
+ false
1012
+ end
1013
+
1014
+ def install_breakdown(**counts)
1015
+ parts = counts.filter_map do |label, n|
1016
+ next if n.zero?
1017
+ color = (label == :failed) ? RED : ""
1018
+ reset = color.empty? ? "" : RESET
1019
+ "#{color}#{n} #{label}#{reset}"
1020
+ end
1021
+ parts.empty? ? "" : " (#{parts.join(", ")})"
1022
+ end
1023
+
1024
+ def parse_options
1025
+ i = 0
1026
+ while i < @argv.length
1027
+ case @argv[i]
1028
+ when "--jobs", "-j"
1029
+ @jobs = @argv[i + 1]&.to_i
1030
+ i += 2
1031
+ when "--path"
1032
+ @path = @argv[i + 1]
1033
+ i += 2
1034
+ when "--verbose"
1035
+ @verbose = true
1036
+ i += 1
1037
+ when "--force", "-f"
1038
+ @force = true
1039
+ i += 1
1040
+ else
1041
+ i += 1
1042
+ end
1043
+ end
1044
+ end
1045
+ end
1046
+ end
1047
+ end