prebake 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/lib/prebake/async_publisher.rb +102 -0
- data/lib/prebake/backends/base.rb +23 -0
- data/lib/prebake/backends/gemstash.rb +97 -0
- data/lib/prebake/backends/http.rb +91 -0
- data/lib/prebake/backends/http_client.rb +40 -0
- data/lib/prebake/backends/s3.rb +81 -0
- data/lib/prebake/cache_key.rb +13 -0
- data/lib/prebake/ext_builder_patch.rb +74 -0
- data/lib/prebake/extractor.rb +45 -0
- data/lib/prebake/hooks.rb +32 -0
- data/lib/prebake/logger.rb +37 -0
- data/lib/prebake/platform.rb +48 -0
- data/lib/prebake/platform_gem_builder.rb +59 -0
- data/lib/prebake.rb +90 -0
- data/plugins.rb +5 -0
- metadata +67 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: dab6849310807672ec24350dbd6343e3adb1875ffd5a84912d3279fcdf4a455d
|
|
4
|
+
data.tar.gz: be4d4d24fee4c87d755eaa85175204772a10e2ba0a1c81047b342f6593a4cab2
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 1cd47249386725f821a1c9e497c1fc5a9c27aa872255a0b745147d96a697f69ca9a648171aa13067c40fe3994882d81cab31cb73a063e7928de88e59c624d4d3
|
|
7
|
+
data.tar.gz: 7997b12e900e9122114ea31539e4e8a448ca898fe4e559abbb4957baa7634277d50e24aa731021eb73c659e66ff39f84f49180c312bd1dceadf445297bf1ab93
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
require_relative "platform_gem_builder"
|
|
5
|
+
require_relative "cache_key"
|
|
6
|
+
require_relative "platform"
|
|
7
|
+
require_relative "logger"
|
|
8
|
+
|
|
9
|
+
module Prebake
|
|
10
|
+
module AsyncPublisher
|
|
11
|
+
@pending_specs = []
|
|
12
|
+
@threads = []
|
|
13
|
+
@mutex = Mutex.new
|
|
14
|
+
|
|
15
|
+
def self.reset!
|
|
16
|
+
@mutex.synchronize do
|
|
17
|
+
@pending_specs.clear
|
|
18
|
+
@threads.clear
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Queue a spec for later processing (no threads, no chdir)
|
|
23
|
+
def self.enqueue(spec, backend)
|
|
24
|
+
@mutex.synchronize { @pending_specs << [spec, backend] }
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# For testing - enqueue a raw block for async execution
|
|
28
|
+
def self.enqueue_block(&block)
|
|
29
|
+
thread = Thread.new do
|
|
30
|
+
block.call
|
|
31
|
+
rescue StandardError => e
|
|
32
|
+
Logger.warn "Error in background task: #{e.message}"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
@mutex.synchronize { @threads << thread }
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Build all queued gems (serial, safe for Dir.chdir) then push in parallel
|
|
39
|
+
def self.wait_for_completion(timeout: 120)
|
|
40
|
+
# First, wait for any raw block threads (from tests)
|
|
41
|
+
block_threads = @mutex.synchronize { @threads.dup }
|
|
42
|
+
block_threads.each { |t| t.join(timeout) }
|
|
43
|
+
|
|
44
|
+
# Then process queued specs
|
|
45
|
+
specs = @mutex.synchronize { @pending_specs.dup }
|
|
46
|
+
return if specs.empty?
|
|
47
|
+
|
|
48
|
+
Logger.info "Building #{specs.size} platform gem(s)..."
|
|
49
|
+
|
|
50
|
+
# Build serially (Dir.chdir is not thread-safe in Ruby 4.0)
|
|
51
|
+
built_gems = specs.filter_map do |spec, backend|
|
|
52
|
+
build_gem(spec, backend)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
if built_gems.any?
|
|
56
|
+
Logger.info "Pushing #{built_gems.size} gem(s) in background..."
|
|
57
|
+
|
|
58
|
+
# Push in parallel (just HTTP, no chdir needed)
|
|
59
|
+
push_threads = built_gems.map do |gem_path, cache_key, checksum, backend|
|
|
60
|
+
Thread.new do
|
|
61
|
+
backend.push(gem_path, cache_key, checksum)
|
|
62
|
+
rescue StandardError => e
|
|
63
|
+
Logger.warn "Push failed for #{cache_key}: #{e.message}"
|
|
64
|
+
ensure
|
|
65
|
+
FileUtils.rm_f(gem_path)
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
push_threads.each { |t| t.join(timeout) }
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
Logger.info "All pushes complete."
|
|
73
|
+
|
|
74
|
+
@mutex.synchronize do
|
|
75
|
+
@pending_specs.clear
|
|
76
|
+
@threads.clear
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def self.build_gem(spec, backend)
|
|
81
|
+
platform = Platform.generalized
|
|
82
|
+
cache_key = CacheKey.for(spec.name, spec.version.to_s, platform)
|
|
83
|
+
|
|
84
|
+
if backend.exists?(cache_key)
|
|
85
|
+
Logger.debug "#{cache_key} already cached, skipping"
|
|
86
|
+
return nil
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
builder = PlatformGemBuilder.new(spec)
|
|
90
|
+
gem_path = builder.build
|
|
91
|
+
checksum = builder.checksum
|
|
92
|
+
|
|
93
|
+
Logger.debug "Built #{cache_key}"
|
|
94
|
+
[gem_path, cache_key, checksum, backend]
|
|
95
|
+
rescue StandardError => e
|
|
96
|
+
Logger.warn "Error building #{spec.name}: #{e.message}"
|
|
97
|
+
nil
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
private_class_method :build_gem
|
|
101
|
+
end
|
|
102
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Prebake
|
|
4
|
+
module Backends
|
|
5
|
+
class Base
|
|
6
|
+
def fetch(cache_key)
|
|
7
|
+
raise NotImplementedError, "#{self.class}#fetch not implemented"
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def fetch_checksum(cache_key)
|
|
11
|
+
raise NotImplementedError, "#{self.class}#fetch_checksum not implemented"
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def push(gem_path, cache_key, checksum)
|
|
15
|
+
raise NotImplementedError, "#{self.class}#push not implemented"
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def exists?(cache_key)
|
|
19
|
+
raise NotImplementedError, "#{self.class}#exists? not implemented"
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "uri"
|
|
4
|
+
require "securerandom"
|
|
5
|
+
require_relative "base"
|
|
6
|
+
require_relative "http_client"
|
|
7
|
+
require_relative "../logger"
|
|
8
|
+
|
|
9
|
+
module Prebake
|
|
10
|
+
module Backends
|
|
11
|
+
class Gemstash < Base
|
|
12
|
+
include HttpClient
|
|
13
|
+
|
|
14
|
+
# Gemstash serves private gems at /private/gems/{name}-{version}-{platform}.gem
|
|
15
|
+
# based on the gem's internal metadata. Our cache key includes the Ruby ABI
|
|
16
|
+
# (e.g., puma-6.4.3-arm64-darwin-ruby4.0.gem) which Gemstash doesn't know about.
|
|
17
|
+
#
|
|
18
|
+
# To make this work, we encode the Ruby ABI into the gem version when building:
|
|
19
|
+
# puma version 6.4.3 becomes 6.4.3.pre.ruby40 in the Gemstash-stored gem.
|
|
20
|
+
# Gemstash then serves it at /private/gems/puma-6.4.3.pre.ruby40-arm64-darwin.gem
|
|
21
|
+
#
|
|
22
|
+
# The push endpoint is POST /api/v1/gems (standard RubyGems push).
|
|
23
|
+
# The fetch endpoint is GET /private/gems/{gem_filename}.
|
|
24
|
+
# Auth is via Authorization header with the Gemstash API key.
|
|
25
|
+
|
|
26
|
+
def initialize(url:, key: nil)
|
|
27
|
+
@url = url.chomp("/")
|
|
28
|
+
@key = key
|
|
29
|
+
|
|
30
|
+
return if @url.start_with?("https://") || ENV.fetch("PREBAKE_ALLOW_INSECURE", "false") == "true"
|
|
31
|
+
return unless @url.start_with?("http://")
|
|
32
|
+
|
|
33
|
+
Logger.warn(
|
|
34
|
+
"Using insecure HTTP connection to #{@url}. " \
|
|
35
|
+
"Set PREBAKE_ALLOW_INSECURE=true to suppress this warning."
|
|
36
|
+
)
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def fetch(cache_key)
|
|
40
|
+
gem_filename = gemstash_filename(cache_key)
|
|
41
|
+
uri = URI("#{@url}/private/gems/#{gem_filename}")
|
|
42
|
+
response = http_request(:get, uri)
|
|
43
|
+
return nil unless response.is_a?(Net::HTTPSuccess)
|
|
44
|
+
|
|
45
|
+
path = File.join(Dir.tmpdir, "prebake-#{SecureRandom.hex(16)}.gem")
|
|
46
|
+
File.binwrite(path, response.body)
|
|
47
|
+
path
|
|
48
|
+
rescue StandardError => e
|
|
49
|
+
Logger.debug "Fetch failed for #{cache_key}: #{e.message}"
|
|
50
|
+
nil
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def fetch_checksum(_cache_key)
|
|
54
|
+
# Gemstash doesn't store arbitrary files alongside gems.
|
|
55
|
+
# Checksum verification is skipped for Gemstash backend.
|
|
56
|
+
nil
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def push(gem_path, cache_key, _checksum)
|
|
60
|
+
uri = URI("#{@url}/private/api/v1/gems")
|
|
61
|
+
gem_content = File.binread(gem_path)
|
|
62
|
+
response = http_request(:post, uri, body: gem_content)
|
|
63
|
+
|
|
64
|
+
case response
|
|
65
|
+
when Net::HTTPSuccess, Net::HTTPConflict
|
|
66
|
+
Logger.info "Pushed #{cache_key} to Gemstash"
|
|
67
|
+
true
|
|
68
|
+
else
|
|
69
|
+
Logger.warn "Failed to push #{cache_key}: #{response.code} #{response.message}"
|
|
70
|
+
false
|
|
71
|
+
end
|
|
72
|
+
rescue StandardError => e
|
|
73
|
+
Logger.warn "Push failed for #{cache_key}: #{e.message}"
|
|
74
|
+
false
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def exists?(cache_key)
|
|
78
|
+
gem_filename = gemstash_filename(cache_key)
|
|
79
|
+
uri = URI("#{@url}/private/gems/#{gem_filename}")
|
|
80
|
+
response = http_request(:head, uri)
|
|
81
|
+
response.is_a?(Net::HTTPSuccess)
|
|
82
|
+
rescue StandardError
|
|
83
|
+
false
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
private
|
|
87
|
+
|
|
88
|
+
def apply_auth_header(request)
|
|
89
|
+
request["Authorization"] = @key if @key
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def gemstash_filename(cache_key)
|
|
93
|
+
cache_key.sub(/-ruby\d+\.\d+(\.gem)/, '\1')
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "uri"
|
|
4
|
+
require "securerandom"
|
|
5
|
+
require_relative "base"
|
|
6
|
+
require_relative "http_client"
|
|
7
|
+
require_relative "../logger"
|
|
8
|
+
|
|
9
|
+
module Prebake
|
|
10
|
+
module Backends
|
|
11
|
+
class Http < Base
|
|
12
|
+
include HttpClient
|
|
13
|
+
|
|
14
|
+
def initialize(url:, token: nil)
|
|
15
|
+
@url = url.chomp("/")
|
|
16
|
+
@token = token
|
|
17
|
+
|
|
18
|
+
return if @url.start_with?("https://") || ENV.fetch("PREBAKE_ALLOW_INSECURE", "false") == "true"
|
|
19
|
+
return unless @url.start_with?("http://")
|
|
20
|
+
|
|
21
|
+
Logger.warn(
|
|
22
|
+
"Using insecure HTTP connection to #{@url}. " \
|
|
23
|
+
"Set PREBAKE_ALLOW_INSECURE=true to suppress this warning."
|
|
24
|
+
)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def fetch(cache_key)
|
|
28
|
+
uri = URI("#{@url}/gems/#{cache_key}")
|
|
29
|
+
response = http_request(:get, uri)
|
|
30
|
+
return nil unless response.is_a?(Net::HTTPSuccess)
|
|
31
|
+
|
|
32
|
+
path = File.join(Dir.tmpdir, "prebake-#{SecureRandom.hex(16)}.gem")
|
|
33
|
+
File.binwrite(path, response.body)
|
|
34
|
+
path
|
|
35
|
+
rescue StandardError => e
|
|
36
|
+
Logger.debug "Fetch failed for #{cache_key}: #{e.message}"
|
|
37
|
+
nil
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def fetch_checksum(cache_key)
|
|
41
|
+
uri = URI("#{@url}/gems/#{cache_key}.sha256")
|
|
42
|
+
response = http_request(:get, uri)
|
|
43
|
+
return nil unless response.is_a?(Net::HTTPSuccess)
|
|
44
|
+
|
|
45
|
+
response.body.strip
|
|
46
|
+
rescue StandardError => e
|
|
47
|
+
Logger.debug "Checksum fetch failed for #{cache_key}: #{e.message}"
|
|
48
|
+
nil
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def push(gem_path, cache_key, checksum)
|
|
52
|
+
gem_uri = URI("#{@url}/gems/#{cache_key}")
|
|
53
|
+
gem_response = http_request(:put, gem_uri, body: File.binread(gem_path))
|
|
54
|
+
|
|
55
|
+
unless gem_response.is_a?(Net::HTTPSuccess)
|
|
56
|
+
Logger.warn "Failed to push #{cache_key}: #{gem_response.code}"
|
|
57
|
+
return false
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Checksum is a secondary artifact - log failure but don't fail the push.
|
|
61
|
+
# The gem itself was pushed successfully; missing checksums can be backfilled.
|
|
62
|
+
checksum_uri = URI("#{@url}/gems/#{cache_key}.sha256")
|
|
63
|
+
checksum_response = http_request(:put, checksum_uri, body: checksum)
|
|
64
|
+
|
|
65
|
+
unless checksum_response.is_a?(Net::HTTPSuccess)
|
|
66
|
+
Logger.warn "Checksum push failed for #{cache_key}: #{checksum_response.code}"
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
Logger.info "Pushed #{cache_key}"
|
|
70
|
+
true
|
|
71
|
+
rescue StandardError => e
|
|
72
|
+
Logger.warn "Push failed for #{cache_key}: #{e.message}"
|
|
73
|
+
false
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def exists?(cache_key)
|
|
77
|
+
uri = URI("#{@url}/gems/#{cache_key}")
|
|
78
|
+
response = http_request(:head, uri)
|
|
79
|
+
response.is_a?(Net::HTTPSuccess)
|
|
80
|
+
rescue StandardError
|
|
81
|
+
false
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
private
|
|
85
|
+
|
|
86
|
+
def apply_auth_header(request)
|
|
87
|
+
request["Authorization"] = "Bearer #{@token}" if @token
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "net/http"
|
|
4
|
+
|
|
5
|
+
module Prebake
|
|
6
|
+
module Backends
|
|
7
|
+
module HttpClient
|
|
8
|
+
TIMEOUT = 30
|
|
9
|
+
HTTP_METHODS = { get: Net::HTTP::Get, head: Net::HTTP::Head,
|
|
10
|
+
post: Net::HTTP::Post, put: Net::HTTP::Put }.freeze
|
|
11
|
+
|
|
12
|
+
private
|
|
13
|
+
|
|
14
|
+
def http_request(method, uri, body: nil)
|
|
15
|
+
request = build_http_request(method, uri)
|
|
16
|
+
if body
|
|
17
|
+
request["Content-Type"] = "application/octet-stream"
|
|
18
|
+
request.body = body
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
Net::HTTP.start(uri.host, uri.port,
|
|
22
|
+
use_ssl: uri.scheme == "https",
|
|
23
|
+
open_timeout: TIMEOUT,
|
|
24
|
+
read_timeout: TIMEOUT) do |http|
|
|
25
|
+
http.request(request)
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def build_http_request(method, uri)
|
|
30
|
+
request = HTTP_METHODS.fetch(method).new(uri)
|
|
31
|
+
apply_auth_header(request)
|
|
32
|
+
request
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def apply_auth_header(request)
|
|
36
|
+
raise NotImplementedError, "#{self.class} must implement apply_auth_header"
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tempfile"
|
|
4
|
+
require "securerandom"
|
|
5
|
+
require_relative "base"
|
|
6
|
+
require_relative "../logger"
|
|
7
|
+
|
|
8
|
+
module Prebake
|
|
9
|
+
module Backends
|
|
10
|
+
class S3 < Base
|
|
11
|
+
def initialize(bucket:, region: "us-east-1", prefix: "prebake")
|
|
12
|
+
@bucket = bucket
|
|
13
|
+
@region = region
|
|
14
|
+
@prefix = prefix
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def fetch(cache_key)
|
|
18
|
+
return nil unless sdk_available?
|
|
19
|
+
|
|
20
|
+
response = client.get_object(bucket: @bucket, key: object_key(cache_key))
|
|
21
|
+
path = File.join(Dir.tmpdir, "prebake-#{SecureRandom.hex(16)}.gem")
|
|
22
|
+
File.binwrite(path, response.body.read)
|
|
23
|
+
path
|
|
24
|
+
rescue StandardError => e
|
|
25
|
+
Logger.debug "S3 fetch failed for #{cache_key}: #{e.message}"
|
|
26
|
+
nil
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def fetch_checksum(cache_key)
|
|
30
|
+
return nil unless sdk_available?
|
|
31
|
+
|
|
32
|
+
response = client.get_object(bucket: @bucket, key: "#{object_key(cache_key)}.sha256")
|
|
33
|
+
response.body.read.strip
|
|
34
|
+
rescue StandardError => e
|
|
35
|
+
Logger.debug "S3 checksum fetch failed for #{cache_key}: #{e.message}"
|
|
36
|
+
nil
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def push(gem_path, cache_key, checksum)
|
|
40
|
+
return false unless sdk_available?
|
|
41
|
+
|
|
42
|
+
File.open(gem_path, "rb") do |file|
|
|
43
|
+
client.put_object(bucket: @bucket, key: object_key(cache_key), body: file)
|
|
44
|
+
end
|
|
45
|
+
client.put_object(bucket: @bucket, key: "#{object_key(cache_key)}.sha256", body: checksum)
|
|
46
|
+
Logger.info "Pushed #{cache_key} to S3"
|
|
47
|
+
true
|
|
48
|
+
rescue StandardError => e
|
|
49
|
+
Logger.warn "S3 push failed for #{cache_key}: #{e.message}"
|
|
50
|
+
false
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def exists?(cache_key)
|
|
54
|
+
return false unless sdk_available?
|
|
55
|
+
|
|
56
|
+
client.head_object(bucket: @bucket, key: object_key(cache_key))
|
|
57
|
+
true
|
|
58
|
+
rescue StandardError
|
|
59
|
+
false
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
private
|
|
63
|
+
|
|
64
|
+
def object_key(cache_key)
|
|
65
|
+
"#{@prefix}/#{cache_key}"
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def sdk_available?
|
|
69
|
+
require "aws-sdk-s3"
|
|
70
|
+
true
|
|
71
|
+
rescue LoadError
|
|
72
|
+
Logger.warn "aws-sdk-s3 not available. Install it to use S3 backend."
|
|
73
|
+
false
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def client
|
|
77
|
+
@client ||= Aws::S3::Client.new(region: @region)
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Prebake
|
|
4
|
+
module CacheKey
|
|
5
|
+
def self.for(name, version, platform)
|
|
6
|
+
"#{name}-#{version}-#{platform}-ruby#{Prebake::RUBY_ABI_VERSION}.gem"
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
def self.checksum_for(name, version, platform)
|
|
10
|
+
"#{self.for(name, version, platform)}.sha256"
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rubygems/ext"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "digest"
|
|
6
|
+
require_relative "cache_key"
|
|
7
|
+
require_relative "platform"
|
|
8
|
+
require_relative "extractor"
|
|
9
|
+
require_relative "logger"
|
|
10
|
+
|
|
11
|
+
module Prebake
|
|
12
|
+
module ExtBuilderPatch
|
|
13
|
+
def build_extensions
|
|
14
|
+
return super unless @spec.extensions.any?
|
|
15
|
+
return super unless Prebake.enabled?
|
|
16
|
+
return super unless Prebake.backend # nil if config failed
|
|
17
|
+
|
|
18
|
+
platform = Platform.generalized
|
|
19
|
+
cache_key = CacheKey.for(@spec.name, @spec.version.to_s, platform)
|
|
20
|
+
|
|
21
|
+
begin
|
|
22
|
+
cached_gem = Prebake.backend.fetch(cache_key)
|
|
23
|
+
rescue StandardError => e
|
|
24
|
+
Logger.debug "Cache fetch error for #{@spec.name}: #{e.message}"
|
|
25
|
+
return super
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
unless cached_gem
|
|
29
|
+
Logger.debug "Cache miss for #{cache_key}"
|
|
30
|
+
return super
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Verify checksum if available
|
|
34
|
+
if verify_checksum(cache_key, cached_gem)
|
|
35
|
+
install_from_cache(cached_gem)
|
|
36
|
+
else
|
|
37
|
+
Logger.warn "Checksum mismatch for #{cache_key}, compiling from source"
|
|
38
|
+
super
|
|
39
|
+
end
|
|
40
|
+
ensure
|
|
41
|
+
FileUtils.rm_f(cached_gem) if cached_gem && File.exist?(cached_gem.to_s)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def verify_checksum(cache_key, gem_path)
|
|
47
|
+
expected = Prebake.backend.fetch_checksum(cache_key)
|
|
48
|
+
|
|
49
|
+
if expected.nil?
|
|
50
|
+
if ENV.fetch("PREBAKE_REQUIRE_CHECKSUM", "false") == "true"
|
|
51
|
+
Logger.warn "No checksum available for #{cache_key} and PREBAKE_REQUIRE_CHECKSUM=true, rejecting"
|
|
52
|
+
return false
|
|
53
|
+
end
|
|
54
|
+
Logger.warn "No checksum available for #{cache_key}, skipping verification"
|
|
55
|
+
return true
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
actual = Digest::SHA256.file(gem_path).hexdigest
|
|
59
|
+
if actual == expected
|
|
60
|
+
true
|
|
61
|
+
else
|
|
62
|
+
Logger.warn "Checksum mismatch for #{cache_key}: expected #{expected}, got #{actual}"
|
|
63
|
+
false
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def install_from_cache(gem_path)
|
|
68
|
+
Logger.info "Installing precompiled #{@spec.name}-#{@spec.version}"
|
|
69
|
+
Extractor.install(gem_path, @spec)
|
|
70
|
+
FileUtils.mkdir_p(File.dirname(@spec.gem_build_complete_path))
|
|
71
|
+
FileUtils.touch(@spec.gem_build_complete_path)
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rubygems/package"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require_relative "logger"
|
|
6
|
+
|
|
7
|
+
module Prebake
|
|
8
|
+
module Extractor
|
|
9
|
+
BINARY_EXTENSIONS = %w[.so .bundle .dll].freeze
|
|
10
|
+
|
|
11
|
+
def self.install(gem_path, spec)
|
|
12
|
+
Logger.debug "Extracting precompiled binaries from #{File.basename(gem_path)}"
|
|
13
|
+
|
|
14
|
+
extracted_count = 0
|
|
15
|
+
|
|
16
|
+
Dir.mktmpdir("prebake-extract") do |tmpdir|
|
|
17
|
+
# Extract all files from the gem into a temp directory
|
|
18
|
+
Gem::Package.new(gem_path).extract_files(tmpdir)
|
|
19
|
+
|
|
20
|
+
# Copy only binary files (.so, .bundle, .dll) to extension_dir
|
|
21
|
+
Dir.glob(File.join(tmpdir, "**/*.{so,bundle,dll}")).each do |binary|
|
|
22
|
+
# Reject symlinks
|
|
23
|
+
next if File.symlink?(binary)
|
|
24
|
+
|
|
25
|
+
# Verify path is within tmpdir (prevent traversal)
|
|
26
|
+
real_binary = File.realpath(binary)
|
|
27
|
+
real_tmpdir = File.realpath(tmpdir)
|
|
28
|
+
next unless real_binary.start_with?("#{real_tmpdir}/")
|
|
29
|
+
|
|
30
|
+
relative = binary.sub("#{tmpdir}/", "")
|
|
31
|
+
dest = File.join(spec.extension_dir, relative)
|
|
32
|
+
FileUtils.mkdir_p(File.dirname(dest))
|
|
33
|
+
FileUtils.cp(binary, dest)
|
|
34
|
+
extracted_count += 1
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
Logger.info "Installed precompiled #{File.basename(gem_path)} " \
|
|
39
|
+
"(#{extracted_count} binary files)"
|
|
40
|
+
rescue StandardError => e
|
|
41
|
+
Logger.warn "Extraction failed for #{File.basename(gem_path)}: #{e.message}"
|
|
42
|
+
raise
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "async_publisher"
|
|
4
|
+
require_relative "logger"
|
|
5
|
+
|
|
6
|
+
module Prebake
|
|
7
|
+
module Hooks
|
|
8
|
+
def self.register!
|
|
9
|
+
return unless defined?(Bundler::Plugin::API)
|
|
10
|
+
|
|
11
|
+
Bundler::Plugin::API.hook("after-install") do |spec_install|
|
|
12
|
+
next unless Prebake.push_enabled?
|
|
13
|
+
next unless spec_install.state == :installed
|
|
14
|
+
|
|
15
|
+
gem_spec = spec_install.spec
|
|
16
|
+
next unless gem_spec.extensions.any?
|
|
17
|
+
next unless gem_spec.platform.to_s == "ruby"
|
|
18
|
+
next unless File.exist?(gem_spec.gem_build_complete_path)
|
|
19
|
+
|
|
20
|
+
AsyncPublisher.enqueue(gem_spec, Prebake.backend)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
Bundler::Plugin::API.hook("after-install-all") do |_deps|
|
|
24
|
+
next unless Prebake.push_enabled?
|
|
25
|
+
|
|
26
|
+
AsyncPublisher.wait_for_completion
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
Logger.debug "Hooks registered"
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Prebake
|
|
4
|
+
module Logger
|
|
5
|
+
LEVELS = { debug: 0, info: 1, warn: 2 }.freeze
|
|
6
|
+
|
|
7
|
+
def self.level
|
|
8
|
+
LEVELS.fetch(ENV.fetch("PREBAKE_LOG_LEVEL", "warn").to_sym, 1)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def self.debug(msg)
|
|
12
|
+
return unless level <= 0
|
|
13
|
+
|
|
14
|
+
output " [prebake] #{msg}"
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def self.info(msg)
|
|
18
|
+
return unless level <= 1
|
|
19
|
+
|
|
20
|
+
output " [prebake] #{msg}"
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def self.warn(msg)
|
|
24
|
+
return unless level <= 2
|
|
25
|
+
|
|
26
|
+
output " [prebake] WARN: #{msg}"
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def self.output(msg)
|
|
30
|
+
if defined?(Bundler) && Bundler.respond_to?(:ui)
|
|
31
|
+
Bundler.ui.info msg
|
|
32
|
+
else
|
|
33
|
+
Kernel.warn msg
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Prebake
|
|
4
|
+
module Platform
|
|
5
|
+
NORMALIZATIONS = [
|
|
6
|
+
[/\Aarm64-darwin(-\d+)?\z/, "arm64-darwin"],
|
|
7
|
+
[/\Ax86_64-darwin(-\d+)?\z/, "x86_64-darwin"],
|
|
8
|
+
[/\Ax86_64-linux-musl\z/, "x86_64-linux-musl"],
|
|
9
|
+
[/\Aaarch64-linux-musl\z/, "aarch64-linux-musl"],
|
|
10
|
+
[/\Ax86_64-linux(-gnu)?\z/, "x86_64-linux"],
|
|
11
|
+
[/\Aaarch64-linux(-gnu)?\z/, "aarch64-linux"]
|
|
12
|
+
].freeze
|
|
13
|
+
|
|
14
|
+
def self.normalize(platform_string)
|
|
15
|
+
NORMALIZATIONS.each do |pattern, normalized|
|
|
16
|
+
return normalized if platform_string.match?(pattern)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
platform_string
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def self.generalized
|
|
23
|
+
platform = Gem::Platform.local
|
|
24
|
+
base = normalize(platform.to_s)
|
|
25
|
+
|
|
26
|
+
# On Linux, detect musl vs glibc - they produce incompatible binaries.
|
|
27
|
+
# Gem::Platform.local may report "gnu" even on musl systems, so we
|
|
28
|
+
# detect explicitly.
|
|
29
|
+
if platform.os == "linux" && !base.include?("musl") && musl?
|
|
30
|
+
base.sub(/\z/, "-musl")
|
|
31
|
+
else
|
|
32
|
+
base
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def self.musl?
|
|
37
|
+
return false unless RUBY_PLATFORM.include?("linux")
|
|
38
|
+
|
|
39
|
+
File.exist?("/lib/ld-musl-x86_64.so.1") ||
|
|
40
|
+
File.exist?("/lib/ld-musl-aarch64.so.1") ||
|
|
41
|
+
begin
|
|
42
|
+
`ldd --version 2>&1`.include?("musl")
|
|
43
|
+
rescue StandardError
|
|
44
|
+
false
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rubygems/package"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "digest"
|
|
6
|
+
require "tempfile"
|
|
7
|
+
require "securerandom"
|
|
8
|
+
require_relative "platform"
|
|
9
|
+
require_relative "logger"
|
|
10
|
+
|
|
11
|
+
module Prebake
|
|
12
|
+
class PlatformGemBuilder
|
|
13
|
+
attr_reader :checksum
|
|
14
|
+
|
|
15
|
+
def initialize(spec)
|
|
16
|
+
@spec = spec
|
|
17
|
+
@checksum = nil
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def build
|
|
21
|
+
# Build in a temp directory; Dir.chdir is scoped to it to
|
|
22
|
+
# isolate from Bundler's working directory.
|
|
23
|
+
Dir.mktmpdir("prebake-build") do |build_dir|
|
|
24
|
+
FileUtils.cp_r(File.join(@spec.gem_dir, "."), build_dir)
|
|
25
|
+
|
|
26
|
+
platform_spec = build_platform_spec(build_dir)
|
|
27
|
+
|
|
28
|
+
gem_file = nil
|
|
29
|
+
Dir.chdir(build_dir) do
|
|
30
|
+
gem_file = Gem::Package.build(platform_spec)
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
built_path = File.join(build_dir, gem_file)
|
|
34
|
+
final = File.join(Dir.tmpdir, "prebake-built-#{SecureRandom.hex(16)}.gem")
|
|
35
|
+
FileUtils.cp(built_path, final)
|
|
36
|
+
|
|
37
|
+
@checksum = Digest::SHA256.file(final).hexdigest
|
|
38
|
+
Logger.debug "Built #{gem_file} (SHA256: #{@checksum})"
|
|
39
|
+
|
|
40
|
+
final
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def build_platform_spec(build_dir)
|
|
47
|
+
platform_spec = @spec.dup
|
|
48
|
+
platform_spec.platform = Gem::Platform.new(Platform.generalized)
|
|
49
|
+
platform_spec.extensions = []
|
|
50
|
+
|
|
51
|
+
prefix = "#{build_dir}/"
|
|
52
|
+
compiled = Dir.glob(File.join(build_dir, "**/*.{so,bundle,dll}"))
|
|
53
|
+
.map { |f| f.delete_prefix(prefix) }
|
|
54
|
+
platform_spec.files = platform_spec.files | compiled
|
|
55
|
+
|
|
56
|
+
platform_spec
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
data/lib/prebake.rb
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "prebake/logger"
|
|
4
|
+
|
|
5
|
+
module Prebake
|
|
6
|
+
class Error < StandardError; end
|
|
7
|
+
|
|
8
|
+
# major.minor only - ABI is stable across patch versions
|
|
9
|
+
RUBY_ABI_VERSION = "#{RbConfig::CONFIG['MAJOR']}.#{RbConfig::CONFIG['MINOR']}".freeze
|
|
10
|
+
|
|
11
|
+
@backend_mutex = Mutex.new
|
|
12
|
+
|
|
13
|
+
def self.enabled?
|
|
14
|
+
ENV.fetch("PREBAKE_ENABLED", "true") != "false"
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def self.push_enabled?
|
|
18
|
+
enabled? && ENV.fetch("PREBAKE_PUSH_ENABLED", "false") == "true"
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.backend
|
|
22
|
+
return @backend if defined?(@backend_loaded)
|
|
23
|
+
|
|
24
|
+
@backend_mutex.synchronize do
|
|
25
|
+
return @backend if defined?(@backend_loaded)
|
|
26
|
+
|
|
27
|
+
@backend_loaded = true
|
|
28
|
+
@backend = load_backend
|
|
29
|
+
end
|
|
30
|
+
rescue Error => e
|
|
31
|
+
Logger.warn "Backend initialization failed: #{e.message}. Plugin disabled for this session."
|
|
32
|
+
@backend = nil
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def self.backend=(backend)
|
|
36
|
+
@backend_loaded = true
|
|
37
|
+
@backend = backend
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def self.reset!
|
|
41
|
+
remove_instance_variable(:@backend_loaded) if defined?(@backend_loaded)
|
|
42
|
+
remove_instance_variable(:@backend) if defined?(@backend)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def self.setup!
|
|
46
|
+
return unless enabled?
|
|
47
|
+
|
|
48
|
+
require_relative "prebake/ext_builder_patch"
|
|
49
|
+
Gem::Ext::Builder.prepend(ExtBuilderPatch)
|
|
50
|
+
|
|
51
|
+
require_relative "prebake/hooks"
|
|
52
|
+
Hooks.register!
|
|
53
|
+
|
|
54
|
+
Logger.info "Plugin active (backend: #{backend_type})"
|
|
55
|
+
rescue StandardError => e
|
|
56
|
+
Logger.warn "Failed to initialize: #{e.message}"
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def self.backend_type
|
|
60
|
+
ENV.fetch("PREBAKE_BACKEND", "http")
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
class << self
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
def load_backend
|
|
67
|
+
case backend_type
|
|
68
|
+
when "gemstash"
|
|
69
|
+
require_relative "prebake/backends/gemstash"
|
|
70
|
+
url = ENV.fetch("PREBAKE_GEMSTASH_URL") { raise Error, "PREBAKE_GEMSTASH_URL is required" }
|
|
71
|
+
key = ENV.fetch("PREBAKE_GEMSTASH_KEY", nil)
|
|
72
|
+
Backends::Gemstash.new(url:, key:)
|
|
73
|
+
when "s3"
|
|
74
|
+
require_relative "prebake/backends/s3"
|
|
75
|
+
bucket = ENV.fetch("PREBAKE_S3_BUCKET") { raise Error, "PREBAKE_S3_BUCKET is required" }
|
|
76
|
+
Backends::S3.new(
|
|
77
|
+
bucket:,
|
|
78
|
+
region: ENV.fetch("PREBAKE_S3_REGION", "us-east-1"),
|
|
79
|
+
prefix: ENV.fetch("PREBAKE_S3_PREFIX", "prebake")
|
|
80
|
+
)
|
|
81
|
+
when "http"
|
|
82
|
+
require_relative "prebake/backends/http"
|
|
83
|
+
url = ENV.fetch("PREBAKE_HTTP_URL", "https://gems.prebake.in")
|
|
84
|
+
Backends::Http.new(url:, token: ENV.fetch("PREBAKE_HTTP_TOKEN", nil))
|
|
85
|
+
else
|
|
86
|
+
raise Error, "Unknown backend: #{backend_type}. Use gemstash, s3, or http."
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
data/plugins.rb
ADDED
metadata
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: prebake
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Thejus Paul
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2026-03-22 00:00:00.000000000 Z
|
|
12
|
+
dependencies: []
|
|
13
|
+
description: Prebake speeds up bundle install by skipping native gem compilation.
|
|
14
|
+
It fetches precompiled binaries for gems like puma, nokogiri, pg, grpc, and bootsnap
|
|
15
|
+
from a shared cache instead of compiling C extensions from source. Drop-in Bundler
|
|
16
|
+
plugin - one line in your Gemfile, no other changes needed. Works out of the box
|
|
17
|
+
with the hosted cache at gems.prebake.in, or self-host with S3-compatible storage
|
|
18
|
+
(AWS S3, Cloudflare R2, Backblaze B2, MinIO) or Gemstash. Works with Ruby 3.2+ and
|
|
19
|
+
Ruby 4.0 on any platform.
|
|
20
|
+
email:
|
|
21
|
+
- thejuspaul@pm.me
|
|
22
|
+
executables: []
|
|
23
|
+
extensions: []
|
|
24
|
+
extra_rdoc_files: []
|
|
25
|
+
files:
|
|
26
|
+
- lib/prebake.rb
|
|
27
|
+
- lib/prebake/async_publisher.rb
|
|
28
|
+
- lib/prebake/backends/base.rb
|
|
29
|
+
- lib/prebake/backends/gemstash.rb
|
|
30
|
+
- lib/prebake/backends/http.rb
|
|
31
|
+
- lib/prebake/backends/http_client.rb
|
|
32
|
+
- lib/prebake/backends/s3.rb
|
|
33
|
+
- lib/prebake/cache_key.rb
|
|
34
|
+
- lib/prebake/ext_builder_patch.rb
|
|
35
|
+
- lib/prebake/extractor.rb
|
|
36
|
+
- lib/prebake/hooks.rb
|
|
37
|
+
- lib/prebake/logger.rb
|
|
38
|
+
- lib/prebake/platform.rb
|
|
39
|
+
- lib/prebake/platform_gem_builder.rb
|
|
40
|
+
- plugins.rb
|
|
41
|
+
homepage: https://github.com/gembakery/prebake
|
|
42
|
+
licenses:
|
|
43
|
+
- MIT
|
|
44
|
+
metadata:
|
|
45
|
+
homepage_uri: https://github.com/gembakery/prebake
|
|
46
|
+
source_code_uri: https://github.com/gembakery/prebake
|
|
47
|
+
rubygems_mfa_required: 'true'
|
|
48
|
+
post_install_message:
|
|
49
|
+
rdoc_options: []
|
|
50
|
+
require_paths:
|
|
51
|
+
- lib
|
|
52
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
53
|
+
requirements:
|
|
54
|
+
- - ">="
|
|
55
|
+
- !ruby/object:Gem::Version
|
|
56
|
+
version: 3.2.0
|
|
57
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
58
|
+
requirements:
|
|
59
|
+
- - ">="
|
|
60
|
+
- !ruby/object:Gem::Version
|
|
61
|
+
version: '0'
|
|
62
|
+
requirements: []
|
|
63
|
+
rubygems_version: 3.4.19
|
|
64
|
+
signing_key:
|
|
65
|
+
specification_version: 4
|
|
66
|
+
summary: Stop compiling. Start installing. Prebake your native gems.
|
|
67
|
+
test_files: []
|