bundler-spinel 0.0.1.pre
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/ARCHITECTURE.md +164 -0
- data/CHANGELOG.md +21 -0
- data/LICENSE +21 -0
- data/README.md +84 -0
- data/RFC.md +120 -0
- data/exe/spinel-compat +4 -0
- data/lib/bundler/spinel/checker.rb +58 -0
- data/lib/bundler/spinel/cli.rb +225 -0
- data/lib/bundler/spinel/command.rb +37 -0
- data/lib/bundler/spinel/engine.rb +96 -0
- data/lib/bundler/spinel/gem_fetcher.rb +45 -0
- data/lib/bundler/spinel/ledger.rb +97 -0
- data/lib/bundler/spinel/platform.rb +24 -0
- data/lib/bundler/spinel/probe.rb +182 -0
- data/lib/bundler/spinel/proxy.rb +154 -0
- data/lib/bundler/spinel/survey.rb +130 -0
- data/lib/bundler/spinel/vendorer.rb +90 -0
- data/lib/bundler/spinel/verifier.rb +100 -0
- data/lib/bundler/spinel/version.rb +7 -0
- data/lib/bundler/spinel.rb +15 -0
- data/plugins.rb +6 -0
- metadata +68 -0
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
require "open3"
|
|
2
|
+
require "tmpdir"
|
|
3
|
+
require "timeout"
|
|
4
|
+
|
|
5
|
+
module Bundler
|
|
6
|
+
module Spinel
|
|
7
|
+
# Decides a compatibility verdict for an unpacked gem against the current
|
|
8
|
+
# Spinel engine. Two complementary signals, because Spinel's failure modes
|
|
9
|
+
# are *not* exit codes:
|
|
10
|
+
#
|
|
11
|
+
# 1. COMPILE SIGNAL. Spinel never exits non-zero on unsupported Ruby; it
|
|
12
|
+
# emits `warning: ... cannot resolve call to 'X' ... (emitting 0)` and
|
|
13
|
+
# degrades the call to a no-op. So we compile the gem's lib entrypoints
|
|
14
|
+
# with `spinel -c` and parse stderr. An unresolved-call warning names
|
|
15
|
+
# the exact missing feature (`unresolved:eval`), which is what makes the
|
|
16
|
+
# verdict forward-compatible — re-probing under a newer engine clears it
|
|
17
|
+
# the moment Spinel learns that call.
|
|
18
|
+
#
|
|
19
|
+
# 2. STATIC RISK SIGNAL. Some constructs (define_method, instance_eval, a
|
|
20
|
+
# bare `send`) are degraded with *no* warning at all, and dead-code
|
|
21
|
+
# elimination can hide an unsupported call that's defined-but-uncalled
|
|
22
|
+
# in a library. So we also scan the source for known-risky tokens. These
|
|
23
|
+
# don't reject on their own (the gem may never hit that path) but they
|
|
24
|
+
# downgrade `clean` → `risky`.
|
|
25
|
+
#
|
|
26
|
+
# NEITHER signal catches Spinel's *silent miscompiles* (local-var-name
|
|
27
|
+
# collapse, Int-0-as-nil). Only the `verified` rung — running the gem's own
|
|
28
|
+
# tests through a Spinel-compiled harness — does. The lock-time gate trusts
|
|
29
|
+
# `clean`; only a curated whitelist / platform variant trusts `verified`.
|
|
30
|
+
class Probe
|
|
31
|
+
# An unsupported *call* — the strongest, most precise signal, and the one
|
|
32
|
+
# that's forward-compatible (names the exact feature Spinel lacks today).
|
|
33
|
+
UNRESOLVED_CALL = /cannot resolve call to '([^']+)'/.freeze
|
|
34
|
+
# A `require "x"` Spinel couldn't follow. Spinel has no load path (plain
|
|
35
|
+
# require resolves only against <spinel>/lib), so a gem's own split files
|
|
36
|
+
# and stdlib deps surface here. Informational, NOT a standalone reject:
|
|
37
|
+
# it's as often a probe limitation as a real incompatibility.
|
|
38
|
+
UNRESOLVED_REQUIRE = /require "([^"]+)" could not be resolved/.freeze
|
|
39
|
+
ANALYZE_FAILED = /\b(analyze failed|fatal)\b/i.freeze
|
|
40
|
+
|
|
41
|
+
# Spinel's analyze pass can spin for minutes on pathological inputs (no
|
|
42
|
+
# internal bound). In a wholesale survey that's indistinguishable from a
|
|
43
|
+
# hang, so we cap each compile and treat an overrun as its own reject
|
|
44
|
+
# reason — `analyze-timeout` is itself a useful roadmap signal (which gems
|
|
45
|
+
# blow up the analyzer). Override with SPINEL_COMPILE_TIMEOUT (seconds).
|
|
46
|
+
COMPILE_TIMEOUT = Integer(ENV.fetch("SPINEL_COMPILE_TIMEOUT", "60"))
|
|
47
|
+
|
|
48
|
+
# token => reason. Tokens Spinel cannot honour and may silently no-op.
|
|
49
|
+
RISK_TOKENS = {
|
|
50
|
+
/\beval\s*\(/ => "eval",
|
|
51
|
+
/\binstance_eval\b/ => "instance_eval",
|
|
52
|
+
/\b(class|module)_eval\b/ => "class_eval",
|
|
53
|
+
/\bdefine_method\b/ => "define_method",
|
|
54
|
+
/\bmethod_missing\b/ => "method_missing",
|
|
55
|
+
/\brespond_to_missing\?/ => "respond_to_missing",
|
|
56
|
+
/\bconst_missing\b/ => "const_missing",
|
|
57
|
+
/\.send\s*\(/ => "send",
|
|
58
|
+
/\bpublic_send\b/ => "public_send",
|
|
59
|
+
/\bObjectSpace\b/ => "objectspace",
|
|
60
|
+
/\b(TracePoint|set_trace_func)\b/ => "tracepoint",
|
|
61
|
+
/\bbinding\b/ => "binding"
|
|
62
|
+
}.freeze
|
|
63
|
+
|
|
64
|
+
def initialize(engine, ledger)
|
|
65
|
+
@engine = engine
|
|
66
|
+
@ledger = ledger
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
# gem_name, version, dir(unpacked) -> recorded Ledger::Verdict
|
|
70
|
+
def probe(gem_name, version, dir)
|
|
71
|
+
@engine.ensure!
|
|
72
|
+
sig = compile_signal(dir, gem_name)
|
|
73
|
+
risks = static_signal(dir)
|
|
74
|
+
# Unfollowed requires are notes, not rejections — record them so a human
|
|
75
|
+
# can see when a verdict is entangled with the no-load-path limitation.
|
|
76
|
+
risks += sig[:requires].map { |r| "needs:#{r}" }
|
|
77
|
+
|
|
78
|
+
verdict, reasons =
|
|
79
|
+
if !sig[:calls].empty?
|
|
80
|
+
# Genuine unsupported call(s): unambiguous, forward-compatible reject.
|
|
81
|
+
["rejected", sig[:calls].map { |s| "unresolved:#{s}" }]
|
|
82
|
+
elsif sig[:timed_out]
|
|
83
|
+
# Analyzer ran past the cap — pathological for Spinel, not the gem.
|
|
84
|
+
["rejected", ["analyze-timeout"]]
|
|
85
|
+
elsif sig[:analyze_failed] || !sig[:exit_ok]
|
|
86
|
+
["rejected", ["analyze-failed"]]
|
|
87
|
+
elsif !static_only_risks(risks).empty?
|
|
88
|
+
["risky", []]
|
|
89
|
+
else
|
|
90
|
+
["clean", []]
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
@ledger.record(@ledger.build(
|
|
94
|
+
gem: gem_name, version: version, rev: @engine.rev,
|
|
95
|
+
verdict: verdict, reasons: reasons.uniq, risks: risks.uniq, probe: "compile+scan"
|
|
96
|
+
))
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
private
|
|
100
|
+
|
|
101
|
+
# Compile the gem's lib entrypoints as a Spinel program; classify stderr.
|
|
102
|
+
def compile_signal(dir, gem_name)
|
|
103
|
+
entries = entrypoints(dir, gem_name)
|
|
104
|
+
return { calls: [], requires: [], analyze_failed: true, exit_ok: false, timed_out: false } if entries.empty?
|
|
105
|
+
|
|
106
|
+
calls = []
|
|
107
|
+
requires = []
|
|
108
|
+
analyze_failed = false
|
|
109
|
+
exit_ok = true
|
|
110
|
+
timed_out = false
|
|
111
|
+
Dir.mktmpdir do |tmp|
|
|
112
|
+
entries.each do |f|
|
|
113
|
+
out, ok, hit_timeout = run_spinel(f, File.join(tmp, "out.c"))
|
|
114
|
+
exit_ok &&= ok
|
|
115
|
+
timed_out ||= hit_timeout
|
|
116
|
+
analyze_failed ||= out =~ ANALYZE_FAILED ? true : false
|
|
117
|
+
out.scan(UNRESOLVED_CALL) { |m| calls << m[0] }
|
|
118
|
+
out.scan(UNRESOLVED_REQUIRE) { |m| requires << m[0] }
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
{ calls: calls.uniq, requires: requires.uniq,
|
|
122
|
+
analyze_failed: analyze_failed, exit_ok: exit_ok, timed_out: timed_out }
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# Run `spinel -c FILE -o OUT_C` with a wall-clock cap. Returns
|
|
126
|
+
# [combined_output, exit_ok, timed_out]. The compiler is a shell script
|
|
127
|
+
# that forks spinel_analyze, so on timeout we KILL the whole process group
|
|
128
|
+
# (pgroup: true makes the child its own group leader) — killing just the
|
|
129
|
+
# spinel pid would orphan the spinning analyzer.
|
|
130
|
+
def run_spinel(file, out_c)
|
|
131
|
+
Open3.popen2e(@engine.bin, "-c", file, "-o", out_c, pgroup: true) do |stdin, out_io, wait_thr|
|
|
132
|
+
stdin.close
|
|
133
|
+
output = +""
|
|
134
|
+
reader = Thread.new { output << out_io.read }
|
|
135
|
+
begin
|
|
136
|
+
Timeout.timeout(COMPILE_TIMEOUT) { wait_thr.value }
|
|
137
|
+
reader.join
|
|
138
|
+
[output, wait_thr.value.success?, false]
|
|
139
|
+
rescue Timeout::Error
|
|
140
|
+
begin
|
|
141
|
+
Process.kill("-KILL", wait_thr.pid)
|
|
142
|
+
rescue StandardError
|
|
143
|
+
nil
|
|
144
|
+
end
|
|
145
|
+
reader.join
|
|
146
|
+
[output, false, true]
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# Risks from the static source scan (exclude the `needs:` require notes).
|
|
152
|
+
def static_only_risks(risks)
|
|
153
|
+
risks.reject { |r| r.start_with?("needs:") }
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
# The gem's conventional require targets: lib/<name>.rb, else top-level
|
|
157
|
+
# lib/*.rb. Spinel inlines their require_relatives, so compiling the entry
|
|
158
|
+
# pulls in the whole tree.
|
|
159
|
+
def entrypoints(dir, gem_name)
|
|
160
|
+
lib = File.join(dir, "lib")
|
|
161
|
+
return [] unless File.directory?(lib)
|
|
162
|
+
|
|
163
|
+
main = File.join(lib, "#{gem_name}.rb")
|
|
164
|
+
return [main] if File.exist?(main)
|
|
165
|
+
|
|
166
|
+
Dir[File.join(lib, "*.rb")]
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def static_signal(dir)
|
|
170
|
+
risks = []
|
|
171
|
+
# C-extension gems can't be compiled by Spinel at all.
|
|
172
|
+
risks << "c-extension" if Dir[File.join(dir, "ext", "**", "*.{c,cpp,cc,h}")].any?
|
|
173
|
+
|
|
174
|
+
Dir[File.join(dir, "lib", "**", "*.rb")].each do |f|
|
|
175
|
+
src = File.read(f)
|
|
176
|
+
RISK_TOKENS.each { |re, reason| risks << reason if src =~ re }
|
|
177
|
+
end
|
|
178
|
+
risks.uniq
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
require "webrick"
|
|
2
|
+
require "digest"
|
|
3
|
+
require "json"
|
|
4
|
+
require "rubygems/package"
|
|
5
|
+
require "time"
|
|
6
|
+
|
|
7
|
+
module Bundler
|
|
8
|
+
module Spinel
|
|
9
|
+
# A curated RubyGems *source* (Compact Index protocol) that serves only
|
|
10
|
+
# vetted gems. Point a Gemfile at it:
|
|
11
|
+
#
|
|
12
|
+
# source "http://localhost:9292"
|
|
13
|
+
# gem "cleangem"
|
|
14
|
+
#
|
|
15
|
+
# and `bundle lock` resolves *only against vetted gems*. A gem that isn't
|
|
16
|
+
# served (no acceptable verdict for the pinned engine rev) becomes a plain
|
|
17
|
+
# "could not find compatible versions" resolution failure — no plugin, no
|
|
18
|
+
# engine-directive trick. The whitelist is not a file: it's the acceptable
|
|
19
|
+
# subset of the ledger at this rev, plus a local store of .gem artifacts we
|
|
20
|
+
# built/verified ourselves.
|
|
21
|
+
#
|
|
22
|
+
# Empirically the third-party rubygems ecosystem is ~all-rejected today, so
|
|
23
|
+
# the load-bearing mode is `:store` — a directory of our own Spinel-vetted
|
|
24
|
+
# .gem files. (Filtered read-through of upstream is a documented extension.)
|
|
25
|
+
#
|
|
26
|
+
# NOTE: this CRuby/WEBrick implementation is the MVP that proves Bundler
|
|
27
|
+
# resolves against it. The dogfood target is to serve the same endpoints
|
|
28
|
+
# from a Spinel-compiled Tep app — see ARCHITECTURE.md §"Dogfooding".
|
|
29
|
+
class Proxy
|
|
30
|
+
# store: dir of vetted *.gem files (the curated artifacts).
|
|
31
|
+
def initialize(store:, ledger: Ledger.new, engine: Engine.new,
|
|
32
|
+
min_verdict: :verified)
|
|
33
|
+
@store = store
|
|
34
|
+
@ledger = ledger
|
|
35
|
+
@engine = engine
|
|
36
|
+
@min_verdict = min_verdict
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# name => { version => spec } for every vetted gem in the store.
|
|
40
|
+
def catalog
|
|
41
|
+
@catalog ||= Dir[File.join(@store, "*.gem")].each_with_object({}) do |path, acc|
|
|
42
|
+
spec = Gem::Package.new(path).spec
|
|
43
|
+
next unless acceptable?(spec.name, spec.version.to_s)
|
|
44
|
+
|
|
45
|
+
(acc[spec.name] ||= {})[spec.version.to_s] = { spec: spec, path: path }
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Write the curated source as a *static* Compact Index tree:
|
|
50
|
+
# out/names out/versions out/info/<gem> out/gems/<file>.gem
|
|
51
|
+
# All digest/JSON happens here, offline, in CRuby. The result is plain
|
|
52
|
+
# text + file bytes — so the dogfood server (Tep/Spinel, which has neither
|
|
53
|
+
# digest nor JSON — probed 2026-05-26) only has to serve static files.
|
|
54
|
+
def write_static(out)
|
|
55
|
+
require "fileutils"
|
|
56
|
+
FileUtils.mkdir_p(File.join(out, "info"))
|
|
57
|
+
FileUtils.mkdir_p(File.join(out, "gems"))
|
|
58
|
+
File.write(File.join(out, "names"), names_body)
|
|
59
|
+
File.write(File.join(out, "versions"), versions_body)
|
|
60
|
+
catalog.each_key { |name| File.write(File.join(out, "info", name), info_body(name)) }
|
|
61
|
+
catalog.values.flat_map(&:values).each do |e|
|
|
62
|
+
FileUtils.cp(e[:path], File.join(out, "gems", File.basename(e[:path])))
|
|
63
|
+
end
|
|
64
|
+
out
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def serve(port: 9292, quiet: true)
|
|
68
|
+
server = WEBrick::HTTPServer.new(
|
|
69
|
+
Port: port,
|
|
70
|
+
Logger: WEBrick::Log.new(quiet ? File::NULL : $stderr),
|
|
71
|
+
AccessLog: []
|
|
72
|
+
)
|
|
73
|
+
mount(server)
|
|
74
|
+
trap("INT") { server.shutdown }
|
|
75
|
+
warn "[spinel-proxy] curated source on http://localhost:#{port} " \
|
|
76
|
+
"(#{catalog.size} gems, min=#{@min_verdict}, rev=#{@engine.rev})"
|
|
77
|
+
server.start
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
private
|
|
81
|
+
|
|
82
|
+
def acceptable?(name, version)
|
|
83
|
+
v = @ledger.lookup(name, version, @engine.rev)
|
|
84
|
+
return false unless v
|
|
85
|
+
|
|
86
|
+
case @min_verdict
|
|
87
|
+
when :verified then v.verified?
|
|
88
|
+
when :clean then v.verified? || v.clean?
|
|
89
|
+
when :risky then !v.rejected?
|
|
90
|
+
else false
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def mount(server)
|
|
95
|
+
server.mount_proc("/names") { |_, res| text(res, names_body) }
|
|
96
|
+
server.mount_proc("/versions") { |_, res| text(res, versions_body) }
|
|
97
|
+
server.mount_proc("/info") do |req, res|
|
|
98
|
+
gem = req.path.sub(%r{\A/info/}, "")
|
|
99
|
+
info = info_body(gem)
|
|
100
|
+
info ? text(res, info) : (res.status = 404)
|
|
101
|
+
end
|
|
102
|
+
server.mount_proc("/gems") do |req, res|
|
|
103
|
+
file = File.basename(req.path)
|
|
104
|
+
entry = catalog.values.flat_map(&:values).find { |e| File.basename(e[:path]) == file }
|
|
105
|
+
if entry
|
|
106
|
+
res["Content-Type"] = "application/octet-stream"
|
|
107
|
+
res.body = File.binread(entry[:path])
|
|
108
|
+
else
|
|
109
|
+
res.status = 404
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# --- Compact Index bodies ---------------------------------------------
|
|
115
|
+
|
|
116
|
+
def names_body
|
|
117
|
+
"---\n" + catalog.keys.sort.join("\n") + "\n"
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def versions_body
|
|
121
|
+
out = +"created_at: #{Time.now.utc.iso8601}\n---\n"
|
|
122
|
+
catalog.sort.each do |name, versions|
|
|
123
|
+
vs = versions.keys.sort.join(",")
|
|
124
|
+
out << "#{name} #{vs} #{::Digest::MD5.hexdigest(info_body(name))}\n"
|
|
125
|
+
end
|
|
126
|
+
out
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def info_body(name)
|
|
130
|
+
gem = catalog[name]
|
|
131
|
+
return nil unless gem
|
|
132
|
+
|
|
133
|
+
out = +"---\n"
|
|
134
|
+
gem.sort.each do |version, entry|
|
|
135
|
+
spec = entry[:spec]
|
|
136
|
+
deps = spec.runtime_dependencies.map do |d|
|
|
137
|
+
"#{d.name}:#{d.requirement.requirements.map { |op, v| "#{op} #{v}" }.join('&')}"
|
|
138
|
+
end.join(",")
|
|
139
|
+
sha = ::Digest::SHA256.hexdigest(File.binread(entry[:path]))
|
|
140
|
+
ruby = spec.required_ruby_version.to_s
|
|
141
|
+
out << "#{version} #{deps}|checksum:#{sha}"
|
|
142
|
+
out << ",ruby:#{ruby}" unless ruby.empty? || ruby == ">= 0"
|
|
143
|
+
out << "\n"
|
|
144
|
+
end
|
|
145
|
+
out
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def text(res, body)
|
|
149
|
+
res["Content-Type"] = "text/plain"
|
|
150
|
+
res.body = body
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
require "open3"
|
|
2
|
+
require "set"
|
|
3
|
+
|
|
4
|
+
module Bundler
|
|
5
|
+
module Spinel
|
|
6
|
+
# Wholesale review: probe a large list of gems and aggregate the results.
|
|
7
|
+
# The point isn't a pass/fail — it's the *histogram of rejection reasons*,
|
|
8
|
+
# which directly prioritises what Spinel should support next (see RFC asks).
|
|
9
|
+
#
|
|
10
|
+
# Embarrassingly parallel: each gem is an independent fetch + compile, both
|
|
11
|
+
# subprocess-bound (Open3 releases the GVL), so a thread pool scales across
|
|
12
|
+
# cores. Verdicts are cached in the ledger, so a survey is resumable and
|
|
13
|
+
# re-runnable; only ledger writes are serialised.
|
|
14
|
+
class Survey
|
|
15
|
+
def initialize(engine: Engine.new, ledger: Ledger.new, jobs: 4)
|
|
16
|
+
@engine = engine
|
|
17
|
+
@ledger = ledger
|
|
18
|
+
@jobs = jobs
|
|
19
|
+
@fetcher = GemFetcher.new
|
|
20
|
+
@probe = Probe.new(@engine, @ledger)
|
|
21
|
+
@mutex = Mutex.new
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# names: Array<String>. Probes each at its latest version (ledger-cached).
|
|
25
|
+
# Returns the Array<Ledger::Verdict> for the surveyed set.
|
|
26
|
+
def run(names, progress: $stderr)
|
|
27
|
+
@engine.ensure!
|
|
28
|
+
queue = Queue.new
|
|
29
|
+
names.each { |n| queue << n }
|
|
30
|
+
results = []
|
|
31
|
+
done = 0
|
|
32
|
+
total = names.size
|
|
33
|
+
|
|
34
|
+
workers = Array.new([@jobs, total].min) do
|
|
35
|
+
Thread.new do
|
|
36
|
+
until queue.empty?
|
|
37
|
+
name = (queue.pop(true) rescue break)
|
|
38
|
+
v = probe_one(name)
|
|
39
|
+
@mutex.synchronize do
|
|
40
|
+
results << v if v
|
|
41
|
+
done += 1
|
|
42
|
+
progress&.print("\r[survey] #{done}/#{total} #{name.ljust(30)}")
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
workers.each(&:join)
|
|
48
|
+
progress&.puts("\r[survey] #{done}/#{total} done#{' ' * 30}")
|
|
49
|
+
results.compact
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Aggregate a markdown report from ledger verdicts for `names` at this rev.
|
|
53
|
+
#
|
|
54
|
+
# Reads straight from the ledger — no network. The just-run probes already
|
|
55
|
+
# recorded a verdict (with its resolved version) per surveyed gem at this
|
|
56
|
+
# rev, so re-resolving each gem's latest version online would only repeat
|
|
57
|
+
# work and serialise a 1k-name survey behind 1k `gem list -r` calls. We
|
|
58
|
+
# take the *last* current-rev entry per gem: append-only means a re-probe
|
|
59
|
+
# supersedes, and the survey probes a gem at one (latest) version per run.
|
|
60
|
+
def report(names)
|
|
61
|
+
wanted = names.to_set
|
|
62
|
+
rev = @engine.rev
|
|
63
|
+
latest = {}
|
|
64
|
+
@ledger.each do |v|
|
|
65
|
+
latest[v.gem] = v if v.rev == rev && wanted.include?(v.gem)
|
|
66
|
+
end
|
|
67
|
+
verdicts = latest.values
|
|
68
|
+
counts = Hash.new(0)
|
|
69
|
+
reasons = Hash.new(0)
|
|
70
|
+
verdicts.each do |v|
|
|
71
|
+
counts[v.verdict] += 1
|
|
72
|
+
(v.reasons + v.risks).each { |r| reasons[normalize(r)] += 1 }
|
|
73
|
+
end
|
|
74
|
+
render(verdicts.size, counts, reasons)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
private
|
|
78
|
+
|
|
79
|
+
def probe_one(name)
|
|
80
|
+
version = latest_version(name) or return nil
|
|
81
|
+
cached = @ledger.lookup(name, version, @engine.rev)
|
|
82
|
+
return cached if cached
|
|
83
|
+
|
|
84
|
+
dir = @fetcher.fetch(name, version)
|
|
85
|
+
# Probe runs spinel (CPU-bound, GVL released) — this is the whole point
|
|
86
|
+
# of the thread pool, so it must NOT hold @mutex. The only shared write
|
|
87
|
+
# is the ledger append, which Ledger#record serialises internally.
|
|
88
|
+
@probe.probe(name, version, dir)
|
|
89
|
+
rescue StandardError => e
|
|
90
|
+
@mutex.synchronize { warn "\n[survey] #{name}: #{e.message}" }
|
|
91
|
+
nil
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def latest_version(name)
|
|
95
|
+
out, st = Open3.capture2e("gem", "list", "-r", "-e", name)
|
|
96
|
+
return nil unless st.success?
|
|
97
|
+
|
|
98
|
+
out[/#{Regexp.escape(name)} \(([^,)]+)/, 1]
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Collapse `unresolved:foo`/`risk:bar`/`needs:baz` into ranked buckets.
|
|
102
|
+
def normalize(reason)
|
|
103
|
+
reason
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def render(n, counts, reasons)
|
|
107
|
+
ok = (counts["clean"] + counts["verified"])
|
|
108
|
+
lines = []
|
|
109
|
+
lines << "# Spinel gem-compatibility survey"
|
|
110
|
+
lines << ""
|
|
111
|
+
lines << "- engine rev: `#{@engine.rev}`"
|
|
112
|
+
lines << "- gems surveyed: **#{n}**"
|
|
113
|
+
lines << "- compatible (clean+verified): **#{ok}** (#{pct(ok, n)}) · " \
|
|
114
|
+
"risky: #{counts['risky']} · rejected: #{counts['rejected']}"
|
|
115
|
+
lines << ""
|
|
116
|
+
lines << "## Top blockers (what to teach Spinel next)"
|
|
117
|
+
lines << ""
|
|
118
|
+
lines << "| count | reason |"
|
|
119
|
+
lines << "|---|---|"
|
|
120
|
+
reasons.sort_by { |_, c| -c }.first(25).each { |r, c| lines << "| #{c} | `#{r}` |" }
|
|
121
|
+
lines << ""
|
|
122
|
+
lines.join("\n")
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def pct(a, b)
|
|
126
|
+
b.zero? ? "0%" : "#{(100.0 * a / b).round(1)}%"
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
require "bundler"
|
|
2
|
+
require "bundler/lockfile_parser"
|
|
3
|
+
require "fileutils"
|
|
4
|
+
|
|
5
|
+
module Bundler
|
|
6
|
+
module Spinel
|
|
7
|
+
# "Make it work" — the plugin's primary job. Spinel has no load path
|
|
8
|
+
# (plain `require "x"` resolves only against <spinel>/lib) and inlines
|
|
9
|
+
# `require_relative`. So to actually *use* a resolved dependency in a Spinel
|
|
10
|
+
# build, its source has to be placed somewhere Spinel will follow, with the
|
|
11
|
+
# require wiring generated. This is the reusable form of what projects do by
|
|
12
|
+
# hand today (e.g. Toy's build_tep_app.sh concatenation, Roundhouse
|
|
13
|
+
# vendoring part of Tep).
|
|
14
|
+
#
|
|
15
|
+
# Given a Gemfile.lock, vendor each gem's `lib/` into `<into>/<name>/` and
|
|
16
|
+
# emit `<into>/deps.rb` — a manifest of `require_relative`s in lock order. A
|
|
17
|
+
# Spinel program then just does `require_relative "vendor/spinel/deps"`.
|
|
18
|
+
#
|
|
19
|
+
# Gating is layered on but advisory here: placement and compatibility are
|
|
20
|
+
# different jobs. `vendor` warns on non-compatible gems (so the experience
|
|
21
|
+
# is nicer) but still places them; `check` is the hard gate.
|
|
22
|
+
class Vendorer
|
|
23
|
+
def initialize(engine: Engine.new, ledger: Ledger.new)
|
|
24
|
+
@engine = engine
|
|
25
|
+
@ledger = ledger
|
|
26
|
+
@fetcher = GemFetcher.new
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def vendor(lockfile = "Gemfile.lock", into: "vendor/spinel", warn_incompatible: true)
|
|
30
|
+
parsed = Bundler::LockfileParser.new(File.read(lockfile))
|
|
31
|
+
into = File.expand_path(into)
|
|
32
|
+
FileUtils.mkdir_p(into)
|
|
33
|
+
|
|
34
|
+
manifest = []
|
|
35
|
+
parsed.specs.each do |spec|
|
|
36
|
+
name = spec.name
|
|
37
|
+
version = spec.version.to_s
|
|
38
|
+
src = @fetcher.fetch(name, version)
|
|
39
|
+
dest = File.join(into, name)
|
|
40
|
+
place(src, dest)
|
|
41
|
+
manifest << require_target(name, dest)
|
|
42
|
+
note_compat(name, version) if warn_incompatible
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
write_manifest(into, manifest)
|
|
46
|
+
{ into: into, count: manifest.size }
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
def place(src, dest)
|
|
52
|
+
FileUtils.rm_rf(dest)
|
|
53
|
+
FileUtils.mkdir_p(dest)
|
|
54
|
+
%w[lib].each do |sub|
|
|
55
|
+
s = File.join(src, sub)
|
|
56
|
+
FileUtils.cp_r(s, dest) if File.directory?(s)
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# The relative path a Spinel program require_relatives. Spinel inlines it
|
|
61
|
+
# and follows the gem's own require_relatives from there.
|
|
62
|
+
def require_target(name, dest)
|
|
63
|
+
base = File.basename(dest)
|
|
64
|
+
main = File.join(dest, "lib", "#{name}.rb")
|
|
65
|
+
if File.exist?(main)
|
|
66
|
+
"#{base}/lib/#{name}"
|
|
67
|
+
else
|
|
68
|
+
first = Dir[File.join(dest, "lib", "*.rb")].sort.first
|
|
69
|
+
first ? "#{base}/lib/#{File.basename(first, '.rb')}" : nil
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def note_compat(name, version)
|
|
74
|
+
v = @ledger.lookup(name, version, @engine.rev)
|
|
75
|
+
return if v&.clean? || v&.verified?
|
|
76
|
+
|
|
77
|
+
label = v ? v.verdict : "unprobed"
|
|
78
|
+
warn "[vendor] #{name} #{version}: #{label} for #{@engine.rev} " \
|
|
79
|
+
"— may not compile (run `spinel-compat check`)"
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def write_manifest(into, targets)
|
|
83
|
+
body = +"# Generated by bundler-spinel. require_relative this from a\n" \
|
|
84
|
+
"# Spinel program to pull in vendored dependencies (lock order).\n"
|
|
85
|
+
targets.compact.each { |t| body << %{require_relative "#{t}"\n} }
|
|
86
|
+
File.write(File.join(into, "deps.rb"), body)
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
require "open3"
|
|
2
|
+
|
|
3
|
+
module Bundler
|
|
4
|
+
module Spinel
|
|
5
|
+
# The `verified` rung: differential testing. Runs a smoke program that
|
|
6
|
+
# exercises the gem once under CRuby and once compiled by Spinel, and
|
|
7
|
+
# compares stdout. This is the *only* signal that catches Spinel's silent
|
|
8
|
+
# miscompiles (local-var-name collapse, Int-0-as-nil) — they emit no warning
|
|
9
|
+
# and exit 0, so the cheap probe can't see them, but a differential run does:
|
|
10
|
+
# CRuby and the miscompiled binary diverge.
|
|
11
|
+
#
|
|
12
|
+
# match -> verified (CRuby and Spinel agree on this smoke)
|
|
13
|
+
# mismatch -> rejected (reason: miscompile, with a short diff)
|
|
14
|
+
# no build -> rejected (reason: build-error / run-error)
|
|
15
|
+
#
|
|
16
|
+
# The smoke is the unit of trust. A require-only default smoke catches
|
|
17
|
+
# load-time divergence; pass `--smoke FILE` (a snippet that drives the gem's
|
|
18
|
+
# API and prints deterministic output) to verify behaviour. Verification is
|
|
19
|
+
# only as good as the smoke — which is why it's opt-in and human-supplied.
|
|
20
|
+
class Verifier
|
|
21
|
+
HARNESS = "__spinel_verify.rb".freeze
|
|
22
|
+
|
|
23
|
+
def initialize(engine, ledger)
|
|
24
|
+
@engine = engine
|
|
25
|
+
@ledger = ledger
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def verify(gem_name, version, dir, smoke: nil)
|
|
29
|
+
@engine.ensure!
|
|
30
|
+
harness = File.join(dir, HARNESS)
|
|
31
|
+
File.write(harness, harness_source(gem_name, dir, smoke))
|
|
32
|
+
|
|
33
|
+
ruby_out, _, ruby_ok = run_ruby(harness)
|
|
34
|
+
spin_out, spin_err, spin_ok = run_spinel(harness)
|
|
35
|
+
|
|
36
|
+
verdict, reasons = classify(ruby_ok, spin_ok, ruby_out, spin_out, spin_err)
|
|
37
|
+
@ledger.record(@ledger.build(
|
|
38
|
+
gem: gem_name, version: version, rev: @engine.rev,
|
|
39
|
+
verdict: verdict, reasons: reasons, probe: "verify"
|
|
40
|
+
))
|
|
41
|
+
ensure
|
|
42
|
+
File.delete(harness) if harness && File.exist?(harness)
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def classify(ruby_ok, spin_ok, ruby_out, spin_out, spin_err)
|
|
48
|
+
unless ruby_ok
|
|
49
|
+
# Smoke is broken under plain Ruby — can't draw a conclusion.
|
|
50
|
+
return ["risky", ["smoke-error:cruby"]]
|
|
51
|
+
end
|
|
52
|
+
return ["rejected", ["build-or-run-error"] + spin_err.lines.grep(/error|fatal/i).first(2).map(&:strip)] unless spin_ok
|
|
53
|
+
|
|
54
|
+
if ruby_out == spin_out
|
|
55
|
+
["verified", []]
|
|
56
|
+
else
|
|
57
|
+
["rejected", ["miscompile", "diff:#{first_diff(ruby_out, spin_out)}"]]
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
# require_relative resolves against the harness's own dir (the gem root),
|
|
62
|
+
# and Spinel inlines it — so this follows require_relative-split gems
|
|
63
|
+
# natively. (Plain `require "gem/part"` gems still under-resolve; that's the
|
|
64
|
+
# documented load-path limitation.)
|
|
65
|
+
def harness_source(gem_name, dir, smoke)
|
|
66
|
+
entry = File.exist?(File.join(dir, "lib", "#{gem_name}.rb")) ? "lib/#{gem_name}" : nil
|
|
67
|
+
body = smoke ? File.read(smoke) : %{puts "spinel-verify: loaded #{gem_name}"}
|
|
68
|
+
src = +""
|
|
69
|
+
src << %{require_relative "#{entry}"\n} if entry
|
|
70
|
+
src << body << "\n"
|
|
71
|
+
src
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def run_ruby(file)
|
|
75
|
+
out, err, st = Open3.capture3("ruby", file)
|
|
76
|
+
[out, err, st.success?]
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def run_spinel(file)
|
|
80
|
+
bin = file.sub(/\.rb$/, ".bin")
|
|
81
|
+
_, cerr, cst = Open3.capture3(@engine.bin, file, "-o", bin)
|
|
82
|
+
return ["", cerr, false] unless cst.success? && File.executable?(bin)
|
|
83
|
+
|
|
84
|
+
out, err, st = Open3.capture3(bin)
|
|
85
|
+
[out, (cerr + err), st.success?]
|
|
86
|
+
ensure
|
|
87
|
+
File.delete(bin) if bin && File.exist?(bin)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def first_diff(a, b)
|
|
91
|
+
al = a.lines
|
|
92
|
+
bl = b.lines
|
|
93
|
+
al.each_index do |i|
|
|
94
|
+
return "L#{i + 1} cruby=#{al[i].inspect} spinel=#{bl[i].inspect}" if al[i] != bl[i]
|
|
95
|
+
end
|
|
96
|
+
"len #{al.size}!=#{bl.size}"
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
module Bundler
|
|
2
|
+
module Spinel
|
|
3
|
+
class Error < StandardError; end
|
|
4
|
+
end
|
|
5
|
+
end
|
|
6
|
+
|
|
7
|
+
require_relative "spinel/version"
|
|
8
|
+
require_relative "spinel/engine"
|
|
9
|
+
require_relative "spinel/ledger"
|
|
10
|
+
require_relative "spinel/gem_fetcher"
|
|
11
|
+
require_relative "spinel/probe"
|
|
12
|
+
require_relative "spinel/verifier"
|
|
13
|
+
require_relative "spinel/vendorer"
|
|
14
|
+
require_relative "spinel/survey"
|
|
15
|
+
require_relative "spinel/checker"
|
data/plugins.rb
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
# Bundler plugin entry point. Read by Bundler when this gem is installed as a
|
|
2
|
+
# plugin (`bundle plugin install bundler-spinel`). Registers the gate commands.
|
|
3
|
+
require_relative "lib/bundler/spinel/command"
|
|
4
|
+
|
|
5
|
+
Bundler::Plugin::API.command("spinel-lock", Bundler::Spinel::Command)
|
|
6
|
+
Bundler::Plugin::API.command("spinel-check", Bundler::Spinel::Command)
|