still_active 1.6.0 → 3.0.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +88 -0
- data/README.md +131 -17
- data/lib/helpers/activity_helper.rb +41 -8
- data/lib/helpers/bundler_helper.rb +87 -26
- data/lib/helpers/catalog_index.rb +4 -1
- data/lib/helpers/constraint_helper.rb +208 -0
- data/lib/helpers/cvss_helper.rb +63 -0
- data/lib/helpers/cyclonedx_helper.rb +44 -12
- data/lib/helpers/diff_markdown_helper.rb +36 -17
- data/lib/helpers/endoflife_helper.rb +114 -0
- data/lib/helpers/http_helper.rb +117 -15
- data/lib/helpers/lockfile_dependency_parser.rb +99 -0
- data/lib/helpers/lockfile_indexer.rb +3 -0
- data/lib/helpers/markdown_escape.rb +58 -0
- data/lib/helpers/markdown_helper.rb +145 -6
- data/lib/helpers/pep440_helper.rb +100 -0
- data/lib/helpers/python_helper.rb +19 -0
- data/lib/helpers/ruby_advisory_db.rb +23 -0
- data/lib/helpers/ruby_helper.rb +18 -28
- data/lib/helpers/runtime_ceiling_helper.rb +121 -0
- data/lib/helpers/sarif_helper.rb +157 -28
- data/lib/helpers/semver_satisfaction.rb +68 -0
- data/lib/helpers/status_helper.rb +68 -0
- data/lib/helpers/summary_helper.rb +52 -0
- data/lib/helpers/terminal_helper.rb +167 -19
- data/lib/helpers/version_helper.rb +16 -1
- data/lib/helpers/vulnerability_helper.rb +73 -7
- data/lib/still_active/artifactory_client.rb +166 -0
- data/lib/still_active/ceiling_reconciler.rb +43 -0
- data/lib/still_active/cli.rb +292 -20
- data/lib/still_active/config.rb +59 -2
- data/lib/still_active/config_file.rb +207 -0
- data/lib/still_active/deps_dev_client.rb +140 -9
- data/lib/still_active/diff.rb +81 -2
- data/lib/still_active/ecosystem_lens.rb +284 -0
- data/lib/still_active/ecosystems_client.rb +123 -0
- data/lib/still_active/forgejo_client.rb +50 -0
- data/lib/still_active/github_client.rb +126 -0
- data/lib/still_active/gitlab_client.rb +15 -20
- data/lib/still_active/options.rb +58 -6
- data/lib/still_active/osv_client.rb +147 -0
- data/lib/still_active/poison_security_correlator.rb +232 -0
- data/lib/still_active/pypi_client.rb +56 -0
- data/lib/still_active/repository.rb +12 -4
- data/lib/still_active/sarif/rules.rb +35 -11
- data/lib/still_active/sbom_reader.rb +191 -0
- data/lib/still_active/sbom_workflow.rb +96 -0
- data/lib/still_active/suppressions.rb +143 -0
- data/lib/still_active/version.rb +1 -1
- data/lib/still_active/workflow.rb +312 -61
- data/lib/still_active.rb +3 -0
- data/still_active.gemspec +34 -9
- metadata +72 -11
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "deps_dev_client"
|
|
4
|
+
require_relative "osv_client"
|
|
5
|
+
require_relative "ecosystems_client"
|
|
6
|
+
require_relative "github_client"
|
|
7
|
+
require_relative "pypi_client"
|
|
8
|
+
require "time"
|
|
9
|
+
require_relative "../helpers/activity_helper"
|
|
10
|
+
require_relative "../helpers/constraint_helper"
|
|
11
|
+
require_relative "../helpers/libyear_helper"
|
|
12
|
+
require_relative "../helpers/pep440_helper"
|
|
13
|
+
require_relative "../helpers/runtime_ceiling_helper"
|
|
14
|
+
require_relative "../helpers/vulnerability_helper"
|
|
15
|
+
|
|
16
|
+
module StillActive
|
|
17
|
+
# The cross-ecosystem maintenance lens. Given one SBOM-derived dependency
|
|
18
|
+
# ({ecosystem, name, version}), it assembles the same maintenance signals the
|
|
19
|
+
# native Ruby path produces -- latest release date, archived, advisories,
|
|
20
|
+
# scorecard -- into a gem_data hash StatusHelper.gem_status can read directly.
|
|
21
|
+
# The breadth counterpart to the depth Bundler path gives Ruby.
|
|
22
|
+
#
|
|
23
|
+
# Security: the SBOM supplies only ecosystem/name/version. The repository is
|
|
24
|
+
# discovered from deps.dev (a public source still_active opts into), never from
|
|
25
|
+
# a lockfile-derived URL, so a hostile SBOM can't redirect a lookup. Anything
|
|
26
|
+
# unresolvable (a private package deps.dev doesn't index) degrades to nil
|
|
27
|
+
# signals -> :unknown, never a fabricated "ok".
|
|
28
|
+
module EcosystemLens
|
|
29
|
+
extend self
|
|
30
|
+
|
|
31
|
+
# still_active ecosystem symbol -> ecosyste.ms registry name for the declared
|
|
32
|
+
# dependency (poison-pill) lookup. Only the ecosystems where the ecosyste.ms
|
|
33
|
+
# registry name and the deps.dev package name align cleanly are wired; maven
|
|
34
|
+
# (group:artifact) and go (module paths) need name-format handling and are
|
|
35
|
+
# left out for now, so they simply carry no constraint signal rather than a
|
|
36
|
+
# wrong one.
|
|
37
|
+
ECOSYSTEM_REGISTRIES = {
|
|
38
|
+
rubygems: "rubygems.org",
|
|
39
|
+
npm: "npmjs.org",
|
|
40
|
+
pypi: "pypi.org",
|
|
41
|
+
cargo: "crates.io",
|
|
42
|
+
}.freeze
|
|
43
|
+
|
|
44
|
+
# A transitive cap only holds the WHOLE tree hostage where the ecosystem
|
|
45
|
+
# forces one version per package. Ruby/Bundler and pip do (flat resolution);
|
|
46
|
+
# npm nests multiple versions and cargo coexists majors, so a below-latest cap
|
|
47
|
+
# there pins a duplicate copy in one subtree -- not a tree-wide block. The one
|
|
48
|
+
# npm case that DOES block (peerDependencies / de-facto singletons) isn't
|
|
49
|
+
# visible in ecosyste.ms data (its parser drops peerDependencies), so emitting
|
|
50
|
+
# poison for npm/cargo would over-claim on the non-blocking cases and miss the
|
|
51
|
+
# blocking ones. Suppress it there until peer requirements are sourced from
|
|
52
|
+
# deps.dev; silence beats a false "holds your tree hostage".
|
|
53
|
+
FLAT_RESOLUTION_ECOSYSTEMS = [:rubygems, :pypi].freeze
|
|
54
|
+
|
|
55
|
+
def assess(ecosystem:, name:, version:, constraint_cache: {}, python_range: nil)
|
|
56
|
+
info = DepsDevClient.version_info(gem_name: name, version: version, system: ecosystem)
|
|
57
|
+
default = DepsDevClient.default_version_info(name: name, system: ecosystem)
|
|
58
|
+
# Retry the version lookup once when it came back empty but the package DID
|
|
59
|
+
# resolve: HttpHelper collapses a genuine 404 and a transient network blip to
|
|
60
|
+
# the same nil, and a healthy version must not be mis-flagged as unresolved
|
|
61
|
+
# (below) on a one-off miss. A real 404 stays nil; a blip recovers the data.
|
|
62
|
+
info ||= DepsDevClient.version_info(gem_name: name, version: version, system: ecosystem) if default
|
|
63
|
+
# Recover the repo from the default version when the exact locked version
|
|
64
|
+
# isn't indexed (yanked/normalization mismatch): otherwise its project link
|
|
65
|
+
# vanishes and a still-fresh package date would read a false :ok with
|
|
66
|
+
# archived/scorecard silently dropped. The native Bundler path resolves the
|
|
67
|
+
# repo independently of deps.dev's per-version record; this gives the lens
|
|
68
|
+
# the same resilience.
|
|
69
|
+
# The pinned version isn't indexed by deps.dev (info nil) while the package IS
|
|
70
|
+
# (default present, so the feed is up): the version is yanked/nonexistent, not a
|
|
71
|
+
# transient miss. Flag it so status reads :unknown rather than letting the still-
|
|
72
|
+
# fresh PACKAGE date report a nonexistent version as :ok.
|
|
73
|
+
version_unresolved = info.nil? && !default.nil?
|
|
74
|
+
project_id = info&.dig(:project_id) || project_id_from(name, ecosystem, default)
|
|
75
|
+
vulnerabilities = vulnerabilities_for(info)
|
|
76
|
+
# Enrich with OSV: a real GHSA severity label (deps.dev can't score a CVSS-4-only
|
|
77
|
+
# advisory) and the fixed-version ranges the "capped below the fix" signal needs.
|
|
78
|
+
OsvClient.enrich(vulnerabilities, ecosystem: ecosystem, name: name)
|
|
79
|
+
scorecard = DepsDevClient.project_scorecard(project_id: project_id)
|
|
80
|
+
repo = repo_signals(project_id)
|
|
81
|
+
|
|
82
|
+
used_release_date = info&.dig(:published_at)
|
|
83
|
+
latest_release_date = default&.dig(:published_at)
|
|
84
|
+
latest_version = default&.dig(:version)
|
|
85
|
+
gem_data = {
|
|
86
|
+
ecosystem: ecosystem,
|
|
87
|
+
name: name,
|
|
88
|
+
version_used: version,
|
|
89
|
+
version_used_release_date: used_release_date,
|
|
90
|
+
latest_version_release_date: latest_release_date,
|
|
91
|
+
# libyear parity with the native path: how far behind latest the locked
|
|
92
|
+
# version is, in release-years. Both dates come from deps.dev responses
|
|
93
|
+
# already fetched above (no extra call); nil when either date is missing.
|
|
94
|
+
libyear: LibyearHelper.gem_libyear(
|
|
95
|
+
version_used_release_date: parse_time(used_release_date),
|
|
96
|
+
latest_version_release_date: parse_time(latest_release_date),
|
|
97
|
+
),
|
|
98
|
+
# Whether the pinned version is at or ahead of deps.dev's latest stable, so a
|
|
99
|
+
# prerelease or an ahead-of-stable pin reads current (true), not "behind" --
|
|
100
|
+
# parity with the native path's `>=` comparison. nil when latest is unknown.
|
|
101
|
+
up_to_date: version_current?(version, latest_version),
|
|
102
|
+
repository_url: project_id && "https://#{project_id}",
|
|
103
|
+
last_commit_date: repo[:last_commit_date],
|
|
104
|
+
archived: repo[:archived],
|
|
105
|
+
scorecard_score: scorecard&.dig(:score),
|
|
106
|
+
scorecard_maintained: scorecard&.dig(:maintained),
|
|
107
|
+
vulnerability_count: vulnerabilities.length,
|
|
108
|
+
vulnerabilities: vulnerabilities,
|
|
109
|
+
}
|
|
110
|
+
gem_data[:version_unresolved] = true if version_unresolved
|
|
111
|
+
attach_constraints(gem_data, ecosystem: ecosystem, name: name, version: version, cache: constraint_cache)
|
|
112
|
+
attach_language_ceiling(gem_data, ecosystem: ecosystem, name: name, version: version, latest_version: default&.dig(:version), python_range: python_range)
|
|
113
|
+
gem_data
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
private
|
|
117
|
+
|
|
118
|
+
# Language-runtime ceiling for the cross-ecosystem path, the sibling of the
|
|
119
|
+
# native Ruby ceiling in Workflow. Python declares its runtime constraint as a
|
|
120
|
+
# PEP 440 `requires_python`, an enforced pip install wall; translate it and run
|
|
121
|
+
# the same generic RuntimeCeilingHelper against Python's support window. Unlike
|
|
122
|
+
# poison this is NOT gated on dormancy (a cap is a fact of the resolved version
|
|
123
|
+
# whether or not the package is maintained), so it costs one PyPI read per
|
|
124
|
+
# pypi package; other ecosystems carry no ceiling here rather than a wrong one
|
|
125
|
+
# (cargo's rust_version is a soft MSRV hint, not an install wall). Best-effort:
|
|
126
|
+
# a nil range (endoflife feed down) or absent requires_python yields nothing.
|
|
127
|
+
def attach_language_ceiling(gem_data, ecosystem:, name:, version:, latest_version:, python_range:)
|
|
128
|
+
return if python_range.nil?
|
|
129
|
+
return unless ecosystem == :pypi
|
|
130
|
+
|
|
131
|
+
requirement = Pep440Helper.to_gem_requirement_string(PypiClient.requires_python(name: name, version: version))
|
|
132
|
+
finding = requirement && RuntimeCeilingHelper.analyze(requirement: requirement, support_window: python_range)
|
|
133
|
+
return if finding.nil?
|
|
134
|
+
|
|
135
|
+
finding[:runtime] = "Python"
|
|
136
|
+
finding[:fixed_by_upgrade] = python_ceiling_lifted_by_upgrade?(name: name, version: version, latest_version: latest_version, python_range: python_range)
|
|
137
|
+
gem_data[:language_ceiling] = finding
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Does upgrading the package to its latest release lift the cap? Requires
|
|
141
|
+
# POSITIVE confirmation, unlike the native ruby_ceiling_lifted_by_upgrade?
|
|
142
|
+
# whose latest requirement is read from data already in hand. Here the latest
|
|
143
|
+
# requires_python is a fresh PyPI read, and PypiClient returns nil for BOTH
|
|
144
|
+
# "declares no constraint" AND "the fetch failed" -- indistinguishable. Reading
|
|
145
|
+
# that nil as "no ceiling, safe to bump" would fabricate remediation on a
|
|
146
|
+
# transient PyPI error, the exact over-claim this tool must never make. So a
|
|
147
|
+
# nil specifier yields false (we don't advise an upgrade we couldn't verify);
|
|
148
|
+
# only a readable constraint that analyze confirms is non-capping lifts it.
|
|
149
|
+
def python_ceiling_lifted_by_upgrade?(name:, version:, latest_version:, python_range:)
|
|
150
|
+
return false if latest_version.nil? || latest_version == version
|
|
151
|
+
|
|
152
|
+
latest_specifier = PypiClient.requires_python(name: name, version: latest_version)
|
|
153
|
+
return false if latest_specifier.nil?
|
|
154
|
+
|
|
155
|
+
latest_requirement = Pep440Helper.to_gem_requirement_string(latest_specifier)
|
|
156
|
+
RuntimeCeilingHelper.analyze(requirement: latest_requirement, support_window: python_range).nil?
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
# Constraint enrichment for the cross-ecosystem path, gated on dormancy so a
|
|
160
|
+
# maintained package is never flagged. Two shapes by ecosystem:
|
|
161
|
+
#
|
|
162
|
+
# FLAT (rubygems/pypi): the poison-pill signal as-is -- a below-latest cap holds
|
|
163
|
+
# the whole tree hostage, so it renders directly (`constraints` + `poison`).
|
|
164
|
+
#
|
|
165
|
+
# NESTED (npm/cargo): the pure below-latest cap is subtree-local noise (nested
|
|
166
|
+
# copies, caret default), so no poison here. Instead keep every declared dep as a
|
|
167
|
+
# security CANDIDATE (`capped_deps`); the correlator, which alone sees the tree's
|
|
168
|
+
# resolved versions and advisories, promotes only the ones that pin a vulnerable
|
|
169
|
+
# copy below its fix. Candidates are NOT filtered to below-latest-major (npm/cargo
|
|
170
|
+
# fixes are mostly same-major patch bumps that filter would miss) and need no
|
|
171
|
+
# dep_latest fetch (the wall test is patch-precise on the requirement itself).
|
|
172
|
+
def attach_constraints(gem_data, ecosystem:, name:, version:, cache:)
|
|
173
|
+
registry = ECOSYSTEM_REGISTRIES[ecosystem]
|
|
174
|
+
return if registry.nil?
|
|
175
|
+
return unless [:critical, :archived].include?(ActivityHelper.activity_level(gem_data))
|
|
176
|
+
|
|
177
|
+
declared = EcosystemsClient.declared_dependencies(name: name, version: version, registry: registry)
|
|
178
|
+
if FLAT_RESOLUTION_ECOSYSTEMS.include?(ecosystem)
|
|
179
|
+
attach_flat_poison(gem_data, ecosystem, declared, cache)
|
|
180
|
+
else
|
|
181
|
+
attach_nested_candidates(gem_data, declared)
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def attach_flat_poison(gem_data, ecosystem, declared, cache)
|
|
186
|
+
findings = ConstraintHelper.poison_findings(declared) do |dep_name|
|
|
187
|
+
latest_version_for(ecosystem, dep_name, cache)
|
|
188
|
+
end
|
|
189
|
+
return if findings.empty?
|
|
190
|
+
|
|
191
|
+
gem_data[:constraints] = findings
|
|
192
|
+
gem_data[:poison] = findings.any? { |finding| finding[:kind] == :ceiling }
|
|
193
|
+
gem_data[:poison_severity] = ConstraintHelper.worst_severity(findings) if gem_data[:poison]
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def attach_nested_candidates(gem_data, declared)
|
|
197
|
+
candidates = declared.filter_map do |dep|
|
|
198
|
+
next if dep[:package_name].to_s.empty? || dep[:requirements].to_s.empty?
|
|
199
|
+
|
|
200
|
+
{ dependency: dep[:package_name], requirement: dep[:requirements] }
|
|
201
|
+
end
|
|
202
|
+
gem_data[:capped_deps] = candidates unless candidates.empty?
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# A capped dep's current latest version in `ecosystem`, via deps.dev's default
|
|
206
|
+
# (latest stable) version. Memoized per run, keyed by ecosystem+name so an npm
|
|
207
|
+
# `foo` and a pypi `foo` never collide. Only a RESOLVED version is cached: a
|
|
208
|
+
# nil can mean "genuinely unindexed" or "deps.dev was momentarily down / rate-
|
|
209
|
+
# limited" (both surface as nil here), and caching the transient case would
|
|
210
|
+
# suppress the pill for every later dormant package capping the same dep. So an
|
|
211
|
+
# unresolved dep is re-attempted rather than remembered as a permanent miss.
|
|
212
|
+
def latest_version_for(ecosystem, dep_name, cache)
|
|
213
|
+
key = "#{ecosystem}/#{dep_name}"
|
|
214
|
+
cache[key] ||= DepsDevClient.default_version_info(name: dep_name, system: ecosystem)&.dig(:version)
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def vulnerabilities_for(info)
|
|
218
|
+
advisory_keys = info&.dig(:advisory_keys) || []
|
|
219
|
+
# The keys ARE the evidence the locked version is vulnerable; the detail
|
|
220
|
+
# fetch only enriches CVSS/title. A failed enrichment must not drop the
|
|
221
|
+
# count to zero and read a known-vulnerable dep as clean, so a key whose
|
|
222
|
+
# detail can't be loaded still contributes a minimal advisory.
|
|
223
|
+
deps_dev = advisory_keys.map { DepsDevClient.advisory_detail(advisory_id: _1) || { id: _1, source: "deps.dev" } }
|
|
224
|
+
# No ruby-advisory-db here: that database is Ruby-only and the native
|
|
225
|
+
# Bundler path already merges it. Cross-ecosystem advisories come from
|
|
226
|
+
# deps.dev/OSV, which covers every system the lens serves.
|
|
227
|
+
VulnerabilityHelper.merge_advisories(deps_dev: deps_dev, ruby_advisory_db: [])
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# The repo project_id from the package's default version, used only when the
|
|
231
|
+
# locked version's deps.dev record is missing or carries no link.
|
|
232
|
+
def project_id_from(name, ecosystem, default)
|
|
233
|
+
version = default&.dig(:version)
|
|
234
|
+
return if version.nil?
|
|
235
|
+
|
|
236
|
+
DepsDevClient.version_info(gem_name: name, version: version, system: ecosystem)&.dig(:project_id)
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
# archived + last-commit for a flat github.com/owner/repo project, or {} for
|
|
240
|
+
# anything else. deps.dev indexes github.com and gitlab.com only, but gitlab
|
|
241
|
+
# subgroups nest arbitrarily and ecosyste.ms's repo crawler is GitHub-centric,
|
|
242
|
+
# so a gitlab (or nested, or unresolved) project keeps archived unknown rather
|
|
243
|
+
# than risk a bogus owner/name lookup.
|
|
244
|
+
def repo_signals(project_id)
|
|
245
|
+
host, owner, name, *rest = project_id.to_s.split("/")
|
|
246
|
+
return {} unless host == "github.com" && owner && name && rest.empty?
|
|
247
|
+
|
|
248
|
+
repo_provider.repo_signals(owner: owner, name: name) || {}
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
# Mirrors Workflow#provider_for(:github): the live GitHub API when a token is
|
|
252
|
+
# configured (freshest), else ecosyste.ms (5000 anonymous req/hr vs GitHub's
|
|
253
|
+
# 60), so an untokened cross-ecosystem run still resolves a large SBOM.
|
|
254
|
+
def repo_provider
|
|
255
|
+
StillActive.config.github_oauth_token ? GithubClient : EcosystemsClient
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# Is the pinned version at or ahead of latest stable? nil when latest is unknown.
|
|
259
|
+
# Gem::Version handles semver prereleases (`16.3.0-canary.7` sorts as a prerelease
|
|
260
|
+
# of 16.3.0, so ahead of a 16.2.x stable -> current), so a beta/ahead pin reads
|
|
261
|
+
# true, not "behind". Versions Gem::Version can't parse (pypi epoch `1!2.3`, build
|
|
262
|
+
# metadata, a `v` prefix) fall back to exact match -- ahead reads false there, the
|
|
263
|
+
# safe direction, and libyear still carries the real magnitude.
|
|
264
|
+
def version_current?(version, latest)
|
|
265
|
+
return false if latest.nil?
|
|
266
|
+
return version == latest unless Gem::Version.correct?(version) && Gem::Version.correct?(latest)
|
|
267
|
+
|
|
268
|
+
Gem::Version.new(version) >= Gem::Version.new(latest)
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
# deps.dev renders dates as ISO8601 strings; libyear needs Time to subtract.
|
|
272
|
+
# nil-safe, and a malformed value degrades to nil rather than raising.
|
|
273
|
+
def parse_time(value)
|
|
274
|
+
# Guard the type, not just nil: a non-String publishedAt (schema drift) would
|
|
275
|
+
# make Time.parse raise TypeError (unrescued) and, since this enrichment runs
|
|
276
|
+
# in assess, demote an otherwise-healthy package. Best-effort must degrade.
|
|
277
|
+
return unless value.is_a?(String)
|
|
278
|
+
|
|
279
|
+
Time.parse(value)
|
|
280
|
+
rescue ArgumentError
|
|
281
|
+
nil
|
|
282
|
+
end
|
|
283
|
+
end
|
|
284
|
+
end
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
require_relative "../helpers/http_helper"
|
|
5
|
+
require_relative "version"
|
|
6
|
+
|
|
7
|
+
module StillActive
|
|
8
|
+
# Tokenless repo signals (archived?, last-commit date) for github.com-hosted
|
|
9
|
+
# gems, sourced from ecosyste.ms instead of the GitHub API. Used as the
|
|
10
|
+
# fallback when no GitHub token is configured, so an unauthenticated run isn't
|
|
11
|
+
# capped at GitHub's 60 req/hr (ecosyste.ms allows 5000 anonymous) -- the
|
|
12
|
+
# difference between "works on a large Gemfile" and "dies after a handful".
|
|
13
|
+
#
|
|
14
|
+
# GitHub-only by design: ecosyste.ms's repos service doesn't populate commit
|
|
15
|
+
# recency for GitLab/Codeberg (its repo crawler is GitHub-centric), so those
|
|
16
|
+
# hosts keep their own live clients.
|
|
17
|
+
#
|
|
18
|
+
# Data is CC-BY-SA 4.0 (https://creativecommons.org/licenses/by-sa/4.0/);
|
|
19
|
+
# still_active queries it live (no redistribution) and attributes ecosyste.ms
|
|
20
|
+
# in the README data sources.
|
|
21
|
+
module EcosystemsClient
|
|
22
|
+
extend self
|
|
23
|
+
|
|
24
|
+
BASE_URI = URI("https://repos.ecosyste.ms/")
|
|
25
|
+
# The packages service is a distinct host from the repos service above; it
|
|
26
|
+
# carries per-version declared dependency constraints (the poison-pill input).
|
|
27
|
+
PACKAGES_BASE_URI = URI("https://packages.ecosyste.ms/")
|
|
28
|
+
# ecosyste.ms asks consumers to identify themselves for its "polite pool".
|
|
29
|
+
USER_AGENT = "still_active/#{StillActive::VERSION} (+https://github.com/SeanLF/still_active)"
|
|
30
|
+
|
|
31
|
+
# archived + last-commit date from a single repository call. ecosyste.ms's
|
|
32
|
+
# pushed_at mirrors GitHub's, so this returns the same shape as GithubClient.
|
|
33
|
+
# Returns {} when the repo can't be read, so the caller leaves both blank.
|
|
34
|
+
def repo_signals(owner:, name:)
|
|
35
|
+
return {} if owner.nil? || name.nil?
|
|
36
|
+
|
|
37
|
+
path = "/api/v1/hosts/GitHub/repositories/#{encode_repo(owner, name)}"
|
|
38
|
+
body = HttpHelper.get_json(BASE_URI, path, headers: { "User-Agent" => USER_AGENT }, params: politeness_params)
|
|
39
|
+
# A non-Hash 200 body (error envelope rendered as an array, schema drift)
|
|
40
|
+
# would otherwise raise on indexing and vanish the gem from the audit via
|
|
41
|
+
# the workflow's rescue; degrade to "no signal" like any other read failure.
|
|
42
|
+
return {} unless body.is_a?(Hash)
|
|
43
|
+
|
|
44
|
+
signals = { last_commit_date: parse_time(body["pushed_at"], owner, name) }
|
|
45
|
+
# Only assert archived when the field is actually present. A missing field
|
|
46
|
+
# must read as unknown, not be invented as false -- otherwise a partial
|
|
47
|
+
# crawl could silently mask the most actionable verdict (gem is archived).
|
|
48
|
+
signals[:archived] = body["archived"] == true if body.key?("archived")
|
|
49
|
+
signals
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# ecosyste.ms dependency-kind labels that ship at RUNTIME (so a cap on one
|
|
53
|
+
# holds the consumer's tree hostage). It is registry-specific vocabulary:
|
|
54
|
+
# rubygems/npm/pypi say "runtime", but cargo says "normal" (its "dev"/"build"
|
|
55
|
+
# kinds are excluded). An ALLOWLIST, not a denylist: an unrecognised kind is
|
|
56
|
+
# dropped rather than risk flagging a dev/build dep and breaking the FP
|
|
57
|
+
# discipline. Compared case-insensitively (rubygems's dev kind is
|
|
58
|
+
# "Development").
|
|
59
|
+
RUNTIME_KINDS = ["runtime", "normal"].freeze
|
|
60
|
+
|
|
61
|
+
# The runtime dependency constraints a package version declares: an array of
|
|
62
|
+
# { package_name:, requirements: } for runtime-shipped deps only (see
|
|
63
|
+
# RUNTIME_KINDS). Dev/build/test deps are dropped -- they don't cap the
|
|
64
|
+
# consumer's tree, so they can't be a poison-pill. `requirements` is the raw
|
|
65
|
+
# constraint string ("< 5.0, >= 4.0.1") ConstraintHelper reads. `registry` is
|
|
66
|
+
# the ecosyste.ms registry name (rubygems.org, pypi.org, npmjs.org,
|
|
67
|
+
# crates.io), defaulting to Ruby.
|
|
68
|
+
#
|
|
69
|
+
# Returns [] whenever the version can't be read (unindexed, 404, timeout,
|
|
70
|
+
# schema drift), so a caller degrades to "no constraints known" rather than
|
|
71
|
+
# crashing the per-gem audit.
|
|
72
|
+
def declared_dependencies(name:, version:, registry: "rubygems.org")
|
|
73
|
+
return [] if name.nil? || version.nil?
|
|
74
|
+
|
|
75
|
+
path = "/api/v1/registries/#{encode(registry)}/packages/#{encode(name)}/versions/#{encode(version)}"
|
|
76
|
+
body = HttpHelper.get_json(PACKAGES_BASE_URI, path, headers: { "User-Agent" => USER_AGENT }, params: politeness_params)
|
|
77
|
+
return [] unless body.is_a?(Hash)
|
|
78
|
+
|
|
79
|
+
dependencies = body["dependencies"]
|
|
80
|
+
return [] unless dependencies.is_a?(Array)
|
|
81
|
+
|
|
82
|
+
dependencies.filter_map do |dep|
|
|
83
|
+
next unless dep.is_a?(Hash) && RUNTIME_KINDS.include?(dep["kind"].to_s.downcase)
|
|
84
|
+
|
|
85
|
+
package_name = dep["package_name"]
|
|
86
|
+
requirements = dep["requirements"]
|
|
87
|
+
next if package_name.nil? || requirements.nil?
|
|
88
|
+
|
|
89
|
+
{ package_name: package_name, requirements: requirements }
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
private
|
|
94
|
+
|
|
95
|
+
# ecosyste.ms raises the rate limit and prioritises requests that identify a
|
|
96
|
+
# contact via a mailto query param (its "polite pool"). Opt-in: empty unless
|
|
97
|
+
# the user configures an email, since anonymous already suffices for a typical
|
|
98
|
+
# lockfile and we don't attribute every user's traffic to one address.
|
|
99
|
+
def politeness_params
|
|
100
|
+
email = StillActive.config.ecosystems_email
|
|
101
|
+
email ? { mailto: email } : {}
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# nil or a non-string (numeric/array from an off-spec payload) -> no date,
|
|
105
|
+
# never a crash; only an unparseable string is worth warning about.
|
|
106
|
+
def parse_time(value, owner, name)
|
|
107
|
+
return unless value.is_a?(String)
|
|
108
|
+
|
|
109
|
+
Time.parse(value)
|
|
110
|
+
rescue ArgumentError
|
|
111
|
+
$stderr.puts("warning: could not parse repo date for #{owner}/#{name}: #{value.inspect}")
|
|
112
|
+
nil
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def encode_repo(owner, name)
|
|
116
|
+
URI.encode_www_form_component("#{owner}/#{name}")
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def encode(value)
|
|
120
|
+
URI.encode_www_form_component(value.to_s)
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
require_relative "../helpers/http_helper"
|
|
5
|
+
|
|
6
|
+
module StillActive
|
|
7
|
+
# Repo signals (archived?, last commit date) for Forgejo/Gitea-hosted gems.
|
|
8
|
+
# Codeberg.org is the only host wired into the workflow today, but every
|
|
9
|
+
# Forgejo/Gitea instance speaks the same `/api/v1` surface, so the host is a
|
|
10
|
+
# parameter for later self-hosted support. Mirrors GitlabClient: HttpHelper
|
|
11
|
+
# against a documented JSON API, anonymous by default with an optional token.
|
|
12
|
+
module ForgejoClient
|
|
13
|
+
extend self
|
|
14
|
+
|
|
15
|
+
DEFAULT_HOST = "codeberg.org"
|
|
16
|
+
|
|
17
|
+
# archived + last-activity date from a single repository call. The repo
|
|
18
|
+
# object's updated_at matches the latest commit date to the day in practice,
|
|
19
|
+
# so folding the two signals into one call halves the per-gem requests.
|
|
20
|
+
# Returns {} when the repo can't be read.
|
|
21
|
+
def repo_signals(owner:, name:, host: DEFAULT_HOST)
|
|
22
|
+
return {} if owner.nil? || name.nil?
|
|
23
|
+
|
|
24
|
+
body = HttpHelper.get_json(base_uri(host), "/api/v1/repos/#{owner}/#{name}", headers: auth_headers)
|
|
25
|
+
return {} if body.nil?
|
|
26
|
+
|
|
27
|
+
{ archived: body["archived"] == true, last_commit_date: parse_time(body["updated_at"], owner, name) }
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def parse_time(value, owner, name)
|
|
33
|
+
return if value.nil?
|
|
34
|
+
|
|
35
|
+
Time.parse(value)
|
|
36
|
+
rescue ArgumentError
|
|
37
|
+
$stderr.puts("warning: could not parse repo date for #{owner}/#{name}: #{value.inspect}")
|
|
38
|
+
nil
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def base_uri(host)
|
|
42
|
+
URI("https://#{host}/")
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def auth_headers
|
|
46
|
+
token = StillActive.config.forgejo_token
|
|
47
|
+
token ? { "Authorization" => "token #{token}" } : {}
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "time"
|
|
4
|
+
require "octokit"
|
|
5
|
+
|
|
6
|
+
module StillActive
|
|
7
|
+
# Repo signals (archived?, last commit date) for github.com-hosted gems.
|
|
8
|
+
# Wraps Octokit so the rest of the workflow dispatches to a provider with the
|
|
9
|
+
# same shape as GitlabClient, rather than reaching into Octokit inline. The
|
|
10
|
+
# Octokit dependency stays internal to this module.
|
|
11
|
+
module GithubClient
|
|
12
|
+
extend self
|
|
13
|
+
|
|
14
|
+
# A rate-limit response whose reset is at most this many seconds away is
|
|
15
|
+
# waited out and retried; a longer wait (hourly-limit exhaustion) is not
|
|
16
|
+
# auto-taken and falls through to the caller's rescue (warn + nil).
|
|
17
|
+
MAX_RATE_LIMIT_WAIT = 60
|
|
18
|
+
|
|
19
|
+
# archived + last-activity date from a single repository call. The repo
|
|
20
|
+
# object's pushed_at (last push) stands in for the last-commit date: it
|
|
21
|
+
# matches the default-branch commit date to the day in practice, and folding
|
|
22
|
+
# the two signals into one call halves the per-gem GitHub requests. Returns
|
|
23
|
+
# {} when the repo can't be read, so the caller leaves both signals blank.
|
|
24
|
+
def repo_signals(owner:, name:)
|
|
25
|
+
return {} if owner.nil? || name.nil?
|
|
26
|
+
|
|
27
|
+
repo = with_rate_limit_retry("repo #{owner}/#{name}") do
|
|
28
|
+
StillActive.config.github_client.repository("#{owner}/#{name}")
|
|
29
|
+
end
|
|
30
|
+
return {} unless repo
|
|
31
|
+
|
|
32
|
+
{ archived: repo.archived, last_commit_date: as_time(repo.pushed_at, owner, name) }
|
|
33
|
+
rescue Octokit::Error, Faraday::Error => e
|
|
34
|
+
$stderr.puts("warning: repo signals failed for #{owner}/#{name}: #{e.class}")
|
|
35
|
+
{}
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Commits on the default branch since the latest release's tag: the
|
|
39
|
+
# "unreleased work" signal. GitHub's compare endpoint returns ahead_by as a
|
|
40
|
+
# single scalar, so this is one cheap call. The git tag name isn't carried
|
|
41
|
+
# by RubyGems, so resolve it from the version by trying the two ubiquitous
|
|
42
|
+
# forms (v7.0.1, 7.0.1) as the compare base; a wrong form 404s and we try
|
|
43
|
+
# the next. Returns nil when neither resolves (non-tagging repo, monorepo
|
|
44
|
+
# tag scheme we don't guess). Only GithubClient implements this; the
|
|
45
|
+
# workflow dispatches by respond_to?, so GitLab/Forgejo simply don't.
|
|
46
|
+
def commits_since_release(owner:, name:, version:)
|
|
47
|
+
return if owner.nil? || name.nil? || version.nil?
|
|
48
|
+
|
|
49
|
+
repo = "#{owner}/#{name}"
|
|
50
|
+
["v#{version}", version.to_s].each do |tag|
|
|
51
|
+
return with_rate_limit_retry("unreleased-commits #{repo}") { StillActive.config.github_client.compare(repo, tag, "HEAD").ahead_by }
|
|
52
|
+
rescue Octokit::NotFound
|
|
53
|
+
# this tag form doesn't exist; fall through to try the next one
|
|
54
|
+
end
|
|
55
|
+
nil
|
|
56
|
+
rescue Octokit::Error, Faraday::Error => e
|
|
57
|
+
$stderr.puts("warning: unreleased-commits check failed for #{owner}/#{name}: #{e.class}")
|
|
58
|
+
nil
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
private
|
|
62
|
+
|
|
63
|
+
# Pause-and-retry on a rate-limit response when the reset is near, so a
|
|
64
|
+
# transient secondary/burst limit (which GitHub's concurrent fan-out can
|
|
65
|
+
# trip even with a token) self-heals instead of dropping the gem's signal.
|
|
66
|
+
# Retries at most once. Under the async reactor, sleep yields to other
|
|
67
|
+
# fibers rather than blocking the thread.
|
|
68
|
+
def with_rate_limit_retry(label)
|
|
69
|
+
retried = false
|
|
70
|
+
begin
|
|
71
|
+
yield
|
|
72
|
+
rescue Octokit::TooManyRequests => e
|
|
73
|
+
wait = rate_limit_wait(e)
|
|
74
|
+
if retried || wait.nil? || wait > MAX_RATE_LIMIT_WAIT
|
|
75
|
+
# Hourly-limit exhaustion (or a far reset): not worth auto-waiting.
|
|
76
|
+
# Surface the one actionable hint rather than a generic class name,
|
|
77
|
+
# then return nil so this signal is simply absent for the gem.
|
|
78
|
+
$stderr.puts("rate limited on #{label}; set GITHUB_TOKEN to raise your limit, or run less often")
|
|
79
|
+
return
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
retried = true
|
|
83
|
+
$stderr.puts("rate limited on #{label}; waiting #{wait}s for reset")
|
|
84
|
+
sleep(wait)
|
|
85
|
+
retry
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
# Seconds to wait before retrying, from the Retry-After header (secondary
|
|
90
|
+
# limits) or x-ratelimit-reset (primary), or nil when neither is present.
|
|
91
|
+
def rate_limit_wait(error)
|
|
92
|
+
# Octokit normalises response headers to lowercase (Faraday is
|
|
93
|
+
# case-insensitive), so a single lowercase lookup covers any casing.
|
|
94
|
+
headers = response_headers(error)
|
|
95
|
+
return if headers.nil? || headers.empty?
|
|
96
|
+
|
|
97
|
+
if (retry_after = headers["retry-after"])
|
|
98
|
+
return retry_after.to_i
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
reset = headers["x-ratelimit-reset"]
|
|
102
|
+
return if reset.nil?
|
|
103
|
+
|
|
104
|
+
[reset.to_i - Time.now.to_i, 0].max
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Octokit raises NoMethodError reading headers off an error with no response
|
|
108
|
+
# attached; treat only that as "can't tell" so the limiter degrades to no
|
|
109
|
+
# wait, while a real NoMethodError elsewhere still surfaces.
|
|
110
|
+
def response_headers(error)
|
|
111
|
+
error.response_headers
|
|
112
|
+
rescue NoMethodError
|
|
113
|
+
nil
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def as_time(value, owner, name)
|
|
117
|
+
return value if value.is_a?(Time)
|
|
118
|
+
return if value.nil?
|
|
119
|
+
|
|
120
|
+
Time.parse(value)
|
|
121
|
+
rescue ArgumentError
|
|
122
|
+
$stderr.puts("warning: could not parse repo date for #{owner}/#{name}: #{value.inspect}")
|
|
123
|
+
nil
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
@@ -9,36 +9,31 @@ module StillActive
|
|
|
9
9
|
|
|
10
10
|
BASE_URI = URI("https://gitlab.com/")
|
|
11
11
|
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
# archived + last-activity date from a single project call. The project
|
|
13
|
+
# object's last_activity_at matches the latest commit date to the day in
|
|
14
|
+
# practice, so folding the two signals into one call halves the per-gem
|
|
15
|
+
# requests. Returns {} when the project can't be read.
|
|
16
|
+
def repo_signals(owner:, name:)
|
|
17
|
+
return {} if owner.nil? || name.nil?
|
|
14
18
|
|
|
15
19
|
path = "/api/v4/projects/#{encode_project(owner, name)}"
|
|
16
20
|
body = HttpHelper.get_json(BASE_URI, path, headers: auth_headers)
|
|
17
|
-
return if body.nil?
|
|
21
|
+
return {} if body.nil?
|
|
18
22
|
|
|
19
|
-
body["archived"] == true
|
|
23
|
+
{ archived: body["archived"] == true, last_commit_date: parse_time(body["last_activity_at"], owner, name) }
|
|
20
24
|
end
|
|
21
25
|
|
|
22
|
-
|
|
23
|
-
return if owner.nil? || name.nil?
|
|
24
|
-
|
|
25
|
-
path = "/api/v4/projects/#{encode_project(owner, name)}/repository/commits"
|
|
26
|
-
body = HttpHelper.get_json(BASE_URI, path, headers: auth_headers, params: { per_page: 1 })
|
|
27
|
-
return if body.nil? || body.empty?
|
|
26
|
+
private
|
|
28
27
|
|
|
29
|
-
|
|
30
|
-
return
|
|
28
|
+
def parse_time(value, owner, name)
|
|
29
|
+
return if value.nil?
|
|
31
30
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
nil
|
|
37
|
-
end
|
|
31
|
+
Time.parse(value)
|
|
32
|
+
rescue ArgumentError
|
|
33
|
+
$stderr.puts("warning: could not parse repo date for #{owner}/#{name}: #{value.inspect}")
|
|
34
|
+
nil
|
|
38
35
|
end
|
|
39
36
|
|
|
40
|
-
private
|
|
41
|
-
|
|
42
37
|
def auth_headers
|
|
43
38
|
token = StillActive.config.gitlab_token
|
|
44
39
|
token ? { "PRIVATE-TOKEN" => token } : {}
|