@aldegad/safedeps 2.1.1 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +273 -463
- package/README.ko.md +76 -12
- package/README.md +107 -38
- package/ROADMAP.md +123 -84
- package/SECURITY.md +45 -0
- package/SKILL.md +86 -143
- package/bin/safedeps +419 -52
- package/lib/gates/audit.sh +36 -0
- package/lib/gates/doctor.sh +212 -0
- package/lib/gates/hooks.sh +131 -0
- package/lib/gates/repo-profile.sh +60 -0
- package/lib/gates/scan.sh +94 -0
- package/lib/gates/templates/gitleaks.private.toml.tmpl +45 -0
- package/lib/gates/templates/gitleaks.toml.tmpl +43 -0
- package/lib/gates/templates/pre-commit.tmpl +49 -0
- package/lib/ledger/ledger.sh +94 -16
- package/lib/npm/closure.sh +115 -0
- package/lib/providers/providers.sh +248 -26
- package/package.json +2 -1
- package/scripts/install/install-safedeps-hooks.mjs +65 -23
- package/scripts/release-gates.sh +252 -0
- package/scripts/safedeps-post-verify.sh +185 -15
- package/scripts/safedeps-pre-guard.sh +309 -39
- package/scripts/test/e2e.sh +228 -4
- package/scripts/test/fixture-provider.mjs +21 -0
- package/scripts/test/smoke.sh +212 -10
package/lib/ledger/ledger.sh
CHANGED
|
@@ -183,16 +183,23 @@ safedeps_ledger_check() {
|
|
|
183
183
|
|
|
184
184
|
safedeps_ledger_atomic_write() {
|
|
185
185
|
local target_path="$1"
|
|
186
|
-
local
|
|
186
|
+
local target_dir
|
|
187
|
+
local target_base
|
|
188
|
+
local temp_path
|
|
187
189
|
|
|
188
190
|
safedeps_ledger_init
|
|
191
|
+
target_dir=$(dirname "${target_path}")
|
|
192
|
+
target_base=$(basename "${target_path}")
|
|
193
|
+
mkdir -p "${target_dir}" || return 1
|
|
194
|
+
temp_path=$(mktemp "${target_dir}/.${target_base}.XXXXXX") || return 1
|
|
195
|
+
|
|
189
196
|
cat > "${temp_path}"
|
|
190
197
|
chmod 600 "${temp_path}" 2>/dev/null || true
|
|
191
198
|
safedeps_ledger_validate_json "${temp_path}" || {
|
|
192
199
|
rm -f "${temp_path}"
|
|
193
200
|
return 1
|
|
194
201
|
}
|
|
195
|
-
mv "${temp_path}" "${target_path}"
|
|
202
|
+
mv -f "${temp_path}" "${target_path}"
|
|
196
203
|
}
|
|
197
204
|
|
|
198
205
|
safedeps_ledger_write_approved_spec() {
|
|
@@ -203,11 +210,13 @@ safedeps_ledger_write_approved_spec() {
|
|
|
203
210
|
local approved_by="${5:-local}"
|
|
204
211
|
local evidence_file="${6:-}"
|
|
205
212
|
local ttl_days="${7:-${SAFEDEPS_LEDGER_DEFAULT_TTL_DAYS}}"
|
|
213
|
+
local transitive_specs_file="${8:-}"
|
|
206
214
|
local approved_at
|
|
207
215
|
local expires_at
|
|
208
216
|
local hash
|
|
209
217
|
local target_path
|
|
210
218
|
local evidence_arg=()
|
|
219
|
+
local transitive_arg=()
|
|
211
220
|
|
|
212
221
|
safedeps_ledger_require_jq || return 1
|
|
213
222
|
safedeps_ledger_init
|
|
@@ -227,6 +236,20 @@ safedeps_ledger_write_approved_spec() {
|
|
|
227
236
|
evidence_arg=(--argjson evidence '{}')
|
|
228
237
|
fi
|
|
229
238
|
|
|
239
|
+
if [[ -n "${transitive_specs_file}" ]]; then
|
|
240
|
+
[[ -f "${transitive_specs_file}" ]] || {
|
|
241
|
+
printf 'safedeps ledger: transitive specs file not found: %s\n' "${transitive_specs_file}" >&2
|
|
242
|
+
return 1
|
|
243
|
+
}
|
|
244
|
+
jq -e 'type == "array"' "${transitive_specs_file}" >/dev/null || {
|
|
245
|
+
printf 'safedeps ledger: transitive specs file must be a JSON array: %s\n' "${transitive_specs_file}" >&2
|
|
246
|
+
return 1
|
|
247
|
+
}
|
|
248
|
+
transitive_arg=(--slurpfile transitive_specs "${transitive_specs_file}")
|
|
249
|
+
else
|
|
250
|
+
transitive_arg=(--argjson transitive_specs '[]')
|
|
251
|
+
fi
|
|
252
|
+
|
|
230
253
|
if [[ -n "${evidence_file}" ]]; then
|
|
231
254
|
jq -cn \
|
|
232
255
|
--arg hash "${hash}" \
|
|
@@ -238,6 +261,7 @@ safedeps_ledger_write_approved_spec() {
|
|
|
238
261
|
--arg expires_at "${expires_at}" \
|
|
239
262
|
--arg approved_by "${approved_by}" \
|
|
240
263
|
"${evidence_arg[@]}" \
|
|
264
|
+
"${transitive_arg[@]}" \
|
|
241
265
|
'{
|
|
242
266
|
hash: $hash,
|
|
243
267
|
ecosystem: $ecosystem,
|
|
@@ -248,7 +272,11 @@ safedeps_ledger_write_approved_spec() {
|
|
|
248
272
|
expires_at: $expires_at,
|
|
249
273
|
approved_by: $approved_by,
|
|
250
274
|
evidence: ($evidence[0] // {}),
|
|
251
|
-
transitive_specs: []
|
|
275
|
+
transitive_specs: (($transitive_specs[0] // $transitive_specs) | map({
|
|
276
|
+
ecosystem: (.ecosystem // $ecosystem),
|
|
277
|
+
package: .package,
|
|
278
|
+
version: (.version | tostring)
|
|
279
|
+
}) | unique_by(.ecosystem + "\u0000" + .package + "\u0000" + .version))
|
|
252
280
|
}' | safedeps_ledger_atomic_write "${target_path}"
|
|
253
281
|
else
|
|
254
282
|
jq -cn \
|
|
@@ -261,6 +289,7 @@ safedeps_ledger_write_approved_spec() {
|
|
|
261
289
|
--arg expires_at "${expires_at}" \
|
|
262
290
|
--arg approved_by "${approved_by}" \
|
|
263
291
|
"${evidence_arg[@]}" \
|
|
292
|
+
"${transitive_arg[@]}" \
|
|
264
293
|
'{
|
|
265
294
|
hash: $hash,
|
|
266
295
|
ecosystem: $ecosystem,
|
|
@@ -271,39 +300,84 @@ safedeps_ledger_write_approved_spec() {
|
|
|
271
300
|
expires_at: $expires_at,
|
|
272
301
|
approved_by: $approved_by,
|
|
273
302
|
evidence: $evidence,
|
|
274
|
-
transitive_specs: []
|
|
303
|
+
transitive_specs: (($transitive_specs[0] // $transitive_specs) | map({
|
|
304
|
+
ecosystem: (.ecosystem // $ecosystem),
|
|
305
|
+
package: .package,
|
|
306
|
+
version: (.version | tostring)
|
|
307
|
+
}) | unique_by(.ecosystem + "\u0000" + .package + "\u0000" + .version))
|
|
275
308
|
}' | safedeps_ledger_atomic_write "${target_path}"
|
|
276
309
|
fi
|
|
277
310
|
|
|
278
311
|
cat "${target_path}"
|
|
279
312
|
}
|
|
280
313
|
|
|
314
|
+
safedeps_ledger_effect_check() {
|
|
315
|
+
local ecosystem="$1"
|
|
316
|
+
local package_name="$2"
|
|
317
|
+
local version="$3"
|
|
318
|
+
local ledger_file
|
|
319
|
+
local now_iso
|
|
320
|
+
|
|
321
|
+
safedeps_ledger_require_jq || return 1
|
|
322
|
+
safedeps_ledger_init
|
|
323
|
+
now_iso=$(safedeps_ledger_now_iso)
|
|
324
|
+
|
|
325
|
+
while IFS= read -r -d '' ledger_file; do
|
|
326
|
+
safedeps_ledger_validate_json "${ledger_file}" || continue
|
|
327
|
+
safedeps_ledger_is_expired_file "${ledger_file}" && continue
|
|
328
|
+
if jq -e \
|
|
329
|
+
--arg ecosystem "${ecosystem}" \
|
|
330
|
+
--arg package "${package_name}" \
|
|
331
|
+
--arg version "${version}" \
|
|
332
|
+
'
|
|
333
|
+
(.revoked_at // "") == ""
|
|
334
|
+
and (
|
|
335
|
+
(.ecosystem == $ecosystem and .package == $package and .version == $version)
|
|
336
|
+
or (((.transitive_specs // []) | map(select(
|
|
337
|
+
(.ecosystem // $ecosystem) == $ecosystem
|
|
338
|
+
and .package == $package
|
|
339
|
+
and (.version | tostring) == $version
|
|
340
|
+
)) | length) > 0)
|
|
341
|
+
)
|
|
342
|
+
' \
|
|
343
|
+
"${ledger_file}" >/dev/null; then
|
|
344
|
+
jq -cn \
|
|
345
|
+
--arg owner_hash "$(jq -r '.hash' "${ledger_file}")" \
|
|
346
|
+
--arg owner_package "$(jq -r '.package' "${ledger_file}")" \
|
|
347
|
+
--arg owner_version "$(jq -r '.version' "${ledger_file}")" \
|
|
348
|
+
--arg checked_at "${now_iso}" \
|
|
349
|
+
'{approved:true, reason:"hit", owner_hash:$owner_hash, owner_package:$owner_package, owner_version:$owner_version, checked_at:$checked_at}'
|
|
350
|
+
return 0
|
|
351
|
+
fi
|
|
352
|
+
done < <(find "${SAFEDEPS_LEDGER_DIR}" -maxdepth 1 -name '*.json' -type f -print0 2>/dev/null)
|
|
353
|
+
|
|
354
|
+
jq -cn \
|
|
355
|
+
--arg ecosystem "${ecosystem}" \
|
|
356
|
+
--arg package "${package_name}" \
|
|
357
|
+
--arg version "${version}" \
|
|
358
|
+
--arg checked_at "${now_iso}" \
|
|
359
|
+
'{approved:false, reason:"miss", ecosystem:$ecosystem, package:$package, version:$version, checked_at:$checked_at}'
|
|
360
|
+
return 1
|
|
361
|
+
}
|
|
362
|
+
|
|
281
363
|
safedeps_ledger_revoke() {
|
|
282
364
|
local ecosystem="$1"
|
|
283
365
|
local package_name="$2"
|
|
284
366
|
local version="$3"
|
|
285
367
|
local reason="${4:-revoked}"
|
|
286
368
|
local ledger_file
|
|
287
|
-
local temp_path
|
|
288
369
|
local revoked_at
|
|
289
370
|
|
|
290
371
|
ledger_file=$(safedeps_ledger_path "${ecosystem}" "${package_name}" "${version}")
|
|
291
372
|
[[ -f "${ledger_file}" ]] || return 1
|
|
292
373
|
safedeps_ledger_validate_json "${ledger_file}" || return 1
|
|
293
374
|
|
|
294
|
-
temp_path="${ledger_file}.$$"
|
|
295
375
|
revoked_at=$(safedeps_ledger_now_iso)
|
|
296
376
|
jq \
|
|
297
377
|
--arg revoked_at "${revoked_at}" \
|
|
298
378
|
--arg reason "${reason}" \
|
|
299
379
|
'. + {revoked_at: $revoked_at, revoked_reason: $reason, expires_at: $revoked_at}' \
|
|
300
|
-
"${ledger_file}"
|
|
301
|
-
chmod 600 "${temp_path}" 2>/dev/null || true
|
|
302
|
-
safedeps_ledger_validate_json "${temp_path}" || {
|
|
303
|
-
rm -f "${temp_path}"
|
|
304
|
-
return 1
|
|
305
|
-
}
|
|
306
|
-
mv "${temp_path}" "${ledger_file}"
|
|
380
|
+
"${ledger_file}" | safedeps_ledger_atomic_write "${ledger_file}"
|
|
307
381
|
cat "${ledger_file}"
|
|
308
382
|
}
|
|
309
383
|
|
|
@@ -325,12 +399,16 @@ if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
|
|
325
399
|
safedeps_ledger_check "$@"
|
|
326
400
|
;;
|
|
327
401
|
approve)
|
|
328
|
-
if [[ "$#" -lt 3 || "$#" -gt
|
|
329
|
-
printf 'usage: %s approve <ecosystem> <package> <version> [version_range] [approved_by] [evidence_file] [ttl_days]\n' "$0" >&2
|
|
402
|
+
if [[ "$#" -lt 3 || "$#" -gt 8 ]]; then
|
|
403
|
+
printf 'usage: %s approve <ecosystem> <package> <version> [version_range] [approved_by] [evidence_file] [ttl_days] [transitive_specs_file]\n' "$0" >&2
|
|
330
404
|
exit 2
|
|
331
405
|
fi
|
|
332
406
|
safedeps_ledger_write_approved_spec "$@"
|
|
333
407
|
;;
|
|
408
|
+
effect-check)
|
|
409
|
+
[[ "$#" -eq 3 ]] || { printf 'usage: %s effect-check <ecosystem> <package> <version>\n' "$0" >&2; exit 2; }
|
|
410
|
+
safedeps_ledger_effect_check "$@"
|
|
411
|
+
;;
|
|
334
412
|
revoke)
|
|
335
413
|
if [[ "$#" -lt 3 || "$#" -gt 4 ]]; then
|
|
336
414
|
printf 'usage: %s revoke <ecosystem> <package> <version> [reason]\n' "$0" >&2
|
|
@@ -339,7 +417,7 @@ if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
|
|
339
417
|
safedeps_ledger_revoke "$@"
|
|
340
418
|
;;
|
|
341
419
|
*)
|
|
342
|
-
printf 'usage: %s {hash|path|check|approve|revoke} ...\n' "$0" >&2
|
|
420
|
+
printf 'usage: %s {hash|path|check|effect-check|approve|revoke} ...\n' "$0" >&2
|
|
343
421
|
exit 2
|
|
344
422
|
;;
|
|
345
423
|
esac
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# npm dependency closure helpers for safedeps.
|
|
3
|
+
|
|
4
|
+
set -euo pipefail
|
|
5
|
+
|
|
6
|
+
safedeps_npm_require_jq() {
|
|
7
|
+
if ! command -v jq >/dev/null 2>&1; then
|
|
8
|
+
printf 'safedeps npm closure: jq is required\n' >&2
|
|
9
|
+
return 1
|
|
10
|
+
fi
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
safedeps_npm_lock_closure() {
|
|
14
|
+
local lockfile="$1"
|
|
15
|
+
local direct_package="${2:-}"
|
|
16
|
+
|
|
17
|
+
safedeps_npm_require_jq || return 1
|
|
18
|
+
[[ -f "${lockfile}" ]] || {
|
|
19
|
+
printf 'safedeps npm closure: lockfile not found: %s\n' "${lockfile}" >&2
|
|
20
|
+
return 1
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
jq -c --arg direct_package "${direct_package}" '
|
|
24
|
+
def package_name_from_path($path):
|
|
25
|
+
($path | split("node_modules/") | last) as $tail
|
|
26
|
+
| if ($tail | startswith("@")) then
|
|
27
|
+
($tail | split("/") | .[0:2] | join("/"))
|
|
28
|
+
else
|
|
29
|
+
($tail | split("/") | .[0])
|
|
30
|
+
end;
|
|
31
|
+
|
|
32
|
+
if ((.packages // null) | type) == "object" then
|
|
33
|
+
[
|
|
34
|
+
.packages
|
|
35
|
+
| to_entries[]
|
|
36
|
+
| select(.key != "")
|
|
37
|
+
| select((.value.version // "") != "")
|
|
38
|
+
| {
|
|
39
|
+
ecosystem: "npm",
|
|
40
|
+
package: (.value.name // package_name_from_path(.key)),
|
|
41
|
+
version: (.value.version | tostring)
|
|
42
|
+
}
|
|
43
|
+
| select(.package != "" and .version != "")
|
|
44
|
+
| . + {direct: (.package == $direct_package)}
|
|
45
|
+
]
|
|
46
|
+
| unique_by(.ecosystem + "\u0000" + .package + "\u0000" + .version)
|
|
47
|
+
| sort_by(.package, .version)
|
|
48
|
+
else
|
|
49
|
+
[]
|
|
50
|
+
end
|
|
51
|
+
' "${lockfile}"
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
safedeps_npm_fixture_closure() {
|
|
55
|
+
local package_name="$1"
|
|
56
|
+
local version="$2"
|
|
57
|
+
local fixture_file="${SAFEDEPS_NPM_CLOSURE_FIXTURE_JSON:-}"
|
|
58
|
+
local key="${package_name}@${version}"
|
|
59
|
+
|
|
60
|
+
[[ -n "${fixture_file}" && -f "${fixture_file}" ]] || return 1
|
|
61
|
+
jq -e -c --arg key "${key}" --arg package "${package_name}" '
|
|
62
|
+
if type == "object" and (.[$key] | type) == "array" then
|
|
63
|
+
.[$key]
|
|
64
|
+
elif type == "array" then
|
|
65
|
+
.
|
|
66
|
+
else
|
|
67
|
+
empty
|
|
68
|
+
end
|
|
69
|
+
| map(. + {ecosystem: (.ecosystem // "npm"), direct: ((.direct // false) or (.package == $package))})
|
|
70
|
+
| unique_by(.ecosystem + "\u0000" + .package + "\u0000" + (.version | tostring))
|
|
71
|
+
| sort_by(.package, .version)
|
|
72
|
+
' "${fixture_file}"
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
safedeps_npm_resolve_spec_closure() {
|
|
76
|
+
local package_name="$1"
|
|
77
|
+
local version="$2"
|
|
78
|
+
local tmp_dir
|
|
79
|
+
local lockfile
|
|
80
|
+
|
|
81
|
+
safedeps_npm_require_jq || return 1
|
|
82
|
+
|
|
83
|
+
if safedeps_npm_fixture_closure "${package_name}" "${version}"; then
|
|
84
|
+
return 0
|
|
85
|
+
fi
|
|
86
|
+
|
|
87
|
+
if ! command -v npm >/dev/null 2>&1; then
|
|
88
|
+
printf 'safedeps npm closure: npm CLI is required\n' >&2
|
|
89
|
+
return 1
|
|
90
|
+
fi
|
|
91
|
+
|
|
92
|
+
tmp_dir=$(mktemp -d "${TMPDIR:-/tmp}/safedeps-npm-closure.XXXXXX") || return 1
|
|
93
|
+
|
|
94
|
+
printf '{"name":"safedeps-closure-probe","version":"0.0.0","private":true}\n' > "${tmp_dir}/package.json"
|
|
95
|
+
if ! (
|
|
96
|
+
cd "${tmp_dir}" &&
|
|
97
|
+
npm install "${package_name}@${version}" \
|
|
98
|
+
--package-lock-only \
|
|
99
|
+
--ignore-scripts \
|
|
100
|
+
--audit=false \
|
|
101
|
+
--fund=false \
|
|
102
|
+
--save-exact \
|
|
103
|
+
>/dev/null
|
|
104
|
+
); then
|
|
105
|
+
printf 'safedeps npm closure: npm lockfile resolution failed for %s@%s\n' "${package_name}" "${version}" >&2
|
|
106
|
+
rm -rf "${tmp_dir}"
|
|
107
|
+
return 1
|
|
108
|
+
fi
|
|
109
|
+
|
|
110
|
+
lockfile="${tmp_dir}/package-lock.json"
|
|
111
|
+
safedeps_npm_lock_closure "${lockfile}" "${package_name}"
|
|
112
|
+
local status=$?
|
|
113
|
+
rm -rf "${tmp_dir}"
|
|
114
|
+
return "${status}"
|
|
115
|
+
}
|
|
@@ -10,6 +10,7 @@ SAFEDEPS_ADVISORY_LOG="${SAFEDEPS_ADVISORY_LOG:-${SAFEDEPS_HOME}/advisory.log}"
|
|
|
10
10
|
SAFEDEPS_PROVIDER_CACHE_TTL_SECONDS="${SAFEDEPS_PROVIDER_CACHE_TTL_SECONDS:-86400}"
|
|
11
11
|
|
|
12
12
|
SAFEDEPS_OSV_API_URL="${SAFEDEPS_OSV_API_URL:-https://api.osv.dev/v1/query}"
|
|
13
|
+
SAFEDEPS_OSV_BATCH_API_URL="${SAFEDEPS_OSV_BATCH_API_URL:-https://api.osv.dev/v1/querybatch}"
|
|
13
14
|
SAFEDEPS_KEV_CATALOG_URL="${SAFEDEPS_KEV_CATALOG_URL:-https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json}"
|
|
14
15
|
SAFEDEPS_GHSA_API_URL="${SAFEDEPS_GHSA_API_URL:-https://api.github.com/advisories}"
|
|
15
16
|
|
|
@@ -51,6 +52,27 @@ safedeps_provider_mktemp_dir() {
|
|
|
51
52
|
mktemp -d "${tmp_root%/}/safedeps-providers.XXXXXX"
|
|
52
53
|
}
|
|
53
54
|
|
|
55
|
+
safedeps_cache_response_temp() {
|
|
56
|
+
local target_path="$1"
|
|
57
|
+
local target_dir
|
|
58
|
+
local target_base
|
|
59
|
+
|
|
60
|
+
target_dir=$(dirname "${target_path}")
|
|
61
|
+
target_base=$(basename "${target_path}")
|
|
62
|
+
mkdir -p "${target_dir}" || return 1
|
|
63
|
+
mktemp "${target_dir}/.${target_base}.XXXXXX"
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
safedeps_url_host() {
|
|
67
|
+
local url="$1"
|
|
68
|
+
local host
|
|
69
|
+
|
|
70
|
+
host="${url#*://}"
|
|
71
|
+
host="${host%%/*}"
|
|
72
|
+
host="${host%%:*}"
|
|
73
|
+
printf '%s' "${host}"
|
|
74
|
+
}
|
|
75
|
+
|
|
54
76
|
safedeps_now_iso() {
|
|
55
77
|
date -u +"%Y-%m-%dT%H:%M:%SZ"
|
|
56
78
|
}
|
|
@@ -71,7 +93,10 @@ safedeps_hash_text() {
|
|
|
71
93
|
safedeps_file_mtime() {
|
|
72
94
|
local path="$1"
|
|
73
95
|
|
|
74
|
-
|
|
96
|
+
# GNU coreutils (Linux) uses `-c %Y`; BSD/macOS uses `-f %m`. GNU must run
|
|
97
|
+
# first: on Linux `stat -f` means --file-system and prints filesystem info to
|
|
98
|
+
# stdout, which would pollute the mtime and break the arithmetic downstream.
|
|
99
|
+
stat -c %Y "${path}" 2>/dev/null || stat -f %m "${path}" 2>/dev/null
|
|
75
100
|
}
|
|
76
101
|
|
|
77
102
|
safedeps_cache_is_fresh() {
|
|
@@ -167,7 +192,7 @@ safedeps_osv_query() {
|
|
|
167
192
|
--arg version "${version}" \
|
|
168
193
|
'{version: $version, package: {name: $package, ecosystem: $ecosystem}}')
|
|
169
194
|
|
|
170
|
-
response_file
|
|
195
|
+
response_file=$(safedeps_cache_response_temp "${cache_path}") || return 1
|
|
171
196
|
http_status=$(curl -fsS \
|
|
172
197
|
--max-time 15 \
|
|
173
198
|
-H 'Content-Type: application/json' \
|
|
@@ -177,7 +202,7 @@ safedeps_osv_query() {
|
|
|
177
202
|
"${SAFEDEPS_OSV_API_URL}" 2>/dev/null || true)
|
|
178
203
|
|
|
179
204
|
if [[ "${http_status}" == "200" ]] && jq -e 'type == "object"' "${response_file}" >/dev/null 2>&1; then
|
|
180
|
-
mv "${response_file}" "${cache_path}"
|
|
205
|
+
mv -f "${response_file}" "${cache_path}"
|
|
181
206
|
safedeps_provider_log "INFO" "OSV live query ok ecosystem=${osv_ecosystem} package=${package_name} version=${version}"
|
|
182
207
|
cat "${cache_path}"
|
|
183
208
|
return 0
|
|
@@ -192,6 +217,134 @@ safedeps_osv_query() {
|
|
|
192
217
|
return 1
|
|
193
218
|
}
|
|
194
219
|
|
|
220
|
+
safedeps_osv_query_batch() {
|
|
221
|
+
local ecosystem="$1"
|
|
222
|
+
local closure_file="$2"
|
|
223
|
+
local osv_ecosystem
|
|
224
|
+
local temp_dir
|
|
225
|
+
local all_items_file
|
|
226
|
+
local miss_items_file
|
|
227
|
+
local payload_file
|
|
228
|
+
local response_file
|
|
229
|
+
local results_file
|
|
230
|
+
local http_status
|
|
231
|
+
local index=0
|
|
232
|
+
local package_name
|
|
233
|
+
local version
|
|
234
|
+
local direct
|
|
235
|
+
|
|
236
|
+
safedeps_require_json_tools || return 1
|
|
237
|
+
safedeps_providers_init
|
|
238
|
+
[[ -f "${closure_file}" ]] || return 1
|
|
239
|
+
|
|
240
|
+
osv_ecosystem=$(safedeps_osv_ecosystem "${ecosystem}")
|
|
241
|
+
temp_dir=$(safedeps_provider_mktemp_dir) || return 1
|
|
242
|
+
all_items_file="${temp_dir}/items.jsonl"
|
|
243
|
+
miss_items_file="${temp_dir}/misses.jsonl"
|
|
244
|
+
payload_file="${temp_dir}/payload.json"
|
|
245
|
+
response_file="${temp_dir}/response.json"
|
|
246
|
+
results_file="${temp_dir}/results.json"
|
|
247
|
+
: > "${all_items_file}"
|
|
248
|
+
: > "${miss_items_file}"
|
|
249
|
+
|
|
250
|
+
while IFS=$'\t' read -r package_name version direct; do
|
|
251
|
+
[[ -n "${package_name}" && -n "${version}" ]] || continue
|
|
252
|
+
local cache_key
|
|
253
|
+
local cache_path
|
|
254
|
+
cache_key=$(safedeps_cache_key "osv" "${osv_ecosystem}" "${package_name}" "${version}")
|
|
255
|
+
cache_path="${SAFEDEPS_CACHE_DIR}/osv/${cache_key}.json"
|
|
256
|
+
|
|
257
|
+
if safedeps_cache_is_fresh "${cache_path}"; then
|
|
258
|
+
safedeps_provider_log "INFO" "OSV batch cache hit ecosystem=${osv_ecosystem} package=${package_name} version=${version}"
|
|
259
|
+
jq -cn \
|
|
260
|
+
--argjson index "${index}" \
|
|
261
|
+
--arg ecosystem "${ecosystem}" \
|
|
262
|
+
--arg package "${package_name}" \
|
|
263
|
+
--arg version "${version}" \
|
|
264
|
+
--argjson direct "${direct}" \
|
|
265
|
+
--slurpfile osv "${cache_path}" \
|
|
266
|
+
'{index:$index, ecosystem:$ecosystem, package:$package, version:$version, direct:$direct, osv:($osv[0] // {vulns:[]})}' >> "${all_items_file}"
|
|
267
|
+
else
|
|
268
|
+
jq -cn \
|
|
269
|
+
--argjson index "${index}" \
|
|
270
|
+
--arg ecosystem "${ecosystem}" \
|
|
271
|
+
--arg package "${package_name}" \
|
|
272
|
+
--arg version "${version}" \
|
|
273
|
+
--argjson direct "${direct}" \
|
|
274
|
+
--arg cache_path "${cache_path}" \
|
|
275
|
+
'{index:$index, ecosystem:$ecosystem, package:$package, version:$version, direct:$direct, cache_path:$cache_path}' >> "${miss_items_file}"
|
|
276
|
+
fi
|
|
277
|
+
index=$((index + 1))
|
|
278
|
+
done < <(jq -r '.[] | [.package, (.version | tostring), ((.direct // false) | tostring)] | @tsv' "${closure_file}")
|
|
279
|
+
|
|
280
|
+
if [[ -s "${miss_items_file}" ]]; then
|
|
281
|
+
safedeps_require_http_client || {
|
|
282
|
+
safedeps_provider_log "ERROR" "OSV batch unavailable; cache miss ecosystem=${osv_ecosystem}"
|
|
283
|
+
rm -rf "${temp_dir}"
|
|
284
|
+
return 1
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
jq -cn --arg ecosystem "${osv_ecosystem}" --slurpfile misses "${miss_items_file}" '
|
|
288
|
+
{
|
|
289
|
+
queries: [
|
|
290
|
+
$misses[]
|
|
291
|
+
| {version: .version, package: {name: .package, ecosystem: $ecosystem}}
|
|
292
|
+
]
|
|
293
|
+
}
|
|
294
|
+
' > "${payload_file}"
|
|
295
|
+
|
|
296
|
+
http_status=$(curl -fsS \
|
|
297
|
+
--max-time 20 \
|
|
298
|
+
-H 'Content-Type: application/json' \
|
|
299
|
+
-o "${response_file}" \
|
|
300
|
+
-w '%{http_code}' \
|
|
301
|
+
-d @"${payload_file}" \
|
|
302
|
+
"${SAFEDEPS_OSV_BATCH_API_URL}" 2>/dev/null || true)
|
|
303
|
+
|
|
304
|
+
if [[ "${http_status}" != "200" ]] || ! jq -e '.results | type == "array"' "${response_file}" >/dev/null 2>&1; then
|
|
305
|
+
safedeps_provider_log "ERROR" "OSV batch query failed status=${http_status:-none}"
|
|
306
|
+
rm -rf "${temp_dir}"
|
|
307
|
+
return 1
|
|
308
|
+
fi
|
|
309
|
+
|
|
310
|
+
local miss_count
|
|
311
|
+
local result_count
|
|
312
|
+
miss_count=$(jq -s 'length' "${miss_items_file}")
|
|
313
|
+
result_count=$(jq '.results | length' "${response_file}")
|
|
314
|
+
if [[ "${miss_count}" != "${result_count}" ]]; then
|
|
315
|
+
safedeps_provider_log "ERROR" "OSV batch result count mismatch misses=${miss_count} results=${result_count}"
|
|
316
|
+
rm -rf "${temp_dir}"
|
|
317
|
+
return 1
|
|
318
|
+
fi
|
|
319
|
+
|
|
320
|
+
local miss_i=0
|
|
321
|
+
while IFS= read -r miss_item; do
|
|
322
|
+
local cache_path
|
|
323
|
+
local response_item_file
|
|
324
|
+
cache_path=$(jq -r '.cache_path' <<< "${miss_item}")
|
|
325
|
+
response_item_file=$(safedeps_cache_response_temp "${cache_path}") || {
|
|
326
|
+
rm -rf "${temp_dir}"
|
|
327
|
+
return 1
|
|
328
|
+
}
|
|
329
|
+
jq -c --argjson i "${miss_i}" '.results[$i] // {vulns: []}' "${response_file}" > "${response_item_file}"
|
|
330
|
+
if ! jq -e 'type == "object"' "${response_item_file}" >/dev/null 2>&1; then
|
|
331
|
+
rm -f "${response_item_file}"
|
|
332
|
+
rm -rf "${temp_dir}"
|
|
333
|
+
return 1
|
|
334
|
+
fi
|
|
335
|
+
mv -f "${response_item_file}" "${cache_path}"
|
|
336
|
+
safedeps_provider_log "INFO" "OSV batch live query ok ecosystem=${osv_ecosystem} package=$(jq -r '.package' <<< "${miss_item}") version=$(jq -r '.version' <<< "${miss_item}")"
|
|
337
|
+
jq -cn --argjson miss "${miss_item}" --slurpfile osv "${cache_path}" \
|
|
338
|
+
'$miss | del(.cache_path) | . + {osv: ($osv[0] // {vulns: []})}' >> "${all_items_file}"
|
|
339
|
+
miss_i=$((miss_i + 1))
|
|
340
|
+
done < "${miss_items_file}"
|
|
341
|
+
fi
|
|
342
|
+
|
|
343
|
+
jq -s 'sort_by(.index)' "${all_items_file}" > "${results_file}"
|
|
344
|
+
cat "${results_file}"
|
|
345
|
+
rm -rf "${temp_dir}"
|
|
346
|
+
}
|
|
347
|
+
|
|
195
348
|
safedeps_extract_cves_from_osv() {
|
|
196
349
|
local osv_json="$1"
|
|
197
350
|
|
|
@@ -230,11 +383,11 @@ safedeps_kev_refresh_catalog() {
|
|
|
230
383
|
return 1
|
|
231
384
|
fi
|
|
232
385
|
|
|
233
|
-
response_path
|
|
386
|
+
response_path=$(safedeps_cache_response_temp "${cache_path}") || return 1
|
|
234
387
|
http_status=$(curl -fsS --max-time 15 -o "${response_path}" -w '%{http_code}' "${SAFEDEPS_KEV_CATALOG_URL}" 2>/dev/null || true)
|
|
235
388
|
|
|
236
389
|
if [[ "${http_status}" == "200" ]] && jq -e '.vulnerabilities | type == "array"' "${response_path}" >/dev/null 2>&1; then
|
|
237
|
-
mv "${response_path}" "${cache_path}"
|
|
390
|
+
mv -f "${response_path}" "${cache_path}"
|
|
238
391
|
safedeps_provider_log "INFO" "CISA KEV catalog refresh ok"
|
|
239
392
|
printf '%s' "${cache_path}"
|
|
240
393
|
return 0
|
|
@@ -301,6 +454,8 @@ safedeps_ghsa_query() {
|
|
|
301
454
|
local cache_path
|
|
302
455
|
local response_file
|
|
303
456
|
local http_status
|
|
457
|
+
local ghsa_host
|
|
458
|
+
local curl_args
|
|
304
459
|
|
|
305
460
|
safedeps_require_json_tools || return 1
|
|
306
461
|
safedeps_providers_init
|
|
@@ -324,29 +479,29 @@ safedeps_ghsa_query() {
|
|
|
324
479
|
|
|
325
480
|
encoded_ecosystem=$(safedeps_json_uri_escape "${ghsa_ecosystem}")
|
|
326
481
|
encoded_package=$(safedeps_json_uri_escape "${package_name}")
|
|
327
|
-
response_file
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
-o "${response_file}" \
|
|
344
|
-
-w '%{http_code}' \
|
|
345
|
-
"${SAFEDEPS_GHSA_API_URL}?ecosystem=${encoded_ecosystem}&affects=${encoded_package}&per_page=100" 2>/dev/null || true)
|
|
482
|
+
response_file=$(safedeps_cache_response_temp "${cache_path}") || return 1
|
|
483
|
+
ghsa_host=$(safedeps_url_host "${SAFEDEPS_GHSA_API_URL}")
|
|
484
|
+
|
|
485
|
+
curl_args=(
|
|
486
|
+
-fsS
|
|
487
|
+
--max-time 15
|
|
488
|
+
-H 'Accept: application/vnd.github+json'
|
|
489
|
+
-H 'X-GitHub-Api-Version: 2022-11-28'
|
|
490
|
+
-o "${response_file}"
|
|
491
|
+
-w '%{http_code}'
|
|
492
|
+
)
|
|
493
|
+
|
|
494
|
+
if [[ -n "${GITHUB_TOKEN:-}" && "${ghsa_host}" == "api.github.com" ]]; then
|
|
495
|
+
curl_args+=(-H "Authorization: Bearer ${GITHUB_TOKEN}")
|
|
496
|
+
elif [[ -n "${GITHUB_TOKEN:-}" ]]; then
|
|
497
|
+
safedeps_provider_log "WARN" "GHSA token withheld for non-GitHub host host=${ghsa_host}"
|
|
346
498
|
fi
|
|
347
499
|
|
|
500
|
+
http_status=$(curl "${curl_args[@]}" \
|
|
501
|
+
"${SAFEDEPS_GHSA_API_URL}?ecosystem=${encoded_ecosystem}&affects=${encoded_package}&per_page=100" 2>/dev/null || true)
|
|
502
|
+
|
|
348
503
|
if [[ "${http_status}" == "200" ]] && jq -e 'type == "array"' "${response_file}" >/dev/null 2>&1; then
|
|
349
|
-
mv "${response_file}" "${cache_path}"
|
|
504
|
+
mv -f "${response_file}" "${cache_path}"
|
|
350
505
|
safedeps_provider_log "INFO" "GHSA live query ok ecosystem=${ghsa_ecosystem} package=${package_name}"
|
|
351
506
|
jq -cn --arg queried_at "${queried_at}" --slurpfile advisories "${cache_path}" \
|
|
352
507
|
'{queried_at: $queried_at, status: "live", advisories: $advisories[0]}'
|
|
@@ -459,6 +614,66 @@ safedeps_providers_query() {
|
|
|
459
614
|
rm -rf "${temp_dir}"
|
|
460
615
|
}
|
|
461
616
|
|
|
617
|
+
safedeps_providers_query_batch() {
|
|
618
|
+
local ecosystem="$1"
|
|
619
|
+
local closure_file="$2"
|
|
620
|
+
local queried_at
|
|
621
|
+
local temp_dir
|
|
622
|
+
local osv_batch_file
|
|
623
|
+
local results_file
|
|
624
|
+
|
|
625
|
+
safedeps_require_json_tools || return 1
|
|
626
|
+
queried_at=$(safedeps_now_iso)
|
|
627
|
+
temp_dir=$(safedeps_provider_mktemp_dir) || return 1
|
|
628
|
+
osv_batch_file="${temp_dir}/osv-batch.json"
|
|
629
|
+
results_file="${temp_dir}/results.jsonl"
|
|
630
|
+
: > "${results_file}"
|
|
631
|
+
|
|
632
|
+
if ! safedeps_osv_query_batch "${ecosystem}" "${closure_file}" > "${osv_batch_file}"; then
|
|
633
|
+
rm -rf "${temp_dir}"
|
|
634
|
+
return 1
|
|
635
|
+
fi
|
|
636
|
+
|
|
637
|
+
while IFS= read -r item; do
|
|
638
|
+
local osv_file
|
|
639
|
+
local kev_json
|
|
640
|
+
local status
|
|
641
|
+
osv_file="${temp_dir}/osv-item.json"
|
|
642
|
+
jq -c '.osv' <<< "${item}" > "${osv_file}"
|
|
643
|
+
kev_json=$(safedeps_kev_overlay "${osv_file}" "${queried_at}")
|
|
644
|
+
status=$(jq -r --argjson kev "${kev_json}" '
|
|
645
|
+
if $kev.exploited then "hard_block"
|
|
646
|
+
elif ((.osv.vulns // []) | length) > 0 then "vulnerable"
|
|
647
|
+
else "clean"
|
|
648
|
+
end
|
|
649
|
+
' <<< "${item}")
|
|
650
|
+
jq -cn \
|
|
651
|
+
--argjson item "${item}" \
|
|
652
|
+
--arg queried_at "${queried_at}" \
|
|
653
|
+
--arg status "${status}" \
|
|
654
|
+
--argjson kev "${kev_json}" \
|
|
655
|
+
'{
|
|
656
|
+
index: $item.index,
|
|
657
|
+
ecosystem: $item.ecosystem,
|
|
658
|
+
package: $item.package,
|
|
659
|
+
version: $item.version,
|
|
660
|
+
direct: ($item.direct // false),
|
|
661
|
+
queried_at: $queried_at,
|
|
662
|
+
status: $status,
|
|
663
|
+
vulnerabilities: ($item.osv.vulns // []),
|
|
664
|
+
kev: $kev,
|
|
665
|
+
provider_status: {
|
|
666
|
+
osv: {status: "ok", canonical: true, batch: true},
|
|
667
|
+
kev: {status: ($kev.status // "ok"), overlay: true},
|
|
668
|
+
ghsa: {status: "skipped", enrichment: true, reason: "closure batch omits GHSA enrichment"}
|
|
669
|
+
}
|
|
670
|
+
}' >> "${results_file}"
|
|
671
|
+
done < <(jq -c '.[]' "${osv_batch_file}")
|
|
672
|
+
|
|
673
|
+
jq -s 'sort_by(.index)' "${results_file}"
|
|
674
|
+
rm -rf "${temp_dir}"
|
|
675
|
+
}
|
|
676
|
+
|
|
462
677
|
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
|
463
678
|
command_name="${1:-}"
|
|
464
679
|
shift || true
|
|
@@ -471,8 +686,15 @@ if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
|
|
|
471
686
|
fi
|
|
472
687
|
safedeps_providers_query "$@"
|
|
473
688
|
;;
|
|
689
|
+
query-batch)
|
|
690
|
+
if [[ "$#" -ne 2 ]]; then
|
|
691
|
+
printf 'usage: %s query-batch <ecosystem> <closure-json-file>\n' "$0" >&2
|
|
692
|
+
exit 2
|
|
693
|
+
fi
|
|
694
|
+
safedeps_providers_query_batch "$@"
|
|
695
|
+
;;
|
|
474
696
|
*)
|
|
475
|
-
printf 'usage: %s query
|
|
697
|
+
printf 'usage: %s {query|query-batch} ...\n' "$0" >&2
|
|
476
698
|
exit 2
|
|
477
699
|
;;
|
|
478
700
|
esac
|