@allansantos-dev/smart-tool 0.9.79 → 0.9.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/README.md +65 -18
- package/browser_control.py +10 -3
- package/browser_service_client.py +3 -32
- package/chromium_flags.py +30 -0
- package/code_health.py +180 -0
- package/code_impact.py +13 -1
- package/config.py +41 -8
- package/credential_check.py +38 -0
- package/dependency_notice.py +262 -14
- package/document_converter.py +221 -0
- package/document_read.py +100 -25
- package/document_text.py +194 -6
- package/dossier.py +35 -8
- package/embedding_cache.py +32 -8
- package/freshness.py +3 -2
- package/hook_decision.py +65 -14
- package/indexer.py +92 -3
- package/layer_check.py +57 -0
- package/local_embedder.py +114 -57
- package/locales/pt-BR.json +52 -17
- package/low_priority.py +41 -0
- package/package.json +1 -1
- package/pr_impact.py +111 -8
- package/process_limits.py +108 -0
- package/project_monitor.py +33 -7
- package/project_scheduler.py +12 -4
- package/scout.py +6 -2
- package/setup_audit.py +34 -1
- package/setup_ui.py +210 -39
- package/smart_tool_daemon.py +398 -91
- package/update_check.py +18 -0
- package/usage_meter.py +78 -0
- package/version.py +1 -1
- package/web_adapters/browser/crawl4ai_search.py +7 -3
- package/web_fetch.py +24 -0
- package/web_search_health.py +31 -0
package/config.py
CHANGED
|
@@ -40,12 +40,15 @@ DEFAULT_CONFIG = {
|
|
|
40
40
|
"research_max_iterations": 3,
|
|
41
41
|
"quick_search_timeout_s": 120,
|
|
42
42
|
"site_memory_similarity_threshold": 0.35,
|
|
43
|
-
"
|
|
43
|
+
"embedder": "home",
|
|
44
44
|
"hook_mode": "redirect",
|
|
45
45
|
"doc_mode": "require",
|
|
46
46
|
"comment_mode": "require",
|
|
47
47
|
"duplicate_mode": "warn",
|
|
48
48
|
"dependency_mode": "warn",
|
|
49
|
+
"layer_mode": "warn",
|
|
50
|
+
"document_converter": "off",
|
|
51
|
+
"document_converter_python": "",
|
|
49
52
|
"docs_extras": {key: True for key in DOCS_EXTRAS},
|
|
50
53
|
"auto_update": True,
|
|
51
54
|
"language": "",
|
|
@@ -60,9 +63,10 @@ DEFAULT_CONFIG = {
|
|
|
60
63
|
},
|
|
61
64
|
}
|
|
62
65
|
|
|
63
|
-
#
|
|
64
|
-
#
|
|
65
|
-
|
|
66
|
+
# Embedder of the project indexes: home = the embeddings home daemon of this machine (casa-de-embeddings), whose rerank
|
|
67
|
+
# connector forwards to the host configured in it; gateway = the gateway's embedding_model and rerank_model, only when
|
|
68
|
+
# the user chooses it. Neither replaces the other when it fails.
|
|
69
|
+
EMBEDDERS = ("home", "gateway")
|
|
66
70
|
# PreToolUse hook: redirect = deny the native tool and point to Smart Tool; advise = let it run and add the tip to the
|
|
67
71
|
# agent's context; off = no routing at all.
|
|
68
72
|
HOOK_MODES = ("redirect", "advise", "off")
|
|
@@ -79,6 +83,11 @@ DUPLICATE_MODES = ("warn", "off")
|
|
|
79
83
|
# Edit hook on package.json, requirements*.txt and go.mod: warn = let the edit run and list the added or changed dependencies that have OSV advisories;
|
|
80
84
|
# off = say nothing and make no network call.
|
|
81
85
|
DEPENDENCY_MODES = ("warn", "off")
|
|
86
|
+
LAYER_MODES = ("warn", "off")
|
|
87
|
+
# External converter read_document falls back to for a file type or a scanned PDF its native reader refuses: off = never;
|
|
88
|
+
# auto = the first of kreuzberg and docling found installed; or one of them by name. The engine is looked for in the
|
|
89
|
+
# interpreter named by `document_converter_python` (empty = the one Smart Tool runs on).
|
|
90
|
+
DOCUMENT_CONVERTERS = ("off", "auto", "kreuzberg", "docling")
|
|
82
91
|
# Router model reviewing, in the background, searches the deterministic redirect rule let run: on = propose block
|
|
83
92
|
# patterns for the user to accept in the setup screen; off = no review.
|
|
84
93
|
PATTERN_PROPOSAL_MODES = ("on", "off")
|
|
@@ -118,6 +127,26 @@ def dependency_mode(cfg=None):
|
|
|
118
127
|
return value
|
|
119
128
|
|
|
120
129
|
|
|
130
|
+
def layer_mode(cfg=None):
|
|
131
|
+
value = (cfg if cfg is not None else load_config()).get("layer_mode") or DEFAULT_CONFIG["layer_mode"]
|
|
132
|
+
if value not in LAYER_MODES:
|
|
133
|
+
raise ValueError(f"Invalid layer_mode ({value!r}) in {CONFIG_PATH}: use warn or off.")
|
|
134
|
+
return value
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def document_converter(cfg=None):
|
|
138
|
+
"""The external converter setting: `{"mode": one of DOCUMENT_CONVERTERS, "python": interpreter path or ""}`. An
|
|
139
|
+
unknown mode or a python that is not text fails loudly."""
|
|
140
|
+
stored = cfg if cfg is not None else load_config()
|
|
141
|
+
mode = stored.get("document_converter") or DEFAULT_CONFIG["document_converter"]
|
|
142
|
+
python = stored.get("document_converter_python") or ""
|
|
143
|
+
if mode not in DOCUMENT_CONVERTERS:
|
|
144
|
+
raise ValueError(f"Invalid document_converter ({mode!r}) in {CONFIG_PATH}: use off, auto, kreuzberg or docling.")
|
|
145
|
+
if not isinstance(python, str):
|
|
146
|
+
raise ValueError(f"Invalid document_converter_python in {CONFIG_PATH}: a path to a Python interpreter.")
|
|
147
|
+
return {"mode": mode, "python": python}
|
|
148
|
+
|
|
149
|
+
|
|
121
150
|
def docs_extras(cfg=None):
|
|
122
151
|
"""Which extra documents the docs generator writes: every key of DOCS_EXTRAS, true unless the stored `docs_extras`
|
|
123
152
|
object sets it to false; an unknown key or a value that is not true or false fails loudly."""
|
|
@@ -183,10 +212,14 @@ def hook_mode(cfg=None):
|
|
|
183
212
|
return value
|
|
184
213
|
|
|
185
214
|
|
|
186
|
-
def
|
|
187
|
-
|
|
188
|
-
if
|
|
189
|
-
raise ValueError(f"
|
|
215
|
+
def embedder(cfg=None):
|
|
216
|
+
cfg = cfg if cfg is not None else load_config()
|
|
217
|
+
if "local_models" in cfg:
|
|
218
|
+
raise ValueError(f"{CONFIG_PATH} has the old 'local_models' setting: delete it and choose the embedder (home or "
|
|
219
|
+
"gateway) in the setup screen.")
|
|
220
|
+
value = cfg.get("embedder") or DEFAULT_CONFIG["embedder"]
|
|
221
|
+
if value not in EMBEDDERS:
|
|
222
|
+
raise ValueError(f"Invalid embedder ({value!r}) in {CONFIG_PATH}: use home or gateway.")
|
|
190
223
|
return value
|
|
191
224
|
|
|
192
225
|
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Credentials an edit writes, for the edit hook. Only the lines an edit adds or changes are read, and only for shapes that
|
|
2
|
+
cannot be anything else (a provider's token prefix with its fixed length, an AWS key pair line, a private-key header at
|
|
3
|
+
the start of a line): a false alarm costs more than a silence here, so a general "high entropy string" rule is not used.
|
|
4
|
+
Lines that say they are an example are skipped. The matched text is never repeated in a note."""
|
|
5
|
+
import re
|
|
6
|
+
|
|
7
|
+
import doc_check
|
|
8
|
+
|
|
9
|
+
_PLACEHOLDER = re.compile(r"example|placeholder|your[_-]|dummy|fake|redacted|x{6,}|\*{4,}|<[^>]*>", re.I)
|
|
10
|
+
_PATTERNS = (
|
|
11
|
+
("private key", re.compile(r"^\s*-----BEGIN (?:[A-Z]+ )*PRIVATE KEY(?: BLOCK)?-----")),
|
|
12
|
+
("AWS access key id", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b")),
|
|
13
|
+
("AWS secret access key", re.compile(r"aws_secret_access_key[\"']?\s*[=:]\s*[\"']?[A-Za-z0-9/+=]{40}\b", re.I)),
|
|
14
|
+
("GitHub token", re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{50,})")),
|
|
15
|
+
("Slack token", re.compile(r"\bxox[abprs]-[A-Za-z0-9-]{10,}")),
|
|
16
|
+
("Google API key", re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b")),
|
|
17
|
+
("Stripe live key", re.compile(r"\b[sr]k_live_[0-9A-Za-z]{24,}")),
|
|
18
|
+
("Anthropic API key", re.compile(r"\bsk-ant-[A-Za-z0-9_-]{20,}")),
|
|
19
|
+
("npm token", re.compile(r"\bnpm_[A-Za-z0-9]{36}\b")),
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def review(before, after):
|
|
24
|
+
"""Credentials among the lines an edit adds or changes.
|
|
25
|
+
|
|
26
|
+
Parameters: before (str or None) text before the edit; after (str) text after it.
|
|
27
|
+
Returns: list of dicts with line (1-based) and kind, in line order, one per line; the secret itself is not included.
|
|
28
|
+
"""
|
|
29
|
+
lines = after.split("\n")
|
|
30
|
+
findings = []
|
|
31
|
+
for number in sorted(doc_check._touched_lines(before, after)):
|
|
32
|
+
if number > len(lines) or _PLACEHOLDER.search(lines[number - 1]):
|
|
33
|
+
continue
|
|
34
|
+
for kind, pattern in _PATTERNS:
|
|
35
|
+
if pattern.search(lines[number - 1]):
|
|
36
|
+
findings.append({"line": number, "kind": kind})
|
|
37
|
+
break
|
|
38
|
+
return findings
|
package/dependency_notice.py
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
"""Vulnerable-dependency notice for an edit of a dependency manifest, built before the edit runs.
|
|
2
2
|
|
|
3
|
-
Manifests: package.json (OSV ecosystem npm), requirements*.txt (PyPI: `==` pins and the lower bound
|
|
4
|
-
and
|
|
3
|
+
Manifests: package.json (OSV ecosystem npm), requirements*.txt and pyproject.toml (PyPI: `==` pins and the lower bound
|
|
4
|
+
of `>=` and `~=`; pyproject reads `project.dependencies` and `project.optional-dependencies`, not Poetry tables),
|
|
5
|
+
go.mod (Go: `require` lines with a plain vX.Y.Z), Cargo.toml (crates.io: registry dependencies in every dependency table,
|
|
6
|
+
asked at the lower bound of the requirement) and pom.xml (Maven: `groupId:artifactId` with a plain or property-resolved
|
|
7
|
+
version, no ranges and no version inherited from a parent or a BOM). The audit asks about the version a sibling package-lock.json resolved
|
|
8
|
+
when it satisfies the range, else about the lower bound. Compares the dependencies of the file before and after the edit,
|
|
5
9
|
asks OSV (one `querybatch` call, short timeout) about the added or changed ones only, and answers from a disk cache of
|
|
6
10
|
(ecosystem, package, version) -> advisory IDs for 6 hours. Only package names and versions leave the machine. The caller (`hook_decision`) turns a `DependencyCheckError` into
|
|
7
11
|
"dependency check skipped: <reason>" and lets the edit go on.
|
|
@@ -11,6 +15,8 @@ import json
|
|
|
11
15
|
import os
|
|
12
16
|
import re
|
|
13
17
|
import time
|
|
18
|
+
import tomllib
|
|
19
|
+
import xml.etree.ElementTree as ElementTree
|
|
14
20
|
|
|
15
21
|
import atomic_io
|
|
16
22
|
import freshness
|
|
@@ -20,6 +26,10 @@ CACHE_PATH = os.path.join(paths.DATA_DIR, "dependency_advisories.json")
|
|
|
20
26
|
CACHE_TTL_S = 6 * 3600
|
|
21
27
|
OSV_TIMEOUT_S = 2.5
|
|
22
28
|
MAX_PACKAGES = 5
|
|
29
|
+
BATCH_SIZE = 100
|
|
30
|
+
AUDIT_MAX_MANIFESTS = 50
|
|
31
|
+
AUDIT_MAX_DEPTH = 4
|
|
32
|
+
AUDIT_SKIP_DIRS = frozenset(("node_modules", "vendor", "dist", "build", "target", "__pycache__", "venv", "site-packages"))
|
|
23
33
|
MAX_IDS_SHOWN = 5
|
|
24
34
|
TABLES = ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies")
|
|
25
35
|
PREFIX = "Smart Tool dependencies: "
|
|
@@ -29,6 +39,11 @@ _REQUIREMENTS_FILE_RE = re.compile(r"requirements[A-Za-z0-9._-]*\.txt")
|
|
|
29
39
|
_REQUIREMENT_RE = re.compile(r"([A-Za-z0-9][A-Za-z0-9._-]*)\s*(?:\[[^\]]*\])?\s*([^;]*)")
|
|
30
40
|
_PIP_VERSION_RE = re.compile(r"[0-9][0-9A-Za-z.+!_-]*")
|
|
31
41
|
_GO_VERSION_RE = re.compile(r"v\d+\.\d+\.\d+")
|
|
42
|
+
_LOCKED_RE = re.compile(r"(\d+)\.(\d+)\.(\d+)")
|
|
43
|
+
_MAVEN_VERSION_RE = re.compile(r"[0-9][0-9A-Za-z.+_-]*")
|
|
44
|
+
_PROPERTY_RE = re.compile(r"\$\{([^}]+)\}")
|
|
45
|
+
_CARGO_TABLES = ("dependencies", "dev-dependencies", "build-dependencies")
|
|
46
|
+
_TAG_RE = re.compile(r"\*|[xX]|[A-Za-z][A-Za-z0-9._-]*")
|
|
32
47
|
|
|
33
48
|
|
|
34
49
|
class DependencyCheckError(RuntimeError):
|
|
@@ -102,17 +117,100 @@ def _go_pairs(text):
|
|
|
102
117
|
return pairs
|
|
103
118
|
|
|
104
119
|
|
|
120
|
+
def _pyproject_pairs(text):
|
|
121
|
+
"""The (name, specifier) pairs of `project.dependencies` and `project.optional-dependencies` in a pyproject.toml text.
|
|
122
|
+
|
|
123
|
+
Parameters: text (str) file content.
|
|
124
|
+
Returns: dict (name, specifier) -> None in file order, read as requirement lines; None when the text is not TOML.
|
|
125
|
+
"""
|
|
126
|
+
try:
|
|
127
|
+
project = tomllib.loads(text).get("project")
|
|
128
|
+
except tomllib.TOMLDecodeError:
|
|
129
|
+
return None
|
|
130
|
+
if not isinstance(project, dict):
|
|
131
|
+
return {}
|
|
132
|
+
lines = [item for item in project.get("dependencies") or [] if isinstance(item, str)]
|
|
133
|
+
optional = project.get("optional-dependencies")
|
|
134
|
+
for group in optional.values() if isinstance(optional, dict) else ():
|
|
135
|
+
lines += [item for item in group if isinstance(item, str)]
|
|
136
|
+
return _requirements_pairs("\n".join(lines))
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _cargo_pairs(text):
|
|
140
|
+
"""The (crate, requirement) pairs of the registry dependencies of a Cargo.toml text: the dependency, dev-dependency and
|
|
141
|
+
build-dependency tables, the `target.*` tables and `workspace.dependencies`. Path, git and `workspace = true` entries
|
|
142
|
+
have no registry version and are skipped; a `package` key renames the crate.
|
|
143
|
+
|
|
144
|
+
Parameters: text (str) file content.
|
|
145
|
+
Returns: dict (crate, requirement) -> None in file order ("" when a registry entry names no version); None when the
|
|
146
|
+
text is not TOML.
|
|
147
|
+
"""
|
|
148
|
+
try:
|
|
149
|
+
data = tomllib.loads(text)
|
|
150
|
+
except tomllib.TOMLDecodeError:
|
|
151
|
+
return None
|
|
152
|
+
groups = [data]
|
|
153
|
+
targets = data.get("target")
|
|
154
|
+
groups += [group for group in targets.values() if isinstance(group, dict)] if isinstance(targets, dict) else []
|
|
155
|
+
workspace = data.get("workspace")
|
|
156
|
+
groups += [workspace] if isinstance(workspace, dict) else []
|
|
157
|
+
pairs = {}
|
|
158
|
+
for group in groups:
|
|
159
|
+
for table in _CARGO_TABLES:
|
|
160
|
+
entries = group.get(table)
|
|
161
|
+
for name, value in entries.items() if isinstance(entries, dict) else ():
|
|
162
|
+
if isinstance(value, str):
|
|
163
|
+
pairs[(name, value)] = None
|
|
164
|
+
elif isinstance(value, dict) and not {"path", "git", "workspace"} & value.keys():
|
|
165
|
+
version = value.get("version")
|
|
166
|
+
pairs[(value.get("package") if isinstance(value.get("package"), str) else name,
|
|
167
|
+
version if isinstance(version, str) else "")] = None
|
|
168
|
+
return pairs
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _pom_pairs(text):
|
|
172
|
+
"""The (`groupId:artifactId`, version) pairs of every `dependency` element of a pom.xml text, dependency management
|
|
173
|
+
included. `${property}` versions take the value of the pom's `properties` and of `project.groupId|artifactId|version`;
|
|
174
|
+
one that stays unresolved is kept as written, so it reports as not verified.
|
|
175
|
+
|
|
176
|
+
Parameters: text (str) file content.
|
|
177
|
+
Returns: dict (name, version) -> None in document order ("" when no version is declared); None when the text is not
|
|
178
|
+
XML or declares entities.
|
|
179
|
+
"""
|
|
180
|
+
if "<!ENTITY" in text:
|
|
181
|
+
return None
|
|
182
|
+
try:
|
|
183
|
+
root = ElementTree.fromstring(text)
|
|
184
|
+
except ElementTree.ParseError:
|
|
185
|
+
return None
|
|
186
|
+
local = lambda element: element.tag.rpartition("}")[2] # noqa: E731
|
|
187
|
+
child = lambda element, tag: next((item.text or "" for item in element if local(item) == tag), "").strip() # noqa: E731
|
|
188
|
+
properties = {f"project.{tag}": child(root, tag) for tag in ("groupId", "artifactId", "version")}
|
|
189
|
+
properties.update({local(item): (item.text or "").strip()
|
|
190
|
+
for block in root if local(block) == "properties" for item in block})
|
|
191
|
+
resolve = lambda value: _PROPERTY_RE.sub(lambda match: properties.get(match.group(1)) or match.group(0), value) # noqa: E731
|
|
192
|
+
return {(resolve(f"{child(item, 'groupId')}:{child(item, 'artifactId')}"), resolve(child(item, "version"))): None
|
|
193
|
+
for item in root.iter() if local(item) == "dependency"}
|
|
194
|
+
|
|
195
|
+
|
|
105
196
|
def ecosystem_of(path):
|
|
106
|
-
"""The
|
|
197
|
+
"""The ecosystem key of a manifest file name; `_ECOSYSTEMS[key].osv` is the name OSV knows it by.
|
|
107
198
|
|
|
108
199
|
Parameters: path (str) file path.
|
|
109
|
-
Returns: "npm" for package.json, "PyPI" for requirements*.txt, "Go" for go.mod
|
|
200
|
+
Returns: "npm" for package.json, "PyPI" for requirements*.txt, "pyproject" for pyproject.toml, "Go" for go.mod, "Cargo" for
|
|
201
|
+
Cargo.toml, "Maven" for pom.xml; None for any other file.
|
|
110
202
|
"""
|
|
111
203
|
name = os.path.basename(path)
|
|
112
204
|
if name == "package.json":
|
|
113
205
|
return "npm"
|
|
114
206
|
if name == "go.mod":
|
|
115
207
|
return "Go"
|
|
208
|
+
if name == "pyproject.toml":
|
|
209
|
+
return "pyproject"
|
|
210
|
+
if name == "Cargo.toml":
|
|
211
|
+
return "Cargo"
|
|
212
|
+
if name == "pom.xml":
|
|
213
|
+
return "Maven"
|
|
116
214
|
return "PyPI" if _REQUIREMENTS_FILE_RE.fullmatch(name) else None
|
|
117
215
|
|
|
118
216
|
|
|
@@ -133,7 +231,7 @@ def changed_dependencies(before, after, ecosystem="npm"):
|
|
|
133
231
|
else:
|
|
134
232
|
old = parse(before)
|
|
135
233
|
if old is None:
|
|
136
|
-
raise DependencyCheckError("current
|
|
234
|
+
raise DependencyCheckError(f"current {_ECOSYSTEMS[ecosystem].file} is not valid {_ECOSYSTEMS[ecosystem].syntax}")
|
|
137
235
|
return [pair for pair in now if pair not in old]
|
|
138
236
|
|
|
139
237
|
|
|
@@ -169,6 +267,21 @@ def _go_version(specifier):
|
|
|
169
267
|
return specifier if _GO_VERSION_RE.fullmatch(specifier) else None
|
|
170
268
|
|
|
171
269
|
|
|
270
|
+
def _cargo_version(specifier):
|
|
271
|
+
"""The Cargo case of `queried_version`: the lower bound of a bare, `^`, `~`, `=`, `>=` or `>` requirement padded to
|
|
272
|
+
three parts; None for `*`, wildcards, `<`, compound requirements and anything else."""
|
|
273
|
+
match = _SPECIFIER_RE.fullmatch(specifier.strip())
|
|
274
|
+
if not match:
|
|
275
|
+
return None
|
|
276
|
+
_operator, major, minor, patch, tail = match.groups()
|
|
277
|
+
return f"{major}.{minor or 0}.{patch or 0}{tail or ''}"
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _maven_version(specifier):
|
|
281
|
+
"""The Maven case of `queried_version`: a plain version; None for ranges, unresolved properties and no version."""
|
|
282
|
+
return specifier if _MAVEN_VERSION_RE.fullmatch(specifier) else None
|
|
283
|
+
|
|
284
|
+
|
|
172
285
|
def queried_version(specifier, ecosystem="npm"):
|
|
173
286
|
"""The version to ask OSV about for a specifier.
|
|
174
287
|
|
|
@@ -178,12 +291,20 @@ def queried_version(specifier, ecosystem="npm"):
|
|
|
178
291
|
return _ECOSYSTEMS[ecosystem].version(specifier)
|
|
179
292
|
|
|
180
293
|
|
|
181
|
-
_Ecosystem = collections.namedtuple("_Ecosystem", "parse version label exact")
|
|
294
|
+
_Ecosystem = collections.namedtuple("_Ecosystem", "parse version label exact osv file syntax")
|
|
295
|
+
_PIP_LABEL = lambda name, spec: f"{name}{spec}" # noqa: E731
|
|
296
|
+
_PIP_EXACT = lambda spec, version: spec == "==" + version # noqa: E731
|
|
182
297
|
_ECOSYSTEMS = {
|
|
183
|
-
"npm": _Ecosystem(_npm_pairs, _npm_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec == version
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
"
|
|
298
|
+
"npm": _Ecosystem(_npm_pairs, _npm_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec == version,
|
|
299
|
+
"npm", "package.json", "JSON"),
|
|
300
|
+
"PyPI": _Ecosystem(_requirements_pairs, _pip_version, _PIP_LABEL, _PIP_EXACT, "PyPI", "requirements file", "text"),
|
|
301
|
+
"pyproject": _Ecosystem(_pyproject_pairs, _pip_version, _PIP_LABEL, _PIP_EXACT, "PyPI", "pyproject.toml", "TOML"),
|
|
302
|
+
"Go": _Ecosystem(_go_pairs, _go_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec == version,
|
|
303
|
+
"Go", "go.mod", "text"),
|
|
304
|
+
"Cargo": _Ecosystem(_cargo_pairs, _cargo_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec in (
|
|
305
|
+
version, "=" + version), "crates.io", "Cargo.toml", "TOML"),
|
|
306
|
+
"Maven": _Ecosystem(_pom_pairs, _maven_version, lambda name, spec: f"{name}@{spec}" if spec else name,
|
|
307
|
+
lambda spec, version: spec == version, "Maven", "pom.xml", "XML"),
|
|
187
308
|
}
|
|
188
309
|
ECOSYSTEMS = tuple(_ECOSYSTEMS)
|
|
189
310
|
|
|
@@ -215,7 +336,7 @@ def _fresh_ids(entry, now):
|
|
|
215
336
|
def _advisories(ecosystem, queries):
|
|
216
337
|
"""Advisory IDs for (name, version) pairs, from the cache and one OSV batch for the rest.
|
|
217
338
|
|
|
218
|
-
Parameters: ecosystem (str)
|
|
339
|
+
Parameters: ecosystem (str) the OSV ecosystem name (`_Ecosystem.osv`); queries (list of (name, version)).
|
|
219
340
|
Returns: tuple (list of IDs or None per query in order, str cache-write problem or "").
|
|
220
341
|
Raises: DependencyCheckError when the cache is corrupt or OSV cannot answer.
|
|
221
342
|
"""
|
|
@@ -226,7 +347,9 @@ def _advisories(ecosystem, queries):
|
|
|
226
347
|
if not missing:
|
|
227
348
|
return [found[query] for query in queries], ""
|
|
228
349
|
try:
|
|
229
|
-
answered =
|
|
350
|
+
answered = []
|
|
351
|
+
for start in range(0, len(missing), BATCH_SIZE):
|
|
352
|
+
answered += freshness.advisories_batch(ecosystem, missing[start:start + BATCH_SIZE], OSV_TIMEOUT_S)
|
|
230
353
|
except RuntimeError as exc:
|
|
231
354
|
raise DependencyCheckError(str(exc)) from exc
|
|
232
355
|
for query, ids in zip(missing, answered):
|
|
@@ -256,14 +379,14 @@ def notice(before, after, ecosystem="npm"):
|
|
|
256
379
|
handled, extra = changed[:MAX_PACKAGES], len(changed) - MAX_PACKAGES
|
|
257
380
|
lines, queries = {}, []
|
|
258
381
|
for name, specifier in handled:
|
|
259
|
-
version = spec.version(specifier) if freshness.NAME_RES[
|
|
382
|
+
version = spec.version(specifier) if freshness.NAME_RES[spec.osv].fullmatch(name) else None
|
|
260
383
|
if version is None:
|
|
261
384
|
lines[(name, specifier)] = f"`{spec.label(name, specifier)}` specifier not verified."
|
|
262
385
|
else:
|
|
263
386
|
queries.append((name, specifier, version))
|
|
264
387
|
problem = ""
|
|
265
388
|
if queries:
|
|
266
|
-
ids, problem = _advisories(
|
|
389
|
+
ids, problem = _advisories(spec.osv, [(name, version) for name, _specifier, version in queries])
|
|
267
390
|
for (name, specifier, version), advisories in zip(queries, ids):
|
|
268
391
|
label = f"`{spec.label(name, specifier)}`" + ("" if spec.exact(specifier, version) else f" (checked {version})")
|
|
269
392
|
if advisories is None:
|
|
@@ -278,3 +401,128 @@ def notice(before, after, ecosystem="npm"):
|
|
|
278
401
|
if problem and parts:
|
|
279
402
|
parts.append(f"({problem})")
|
|
280
403
|
return PREFIX + " ".join(parts) if parts else ""
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def manifests(root):
|
|
407
|
+
"""Project-relative paths of the dependency manifests under root (package.json, requirements*.txt, pyproject.toml,
|
|
408
|
+
go.mod, Cargo.toml, pom.xml), at most
|
|
409
|
+
AUDIT_MAX_MANIFESTS and AUDIT_MAX_DEPTH folders deep, skipping hidden and dependency folders."""
|
|
410
|
+
found = []
|
|
411
|
+
base_depth = os.path.abspath(root).count(os.sep)
|
|
412
|
+
for folder, dirs, files in os.walk(root):
|
|
413
|
+
dirs[:] = sorted(name for name in dirs if not name.startswith(".") and name not in AUDIT_SKIP_DIRS)
|
|
414
|
+
if folder.count(os.sep) - base_depth >= AUDIT_MAX_DEPTH:
|
|
415
|
+
dirs[:] = []
|
|
416
|
+
found += [os.path.relpath(os.path.join(folder, name), root).replace(os.sep, "/")
|
|
417
|
+
for name in sorted(files) if ecosystem_of(name)]
|
|
418
|
+
return found[:AUDIT_MAX_MANIFESTS]
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _locked_versions(folder):
|
|
422
|
+
"""The versions a package-lock.json next to a package.json resolved, by package name.
|
|
423
|
+
|
|
424
|
+
Parameters: folder (str) the folder of the package.json.
|
|
425
|
+
Returns: dict name -> version of the top-level installs (lockfile v2 and v3 `packages`, v1 `dependencies`); empty
|
|
426
|
+
when there is no lockfile or it cannot be read.
|
|
427
|
+
"""
|
|
428
|
+
try:
|
|
429
|
+
with open(os.path.join(folder, "package-lock.json"), encoding="utf-8") as stream:
|
|
430
|
+
data = json.load(stream)
|
|
431
|
+
except (OSError, UnicodeDecodeError, ValueError):
|
|
432
|
+
return {}
|
|
433
|
+
if not isinstance(data, dict):
|
|
434
|
+
return {}
|
|
435
|
+
found = {}
|
|
436
|
+
for name, entry in (data.get("dependencies") if isinstance(data.get("dependencies"), dict) else {}).items():
|
|
437
|
+
if isinstance(entry, dict) and isinstance(entry.get("version"), str):
|
|
438
|
+
found[name] = entry["version"]
|
|
439
|
+
prefix = "node_modules/"
|
|
440
|
+
for key, entry in (data.get("packages") if isinstance(data.get("packages"), dict) else {}).items():
|
|
441
|
+
if key.startswith(prefix) and "/node_modules/" not in key and isinstance(entry, dict) \
|
|
442
|
+
and isinstance(entry.get("version"), str):
|
|
443
|
+
found[key[len(prefix):]] = entry["version"]
|
|
444
|
+
return found
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def _locked_version(specifier, locked):
|
|
448
|
+
"""The lockfile version to ask about for an npm specifier, or None when the lock does not apply to it.
|
|
449
|
+
|
|
450
|
+
Parameters: specifier (str) the value in the dependency table; locked (str or None) the resolved version.
|
|
451
|
+
Returns: `locked` when it satisfies the specifier (an exact pin, `^` on the same major, `~` on the same minor, `>=` and
|
|
452
|
+
`>` at or above the bound) or the specifier is a dist-tag, `*` or an x-range; None for a stale lock, a lock that is not
|
|
453
|
+
a plain version and specifiers that point elsewhere (URLs, git, file:, workspace:, aliases, compound ranges).
|
|
454
|
+
"""
|
|
455
|
+
resolved = _LOCKED_RE.match(locked or "")
|
|
456
|
+
if not resolved:
|
|
457
|
+
return None
|
|
458
|
+
if _TAG_RE.fullmatch(specifier.strip()):
|
|
459
|
+
return locked
|
|
460
|
+
match = _SPECIFIER_RE.fullmatch(specifier.strip())
|
|
461
|
+
if not match:
|
|
462
|
+
return None
|
|
463
|
+
operator, major, minor, patch, _tail = match.groups()
|
|
464
|
+
have = tuple(int(part) for part in resolved.groups())
|
|
465
|
+
bound = tuple(int(part or 0) for part in (major, minor, patch))
|
|
466
|
+
if operator in (">=", ">"):
|
|
467
|
+
ok = have >= bound
|
|
468
|
+
elif operator == "^":
|
|
469
|
+
ok = have >= bound and have[0] == bound[0]
|
|
470
|
+
elif operator == "~":
|
|
471
|
+
ok = have >= bound and have[:2] == bound[:2]
|
|
472
|
+
else:
|
|
473
|
+
ok = None not in (minor, patch) and have == bound
|
|
474
|
+
return locked if ok else None
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def audit(root):
|
|
478
|
+
"""Advisories of every dependency the manifests of a project declare, not only the ones an edit changed.
|
|
479
|
+
|
|
480
|
+
Parameters: root (str) project folder.
|
|
481
|
+
Returns: dict {"checked": packages asked about, "manifests": [{"path", "ecosystem", "packages", "vulnerable":
|
|
482
|
+
[{"package", "specifier", "version", "advisories"}], "unverified": [str]}], "note": str}. A manifest that cannot be
|
|
483
|
+
read or parsed is listed with its reason in `unverified`. For a package.json with a package-lock.json beside it, the
|
|
484
|
+
version asked about is the locked one when it satisfies the range (reported in `version`). One OSV `querybatch` per
|
|
485
|
+
BATCH_SIZE packages and ecosystem; the 6-hour advisory cache is shared with the edit notice.
|
|
486
|
+
Raises: DependencyCheckError when OSV or the cache cannot answer.
|
|
487
|
+
"""
|
|
488
|
+
report, plans, queries = [], [], {}
|
|
489
|
+
for rel in manifests(root):
|
|
490
|
+
spec = _ECOSYSTEMS[ecosystem_of(rel)]
|
|
491
|
+
ecosystem = spec.osv
|
|
492
|
+
entry = {"path": rel, "ecosystem": ecosystem, "packages": 0, "vulnerable": [], "unverified": []}
|
|
493
|
+
report.append(entry)
|
|
494
|
+
try:
|
|
495
|
+
with open(os.path.join(root, rel), encoding="utf-8") as stream:
|
|
496
|
+
pairs = spec.parse(stream.read())
|
|
497
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
498
|
+
entry["unverified"].append(f"manifest not read: {type(exc).__name__}")
|
|
499
|
+
continue
|
|
500
|
+
if pairs is None:
|
|
501
|
+
entry["unverified"].append(f"manifest is not valid {spec.syntax}")
|
|
502
|
+
continue
|
|
503
|
+
entry["packages"] = len(pairs)
|
|
504
|
+
locks = _locked_versions(os.path.dirname(os.path.join(root, rel))) if ecosystem == "npm" else {}
|
|
505
|
+
for name, specifier in pairs:
|
|
506
|
+
version = None
|
|
507
|
+
if freshness.NAME_RES[ecosystem].fullmatch(name):
|
|
508
|
+
version = _locked_version(specifier, locks.get(name)) or spec.version(specifier)
|
|
509
|
+
if version is None:
|
|
510
|
+
entry["unverified"].append(f"{spec.label(name, specifier)}: specifier not verified")
|
|
511
|
+
else:
|
|
512
|
+
plans.append((entry, ecosystem, name, specifier, version, spec.label))
|
|
513
|
+
queries.setdefault(ecosystem, {})[(name, version)] = None
|
|
514
|
+
answers, problem = {}, ""
|
|
515
|
+
for ecosystem, wanted in queries.items():
|
|
516
|
+
keys = list(wanted)
|
|
517
|
+
ids, problem = _advisories(ecosystem, keys)
|
|
518
|
+
answers[ecosystem] = dict(zip(keys, ids))
|
|
519
|
+
for entry, ecosystem, name, specifier, version, describe in plans:
|
|
520
|
+
advisories = answers[ecosystem][(name, version)]
|
|
521
|
+
label = describe(name, specifier)
|
|
522
|
+
if advisories is None:
|
|
523
|
+
entry["unverified"].append(f"{label}: advisory check incomplete")
|
|
524
|
+
elif advisories:
|
|
525
|
+
entry["vulnerable"].append({"package": name, "specifier": specifier, "version": version,
|
|
526
|
+
"advisories": advisories})
|
|
527
|
+
total = sum(len(wanted) for wanted in queries.values())
|
|
528
|
+
return {"checked": total, "manifests": report, "note": problem}
|