@allansantos-dev/smart-tool 0.9.79 → 0.9.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/config.py CHANGED
@@ -40,12 +40,15 @@ DEFAULT_CONFIG = {
40
40
  "research_max_iterations": 3,
41
41
  "quick_search_timeout_s": 120,
42
42
  "site_memory_similarity_threshold": 0.35,
43
- "local_models": "fallback",
43
+ "embedder": "home",
44
44
  "hook_mode": "redirect",
45
45
  "doc_mode": "require",
46
46
  "comment_mode": "require",
47
47
  "duplicate_mode": "warn",
48
48
  "dependency_mode": "warn",
49
+ "layer_mode": "warn",
50
+ "document_converter": "off",
51
+ "document_converter_python": "",
49
52
  "docs_extras": {key: True for key in DOCS_EXTRAS},
50
53
  "auto_update": True,
51
54
  "language": "",
@@ -60,9 +63,10 @@ DEFAULT_CONFIG = {
60
63
  },
61
64
  }
62
65
 
63
- # Local models from the mcp-memory sidecar: fallback = only when the gateway has no model; off = never;
64
- # prefer = even when the gateway has one (embedding quality measured equal on the rerank gold set).
65
- LOCAL_MODEL_MODES = ("fallback", "off", "prefer")
66
+ # Embedder of the project indexes: home = the embeddings home daemon of this machine (casa-de-embeddings), whose rerank
67
+ # connector forwards to the host configured in it; gateway = the gateway's embedding_model and rerank_model, only when
68
+ # the user chooses it. Neither replaces the other when it fails.
69
+ EMBEDDERS = ("home", "gateway")
66
70
  # PreToolUse hook: redirect = deny the native tool and point to Smart Tool; advise = let it run and add the tip to the
67
71
  # agent's context; off = no routing at all.
68
72
  HOOK_MODES = ("redirect", "advise", "off")
@@ -79,6 +83,11 @@ DUPLICATE_MODES = ("warn", "off")
79
83
  # Edit hook on package.json, requirements*.txt and go.mod: warn = let the edit run and list the added or changed dependencies that have OSV advisories;
80
84
  # off = say nothing and make no network call.
81
85
  DEPENDENCY_MODES = ("warn", "off")
86
+ LAYER_MODES = ("warn", "off")
87
+ # External converter read_document falls back to for a file type or a scanned PDF its native reader refuses: off = never;
88
+ # auto = the first of kreuzberg and docling found installed; or one of them by name. The engine is looked for in the
89
+ # interpreter named by `document_converter_python` (empty = the one Smart Tool runs on).
90
+ DOCUMENT_CONVERTERS = ("off", "auto", "kreuzberg", "docling")
82
91
  # Router model reviewing, in the background, searches the deterministic redirect rule let run: on = propose block
83
92
  # patterns for the user to accept in the setup screen; off = no review.
84
93
  PATTERN_PROPOSAL_MODES = ("on", "off")
@@ -118,6 +127,26 @@ def dependency_mode(cfg=None):
118
127
  return value
119
128
 
120
129
 
130
+ def layer_mode(cfg=None):
131
+ value = (cfg if cfg is not None else load_config()).get("layer_mode") or DEFAULT_CONFIG["layer_mode"]
132
+ if value not in LAYER_MODES:
133
+ raise ValueError(f"Invalid layer_mode ({value!r}) in {CONFIG_PATH}: use warn or off.")
134
+ return value
135
+
136
+
137
+ def document_converter(cfg=None):
138
+ """The external converter setting: `{"mode": one of DOCUMENT_CONVERTERS, "python": interpreter path or ""}`. An
139
+ unknown mode or a python that is not text fails loudly."""
140
+ stored = cfg if cfg is not None else load_config()
141
+ mode = stored.get("document_converter") or DEFAULT_CONFIG["document_converter"]
142
+ python = stored.get("document_converter_python") or ""
143
+ if mode not in DOCUMENT_CONVERTERS:
144
+ raise ValueError(f"Invalid document_converter ({mode!r}) in {CONFIG_PATH}: use off, auto, kreuzberg or docling.")
145
+ if not isinstance(python, str):
146
+ raise ValueError(f"Invalid document_converter_python in {CONFIG_PATH}: a path to a Python interpreter.")
147
+ return {"mode": mode, "python": python}
148
+
149
+
121
150
  def docs_extras(cfg=None):
122
151
  """Which extra documents the docs generator writes: every key of DOCS_EXTRAS, true unless the stored `docs_extras`
123
152
  object sets it to false; an unknown key or a value that is not true or false fails loudly."""
@@ -183,10 +212,14 @@ def hook_mode(cfg=None):
183
212
  return value
184
213
 
185
214
 
186
- def local_models_mode(cfg=None):
187
- value = (cfg if cfg is not None else load_config()).get("local_models") or DEFAULT_CONFIG["local_models"]
188
- if value not in LOCAL_MODEL_MODES:
189
- raise ValueError(f"Invalid local_models ({value!r}) in {CONFIG_PATH}: use fallback, off or prefer.")
215
+ def embedder(cfg=None):
216
+ cfg = cfg if cfg is not None else load_config()
217
+ if "local_models" in cfg:
218
+ raise ValueError(f"{CONFIG_PATH} has the old 'local_models' setting: delete it and choose the embedder (home or "
219
+ "gateway) in the setup screen.")
220
+ value = cfg.get("embedder") or DEFAULT_CONFIG["embedder"]
221
+ if value not in EMBEDDERS:
222
+ raise ValueError(f"Invalid embedder ({value!r}) in {CONFIG_PATH}: use home or gateway.")
190
223
  return value
191
224
 
192
225
 
@@ -0,0 +1,38 @@
1
+ """Credentials an edit writes, for the edit hook. Only the lines an edit adds or changes are read, and only for shapes that
2
+ cannot be anything else (a provider's token prefix with its fixed length, an AWS key pair line, a private-key header at
3
+ the start of a line): a false alarm costs more than a silence here, so a general "high entropy string" rule is not used.
4
+ Lines that say they are an example are skipped. The matched text is never repeated in a note."""
5
+ import re
6
+
7
+ import doc_check
8
+
9
+ _PLACEHOLDER = re.compile(r"example|placeholder|your[_-]|dummy|fake|redacted|x{6,}|\*{4,}|<[^>]*>", re.I)
10
+ _PATTERNS = (
11
+ ("private key", re.compile(r"^\s*-----BEGIN (?:[A-Z]+ )*PRIVATE KEY(?: BLOCK)?-----")),
12
+ ("AWS access key id", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b")),
13
+ ("AWS secret access key", re.compile(r"aws_secret_access_key[\"']?\s*[=:]\s*[\"']?[A-Za-z0-9/+=]{40}\b", re.I)),
14
+ ("GitHub token", re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{36,}|github_pat_[A-Za-z0-9_]{50,})")),
15
+ ("Slack token", re.compile(r"\bxox[abprs]-[A-Za-z0-9-]{10,}")),
16
+ ("Google API key", re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b")),
17
+ ("Stripe live key", re.compile(r"\b[sr]k_live_[0-9A-Za-z]{24,}")),
18
+ ("Anthropic API key", re.compile(r"\bsk-ant-[A-Za-z0-9_-]{20,}")),
19
+ ("npm token", re.compile(r"\bnpm_[A-Za-z0-9]{36}\b")),
20
+ )
21
+
22
+
23
+ def review(before, after):
24
+ """Credentials among the lines an edit adds or changes.
25
+
26
+ Parameters: before (str or None) text before the edit; after (str) text after it.
27
+ Returns: list of dicts with line (1-based) and kind, in line order, one per line; the secret itself is not included.
28
+ """
29
+ lines = after.split("\n")
30
+ findings = []
31
+ for number in sorted(doc_check._touched_lines(before, after)):
32
+ if number > len(lines) or _PLACEHOLDER.search(lines[number - 1]):
33
+ continue
34
+ for kind, pattern in _PATTERNS:
35
+ if pattern.search(lines[number - 1]):
36
+ findings.append({"line": number, "kind": kind})
37
+ break
38
+ return findings
@@ -1,7 +1,11 @@
1
1
  """Vulnerable-dependency notice for an edit of a dependency manifest, built before the edit runs.
2
2
 
3
- Manifests: package.json (OSV ecosystem npm), requirements*.txt (PyPI: `==` pins and the lower bound of `>=` and `~=`)
4
- and go.mod (Go: `require` lines with a plain vX.Y.Z). Compares the dependencies of the file before and after the edit,
3
+ Manifests: package.json (OSV ecosystem npm), requirements*.txt and pyproject.toml (PyPI: `==` pins and the lower bound
4
+ of `>=` and `~=`; pyproject reads `project.dependencies` and `project.optional-dependencies`, not Poetry tables),
5
+ go.mod (Go: `require` lines with a plain vX.Y.Z), Cargo.toml (crates.io: registry dependencies in every dependency table,
6
+ asked at the lower bound of the requirement) and pom.xml (Maven: `groupId:artifactId` with a plain or property-resolved
7
+ version, no ranges and no version inherited from a parent or a BOM). The audit asks about the version a sibling package-lock.json resolved
8
+ when it satisfies the range, else about the lower bound. Compares the dependencies of the file before and after the edit,
5
9
  asks OSV (one `querybatch` call, short timeout) about the added or changed ones only, and answers from a disk cache of
6
10
  (ecosystem, package, version) -> advisory IDs for 6 hours. Only package names and versions leave the machine. The caller (`hook_decision`) turns a `DependencyCheckError` into
7
11
  "dependency check skipped: <reason>" and lets the edit go on.
@@ -11,6 +15,8 @@ import json
11
15
  import os
12
16
  import re
13
17
  import time
18
+ import tomllib
19
+ import xml.etree.ElementTree as ElementTree
14
20
 
15
21
  import atomic_io
16
22
  import freshness
@@ -20,6 +26,10 @@ CACHE_PATH = os.path.join(paths.DATA_DIR, "dependency_advisories.json")
20
26
  CACHE_TTL_S = 6 * 3600
21
27
  OSV_TIMEOUT_S = 2.5
22
28
  MAX_PACKAGES = 5
29
+ BATCH_SIZE = 100
30
+ AUDIT_MAX_MANIFESTS = 50
31
+ AUDIT_MAX_DEPTH = 4
32
+ AUDIT_SKIP_DIRS = frozenset(("node_modules", "vendor", "dist", "build", "target", "__pycache__", "venv", "site-packages"))
23
33
  MAX_IDS_SHOWN = 5
24
34
  TABLES = ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies")
25
35
  PREFIX = "Smart Tool dependencies: "
@@ -29,6 +39,11 @@ _REQUIREMENTS_FILE_RE = re.compile(r"requirements[A-Za-z0-9._-]*\.txt")
29
39
  _REQUIREMENT_RE = re.compile(r"([A-Za-z0-9][A-Za-z0-9._-]*)\s*(?:\[[^\]]*\])?\s*([^;]*)")
30
40
  _PIP_VERSION_RE = re.compile(r"[0-9][0-9A-Za-z.+!_-]*")
31
41
  _GO_VERSION_RE = re.compile(r"v\d+\.\d+\.\d+")
42
+ _LOCKED_RE = re.compile(r"(\d+)\.(\d+)\.(\d+)")
43
+ _MAVEN_VERSION_RE = re.compile(r"[0-9][0-9A-Za-z.+_-]*")
44
+ _PROPERTY_RE = re.compile(r"\$\{([^}]+)\}")
45
+ _CARGO_TABLES = ("dependencies", "dev-dependencies", "build-dependencies")
46
+ _TAG_RE = re.compile(r"\*|[xX]|[A-Za-z][A-Za-z0-9._-]*")
32
47
 
33
48
 
34
49
  class DependencyCheckError(RuntimeError):
@@ -102,17 +117,100 @@ def _go_pairs(text):
102
117
  return pairs
103
118
 
104
119
 
120
+ def _pyproject_pairs(text):
121
+ """The (name, specifier) pairs of `project.dependencies` and `project.optional-dependencies` in a pyproject.toml text.
122
+
123
+ Parameters: text (str) file content.
124
+ Returns: dict (name, specifier) -> None in file order, read as requirement lines; None when the text is not TOML.
125
+ """
126
+ try:
127
+ project = tomllib.loads(text).get("project")
128
+ except tomllib.TOMLDecodeError:
129
+ return None
130
+ if not isinstance(project, dict):
131
+ return {}
132
+ lines = [item for item in project.get("dependencies") or [] if isinstance(item, str)]
133
+ optional = project.get("optional-dependencies")
134
+ for group in optional.values() if isinstance(optional, dict) else ():
135
+ lines += [item for item in group if isinstance(item, str)]
136
+ return _requirements_pairs("\n".join(lines))
137
+
138
+
139
+ def _cargo_pairs(text):
140
+ """The (crate, requirement) pairs of the registry dependencies of a Cargo.toml text: the dependency, dev-dependency and
141
+ build-dependency tables, the `target.*` tables and `workspace.dependencies`. Path, git and `workspace = true` entries
142
+ have no registry version and are skipped; a `package` key renames the crate.
143
+
144
+ Parameters: text (str) file content.
145
+ Returns: dict (crate, requirement) -> None in file order ("" when a registry entry names no version); None when the
146
+ text is not TOML.
147
+ """
148
+ try:
149
+ data = tomllib.loads(text)
150
+ except tomllib.TOMLDecodeError:
151
+ return None
152
+ groups = [data]
153
+ targets = data.get("target")
154
+ groups += [group for group in targets.values() if isinstance(group, dict)] if isinstance(targets, dict) else []
155
+ workspace = data.get("workspace")
156
+ groups += [workspace] if isinstance(workspace, dict) else []
157
+ pairs = {}
158
+ for group in groups:
159
+ for table in _CARGO_TABLES:
160
+ entries = group.get(table)
161
+ for name, value in entries.items() if isinstance(entries, dict) else ():
162
+ if isinstance(value, str):
163
+ pairs[(name, value)] = None
164
+ elif isinstance(value, dict) and not {"path", "git", "workspace"} & value.keys():
165
+ version = value.get("version")
166
+ pairs[(value.get("package") if isinstance(value.get("package"), str) else name,
167
+ version if isinstance(version, str) else "")] = None
168
+ return pairs
169
+
170
+
171
+ def _pom_pairs(text):
172
+ """The (`groupId:artifactId`, version) pairs of every `dependency` element of a pom.xml text, dependency management
173
+ included. `${property}` versions take the value of the pom's `properties` and of `project.groupId|artifactId|version`;
174
+ one that stays unresolved is kept as written, so it reports as not verified.
175
+
176
+ Parameters: text (str) file content.
177
+ Returns: dict (name, version) -> None in document order ("" when no version is declared); None when the text is not
178
+ XML or declares entities.
179
+ """
180
+ if "<!ENTITY" in text:
181
+ return None
182
+ try:
183
+ root = ElementTree.fromstring(text)
184
+ except ElementTree.ParseError:
185
+ return None
186
+ local = lambda element: element.tag.rpartition("}")[2] # noqa: E731
187
+ child = lambda element, tag: next((item.text or "" for item in element if local(item) == tag), "").strip() # noqa: E731
188
+ properties = {f"project.{tag}": child(root, tag) for tag in ("groupId", "artifactId", "version")}
189
+ properties.update({local(item): (item.text or "").strip()
190
+ for block in root if local(block) == "properties" for item in block})
191
+ resolve = lambda value: _PROPERTY_RE.sub(lambda match: properties.get(match.group(1)) or match.group(0), value) # noqa: E731
192
+ return {(resolve(f"{child(item, 'groupId')}:{child(item, 'artifactId')}"), resolve(child(item, "version"))): None
193
+ for item in root.iter() if local(item) == "dependency"}
194
+
195
+
105
196
  def ecosystem_of(path):
106
- """The OSV ecosystem of a manifest file name.
197
+ """The ecosystem key of a manifest file name; `_ECOSYSTEMS[key].osv` is the name OSV knows it by.
107
198
 
108
199
  Parameters: path (str) file path.
109
- Returns: "npm" for package.json, "PyPI" for requirements*.txt, "Go" for go.mod; None for any other file.
200
+ Returns: "npm" for package.json, "PyPI" for requirements*.txt, "pyproject" for pyproject.toml, "Go" for go.mod, "Cargo" for
201
+ Cargo.toml, "Maven" for pom.xml; None for any other file.
110
202
  """
111
203
  name = os.path.basename(path)
112
204
  if name == "package.json":
113
205
  return "npm"
114
206
  if name == "go.mod":
115
207
  return "Go"
208
+ if name == "pyproject.toml":
209
+ return "pyproject"
210
+ if name == "Cargo.toml":
211
+ return "Cargo"
212
+ if name == "pom.xml":
213
+ return "Maven"
116
214
  return "PyPI" if _REQUIREMENTS_FILE_RE.fullmatch(name) else None
117
215
 
118
216
 
@@ -133,7 +231,7 @@ def changed_dependencies(before, after, ecosystem="npm"):
133
231
  else:
134
232
  old = parse(before)
135
233
  if old is None:
136
- raise DependencyCheckError("current package.json is not valid JSON")
234
+ raise DependencyCheckError(f"current {_ECOSYSTEMS[ecosystem].file} is not valid {_ECOSYSTEMS[ecosystem].syntax}")
137
235
  return [pair for pair in now if pair not in old]
138
236
 
139
237
 
@@ -169,6 +267,21 @@ def _go_version(specifier):
169
267
  return specifier if _GO_VERSION_RE.fullmatch(specifier) else None
170
268
 
171
269
 
270
+ def _cargo_version(specifier):
271
+ """The Cargo case of `queried_version`: the lower bound of a bare, `^`, `~`, `=`, `>=` or `>` requirement padded to
272
+ three parts; None for `*`, wildcards, `<`, compound requirements and anything else."""
273
+ match = _SPECIFIER_RE.fullmatch(specifier.strip())
274
+ if not match:
275
+ return None
276
+ _operator, major, minor, patch, tail = match.groups()
277
+ return f"{major}.{minor or 0}.{patch or 0}{tail or ''}"
278
+
279
+
280
+ def _maven_version(specifier):
281
+ """The Maven case of `queried_version`: a plain version; None for ranges, unresolved properties and no version."""
282
+ return specifier if _MAVEN_VERSION_RE.fullmatch(specifier) else None
283
+
284
+
172
285
  def queried_version(specifier, ecosystem="npm"):
173
286
  """The version to ask OSV about for a specifier.
174
287
 
@@ -178,12 +291,20 @@ def queried_version(specifier, ecosystem="npm"):
178
291
  return _ECOSYSTEMS[ecosystem].version(specifier)
179
292
 
180
293
 
181
- _Ecosystem = collections.namedtuple("_Ecosystem", "parse version label exact")
294
+ _Ecosystem = collections.namedtuple("_Ecosystem", "parse version label exact osv file syntax")
295
+ _PIP_LABEL = lambda name, spec: f"{name}{spec}" # noqa: E731
296
+ _PIP_EXACT = lambda spec, version: spec == "==" + version # noqa: E731
182
297
  _ECOSYSTEMS = {
183
- "npm": _Ecosystem(_npm_pairs, _npm_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec == version),
184
- "PyPI": _Ecosystem(_requirements_pairs, _pip_version, lambda name, spec: f"{name}{spec}",
185
- lambda spec, version: spec == "==" + version),
186
- "Go": _Ecosystem(_go_pairs, _go_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec == version),
298
+ "npm": _Ecosystem(_npm_pairs, _npm_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec == version,
299
+ "npm", "package.json", "JSON"),
300
+ "PyPI": _Ecosystem(_requirements_pairs, _pip_version, _PIP_LABEL, _PIP_EXACT, "PyPI", "requirements file", "text"),
301
+ "pyproject": _Ecosystem(_pyproject_pairs, _pip_version, _PIP_LABEL, _PIP_EXACT, "PyPI", "pyproject.toml", "TOML"),
302
+ "Go": _Ecosystem(_go_pairs, _go_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec == version,
303
+ "Go", "go.mod", "text"),
304
+ "Cargo": _Ecosystem(_cargo_pairs, _cargo_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec in (
305
+ version, "=" + version), "crates.io", "Cargo.toml", "TOML"),
306
+ "Maven": _Ecosystem(_pom_pairs, _maven_version, lambda name, spec: f"{name}@{spec}" if spec else name,
307
+ lambda spec, version: spec == version, "Maven", "pom.xml", "XML"),
187
308
  }
188
309
  ECOSYSTEMS = tuple(_ECOSYSTEMS)
189
310
 
@@ -215,7 +336,7 @@ def _fresh_ids(entry, now):
215
336
  def _advisories(ecosystem, queries):
216
337
  """Advisory IDs for (name, version) pairs, from the cache and one OSV batch for the rest.
217
338
 
218
- Parameters: ecosystem (str) one of ECOSYSTEMS; queries (list of (name, version)).
339
+ Parameters: ecosystem (str) the OSV ecosystem name (`_Ecosystem.osv`); queries (list of (name, version)).
219
340
  Returns: tuple (list of IDs or None per query in order, str cache-write problem or "").
220
341
  Raises: DependencyCheckError when the cache is corrupt or OSV cannot answer.
221
342
  """
@@ -226,7 +347,9 @@ def _advisories(ecosystem, queries):
226
347
  if not missing:
227
348
  return [found[query] for query in queries], ""
228
349
  try:
229
- answered = freshness.advisories_batch(ecosystem, missing, OSV_TIMEOUT_S)
350
+ answered = []
351
+ for start in range(0, len(missing), BATCH_SIZE):
352
+ answered += freshness.advisories_batch(ecosystem, missing[start:start + BATCH_SIZE], OSV_TIMEOUT_S)
230
353
  except RuntimeError as exc:
231
354
  raise DependencyCheckError(str(exc)) from exc
232
355
  for query, ids in zip(missing, answered):
@@ -256,14 +379,14 @@ def notice(before, after, ecosystem="npm"):
256
379
  handled, extra = changed[:MAX_PACKAGES], len(changed) - MAX_PACKAGES
257
380
  lines, queries = {}, []
258
381
  for name, specifier in handled:
259
- version = spec.version(specifier) if freshness.NAME_RES[ecosystem].fullmatch(name) else None
382
+ version = spec.version(specifier) if freshness.NAME_RES[spec.osv].fullmatch(name) else None
260
383
  if version is None:
261
384
  lines[(name, specifier)] = f"`{spec.label(name, specifier)}` specifier not verified."
262
385
  else:
263
386
  queries.append((name, specifier, version))
264
387
  problem = ""
265
388
  if queries:
266
- ids, problem = _advisories(ecosystem, [(name, version) for name, _specifier, version in queries])
389
+ ids, problem = _advisories(spec.osv, [(name, version) for name, _specifier, version in queries])
267
390
  for (name, specifier, version), advisories in zip(queries, ids):
268
391
  label = f"`{spec.label(name, specifier)}`" + ("" if spec.exact(specifier, version) else f" (checked {version})")
269
392
  if advisories is None:
@@ -278,3 +401,128 @@ def notice(before, after, ecosystem="npm"):
278
401
  if problem and parts:
279
402
  parts.append(f"({problem})")
280
403
  return PREFIX + " ".join(parts) if parts else ""
404
+
405
+
406
+ def manifests(root):
407
+ """Project-relative paths of the dependency manifests under root (package.json, requirements*.txt, pyproject.toml,
408
+ go.mod, Cargo.toml, pom.xml), at most
409
+ AUDIT_MAX_MANIFESTS and AUDIT_MAX_DEPTH folders deep, skipping hidden and dependency folders."""
410
+ found = []
411
+ base_depth = os.path.abspath(root).count(os.sep)
412
+ for folder, dirs, files in os.walk(root):
413
+ dirs[:] = sorted(name for name in dirs if not name.startswith(".") and name not in AUDIT_SKIP_DIRS)
414
+ if folder.count(os.sep) - base_depth >= AUDIT_MAX_DEPTH:
415
+ dirs[:] = []
416
+ found += [os.path.relpath(os.path.join(folder, name), root).replace(os.sep, "/")
417
+ for name in sorted(files) if ecosystem_of(name)]
418
+ return found[:AUDIT_MAX_MANIFESTS]
419
+
420
+
421
+ def _locked_versions(folder):
422
+ """The versions a package-lock.json next to a package.json resolved, by package name.
423
+
424
+ Parameters: folder (str) the folder of the package.json.
425
+ Returns: dict name -> version of the top-level installs (lockfile v2 and v3 `packages`, v1 `dependencies`); empty
426
+ when there is no lockfile or it cannot be read.
427
+ """
428
+ try:
429
+ with open(os.path.join(folder, "package-lock.json"), encoding="utf-8") as stream:
430
+ data = json.load(stream)
431
+ except (OSError, UnicodeDecodeError, ValueError):
432
+ return {}
433
+ if not isinstance(data, dict):
434
+ return {}
435
+ found = {}
436
+ for name, entry in (data.get("dependencies") if isinstance(data.get("dependencies"), dict) else {}).items():
437
+ if isinstance(entry, dict) and isinstance(entry.get("version"), str):
438
+ found[name] = entry["version"]
439
+ prefix = "node_modules/"
440
+ for key, entry in (data.get("packages") if isinstance(data.get("packages"), dict) else {}).items():
441
+ if key.startswith(prefix) and "/node_modules/" not in key and isinstance(entry, dict) \
442
+ and isinstance(entry.get("version"), str):
443
+ found[key[len(prefix):]] = entry["version"]
444
+ return found
445
+
446
+
447
+ def _locked_version(specifier, locked):
448
+ """The lockfile version to ask about for an npm specifier, or None when the lock does not apply to it.
449
+
450
+ Parameters: specifier (str) the value in the dependency table; locked (str or None) the resolved version.
451
+ Returns: `locked` when it satisfies the specifier (an exact pin, `^` on the same major, `~` on the same minor, `>=` and
452
+ `>` at or above the bound) or the specifier is a dist-tag, `*` or an x-range; None for a stale lock, a lock that is not
453
+ a plain version and specifiers that point elsewhere (URLs, git, file:, workspace:, aliases, compound ranges).
454
+ """
455
+ resolved = _LOCKED_RE.match(locked or "")
456
+ if not resolved:
457
+ return None
458
+ if _TAG_RE.fullmatch(specifier.strip()):
459
+ return locked
460
+ match = _SPECIFIER_RE.fullmatch(specifier.strip())
461
+ if not match:
462
+ return None
463
+ operator, major, minor, patch, _tail = match.groups()
464
+ have = tuple(int(part) for part in resolved.groups())
465
+ bound = tuple(int(part or 0) for part in (major, minor, patch))
466
+ if operator in (">=", ">"):
467
+ ok = have >= bound
468
+ elif operator == "^":
469
+ ok = have >= bound and have[0] == bound[0]
470
+ elif operator == "~":
471
+ ok = have >= bound and have[:2] == bound[:2]
472
+ else:
473
+ ok = None not in (minor, patch) and have == bound
474
+ return locked if ok else None
475
+
476
+
477
+ def audit(root):
478
+ """Advisories of every dependency the manifests of a project declare, not only the ones an edit changed.
479
+
480
+ Parameters: root (str) project folder.
481
+ Returns: dict {"checked": packages asked about, "manifests": [{"path", "ecosystem", "packages", "vulnerable":
482
+ [{"package", "specifier", "version", "advisories"}], "unverified": [str]}], "note": str}. A manifest that cannot be
483
+ read or parsed is listed with its reason in `unverified`. For a package.json with a package-lock.json beside it, the
484
+ version asked about is the locked one when it satisfies the range (reported in `version`). One OSV `querybatch` per
485
+ BATCH_SIZE packages and ecosystem; the 6-hour advisory cache is shared with the edit notice.
486
+ Raises: DependencyCheckError when OSV or the cache cannot answer.
487
+ """
488
+ report, plans, queries = [], [], {}
489
+ for rel in manifests(root):
490
+ spec = _ECOSYSTEMS[ecosystem_of(rel)]
491
+ ecosystem = spec.osv
492
+ entry = {"path": rel, "ecosystem": ecosystem, "packages": 0, "vulnerable": [], "unverified": []}
493
+ report.append(entry)
494
+ try:
495
+ with open(os.path.join(root, rel), encoding="utf-8") as stream:
496
+ pairs = spec.parse(stream.read())
497
+ except (OSError, UnicodeDecodeError) as exc:
498
+ entry["unverified"].append(f"manifest not read: {type(exc).__name__}")
499
+ continue
500
+ if pairs is None:
501
+ entry["unverified"].append(f"manifest is not valid {spec.syntax}")
502
+ continue
503
+ entry["packages"] = len(pairs)
504
+ locks = _locked_versions(os.path.dirname(os.path.join(root, rel))) if ecosystem == "npm" else {}
505
+ for name, specifier in pairs:
506
+ version = None
507
+ if freshness.NAME_RES[ecosystem].fullmatch(name):
508
+ version = _locked_version(specifier, locks.get(name)) or spec.version(specifier)
509
+ if version is None:
510
+ entry["unverified"].append(f"{spec.label(name, specifier)}: specifier not verified")
511
+ else:
512
+ plans.append((entry, ecosystem, name, specifier, version, spec.label))
513
+ queries.setdefault(ecosystem, {})[(name, version)] = None
514
+ answers, problem = {}, ""
515
+ for ecosystem, wanted in queries.items():
516
+ keys = list(wanted)
517
+ ids, problem = _advisories(ecosystem, keys)
518
+ answers[ecosystem] = dict(zip(keys, ids))
519
+ for entry, ecosystem, name, specifier, version, describe in plans:
520
+ advisories = answers[ecosystem][(name, version)]
521
+ label = describe(name, specifier)
522
+ if advisories is None:
523
+ entry["unverified"].append(f"{label}: advisory check incomplete")
524
+ elif advisories:
525
+ entry["vulnerable"].append({"package": name, "specifier": specifier, "version": version,
526
+ "advisories": advisories})
527
+ total = sum(len(wanted) for wanted in queries.values())
528
+ return {"checked": total, "manifests": report, "note": problem}