@allansantos-dev/smart-tool 0.9.79 → 0.9.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +132 -0
- package/README.md +82 -18
- package/browser_control.py +10 -3
- package/browser_service_client.py +3 -32
- package/chromium_flags.py +30 -0
- package/client_hooks.py +10 -8
- package/code_health.py +182 -0
- package/code_impact.py +14 -2
- package/config.py +76 -8
- package/credential_check.py +38 -0
- package/dependency_notice.py +262 -14
- package/document_converter.py +221 -0
- package/document_read.py +100 -25
- package/document_text.py +194 -6
- package/dossier.py +71 -9
- package/embedding_cache.py +32 -8
- package/freshness.py +3 -2
- package/hook_decision.py +289 -18
- package/hooks/pretool_router.py +5 -4
- package/indexer.py +92 -3
- package/install.py +66 -27
- package/install_runtime.py +24 -10
- package/layer_check.py +57 -0
- package/link_check.py +69 -0
- package/local_embedder.py +115 -58
- package/locales/pt-BR.json +122 -18
- package/low_priority.py +41 -0
- package/model_client.py +1 -1
- package/model_health.py +24 -6
- package/notice_action.py +9 -0
- package/package.json +1 -1
- package/pr_comment.py +94 -0
- package/pr_impact.py +292 -8
- package/process_limits.py +108 -0
- package/project_admission.py +17 -15
- package/project_monitor.py +33 -7
- package/project_scheduler.py +12 -4
- package/removed_callers.py +146 -0
- package/scout.py +6 -2
- package/setup_audit.py +34 -1
- package/setup_ui.py +241 -41
- package/smart_tool_daemon.py +403 -91
- package/tray.py +29 -9
- package/update_check.py +18 -0
- package/update_splash.py +2 -2
- package/usage_meter.py +78 -0
- package/version.py +1 -1
- package/web_adapters/browser/crawl4ai_search.py +7 -3
- package/web_document_graph.py +12 -1
- package/web_fetch.py +24 -0
- package/web_search_health.py +31 -0
package/dependency_notice.py
CHANGED
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
"""Vulnerable-dependency notice for an edit of a dependency manifest, built before the edit runs.
|
|
2
2
|
|
|
3
|
-
Manifests: package.json (OSV ecosystem npm), requirements*.txt (PyPI: `==` pins and the lower bound
|
|
4
|
-
and
|
|
3
|
+
Manifests: package.json (OSV ecosystem npm), requirements*.txt and pyproject.toml (PyPI: `==` pins and the lower bound
|
|
4
|
+
of `>=` and `~=`; pyproject reads `project.dependencies` and `project.optional-dependencies`, not Poetry tables),
|
|
5
|
+
go.mod (Go: `require` lines with a plain vX.Y.Z), Cargo.toml (crates.io: registry dependencies in every dependency table,
|
|
6
|
+
asked at the lower bound of the requirement) and pom.xml (Maven: `groupId:artifactId` with a plain or property-resolved
|
|
7
|
+
version, no ranges and no version inherited from a parent or a BOM). The audit asks about the version a sibling package-lock.json resolved
|
|
8
|
+
when it satisfies the range, else about the lower bound. Compares the dependencies of the file before and after the edit,
|
|
5
9
|
asks OSV (one `querybatch` call, short timeout) about the added or changed ones only, and answers from a disk cache of
|
|
6
10
|
(ecosystem, package, version) -> advisory IDs for 6 hours. Only package names and versions leave the machine. The caller (`hook_decision`) turns a `DependencyCheckError` into
|
|
7
11
|
"dependency check skipped: <reason>" and lets the edit go on.
|
|
@@ -11,6 +15,8 @@ import json
|
|
|
11
15
|
import os
|
|
12
16
|
import re
|
|
13
17
|
import time
|
|
18
|
+
import tomllib
|
|
19
|
+
import xml.etree.ElementTree as ElementTree
|
|
14
20
|
|
|
15
21
|
import atomic_io
|
|
16
22
|
import freshness
|
|
@@ -20,6 +26,10 @@ CACHE_PATH = os.path.join(paths.DATA_DIR, "dependency_advisories.json")
|
|
|
20
26
|
CACHE_TTL_S = 6 * 3600
|
|
21
27
|
OSV_TIMEOUT_S = 2.5
|
|
22
28
|
MAX_PACKAGES = 5
|
|
29
|
+
BATCH_SIZE = 100
|
|
30
|
+
AUDIT_MAX_MANIFESTS = 50
|
|
31
|
+
AUDIT_MAX_DEPTH = 4
|
|
32
|
+
AUDIT_SKIP_DIRS = frozenset(("node_modules", "vendor", "dist", "build", "target", "__pycache__", "venv", "site-packages"))
|
|
23
33
|
MAX_IDS_SHOWN = 5
|
|
24
34
|
TABLES = ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies")
|
|
25
35
|
PREFIX = "Smart Tool dependencies: "
|
|
@@ -29,6 +39,11 @@ _REQUIREMENTS_FILE_RE = re.compile(r"requirements[A-Za-z0-9._-]*\.txt")
|
|
|
29
39
|
_REQUIREMENT_RE = re.compile(r"([A-Za-z0-9][A-Za-z0-9._-]*)\s*(?:\[[^\]]*\])?\s*([^;]*)")
|
|
30
40
|
_PIP_VERSION_RE = re.compile(r"[0-9][0-9A-Za-z.+!_-]*")
|
|
31
41
|
_GO_VERSION_RE = re.compile(r"v\d+\.\d+\.\d+")
|
|
42
|
+
_LOCKED_RE = re.compile(r"(\d+)\.(\d+)\.(\d+)")
|
|
43
|
+
_MAVEN_VERSION_RE = re.compile(r"[0-9][0-9A-Za-z.+_-]*")
|
|
44
|
+
_PROPERTY_RE = re.compile(r"\$\{([^}]+)\}")
|
|
45
|
+
_CARGO_TABLES = ("dependencies", "dev-dependencies", "build-dependencies")
|
|
46
|
+
_TAG_RE = re.compile(r"\*|[xX]|[A-Za-z][A-Za-z0-9._-]*")
|
|
32
47
|
|
|
33
48
|
|
|
34
49
|
class DependencyCheckError(RuntimeError):
|
|
@@ -102,17 +117,100 @@ def _go_pairs(text):
|
|
|
102
117
|
return pairs
|
|
103
118
|
|
|
104
119
|
|
|
120
|
+
def _pyproject_pairs(text):
|
|
121
|
+
"""The (name, specifier) pairs of `project.dependencies` and `project.optional-dependencies` in a pyproject.toml text.
|
|
122
|
+
|
|
123
|
+
Parameters: text (str) file content.
|
|
124
|
+
Returns: dict (name, specifier) -> None in file order, read as requirement lines; None when the text is not TOML.
|
|
125
|
+
"""
|
|
126
|
+
try:
|
|
127
|
+
project = tomllib.loads(text).get("project")
|
|
128
|
+
except tomllib.TOMLDecodeError:
|
|
129
|
+
return None
|
|
130
|
+
if not isinstance(project, dict):
|
|
131
|
+
return {}
|
|
132
|
+
lines = [item for item in project.get("dependencies") or [] if isinstance(item, str)]
|
|
133
|
+
optional = project.get("optional-dependencies")
|
|
134
|
+
for group in optional.values() if isinstance(optional, dict) else ():
|
|
135
|
+
lines += [item for item in group if isinstance(item, str)]
|
|
136
|
+
return _requirements_pairs("\n".join(lines))
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def _cargo_pairs(text):
|
|
140
|
+
"""The (crate, requirement) pairs of the registry dependencies of a Cargo.toml text: the dependency, dev-dependency and
|
|
141
|
+
build-dependency tables, the `target.*` tables and `workspace.dependencies`. Path, git and `workspace = true` entries
|
|
142
|
+
have no registry version and are skipped; a `package` key renames the crate.
|
|
143
|
+
|
|
144
|
+
Parameters: text (str) file content.
|
|
145
|
+
Returns: dict (crate, requirement) -> None in file order ("" when a registry entry names no version); None when the
|
|
146
|
+
text is not TOML.
|
|
147
|
+
"""
|
|
148
|
+
try:
|
|
149
|
+
data = tomllib.loads(text)
|
|
150
|
+
except tomllib.TOMLDecodeError:
|
|
151
|
+
return None
|
|
152
|
+
groups = [data]
|
|
153
|
+
targets = data.get("target")
|
|
154
|
+
groups += [group for group in targets.values() if isinstance(group, dict)] if isinstance(targets, dict) else []
|
|
155
|
+
workspace = data.get("workspace")
|
|
156
|
+
groups += [workspace] if isinstance(workspace, dict) else []
|
|
157
|
+
pairs = {}
|
|
158
|
+
for group in groups:
|
|
159
|
+
for table in _CARGO_TABLES:
|
|
160
|
+
entries = group.get(table)
|
|
161
|
+
for name, value in entries.items() if isinstance(entries, dict) else ():
|
|
162
|
+
if isinstance(value, str):
|
|
163
|
+
pairs[(name, value)] = None
|
|
164
|
+
elif isinstance(value, dict) and not {"path", "git", "workspace"} & value.keys():
|
|
165
|
+
version = value.get("version")
|
|
166
|
+
pairs[(value.get("package") if isinstance(value.get("package"), str) else name,
|
|
167
|
+
version if isinstance(version, str) else "")] = None
|
|
168
|
+
return pairs
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _pom_pairs(text):
|
|
172
|
+
"""The (`groupId:artifactId`, version) pairs of every `dependency` element of a pom.xml text, dependency management
|
|
173
|
+
included. `${property}` versions take the value of the pom's `properties` and of `project.groupId|artifactId|version`;
|
|
174
|
+
one that stays unresolved is kept as written, so it reports as not verified.
|
|
175
|
+
|
|
176
|
+
Parameters: text (str) file content.
|
|
177
|
+
Returns: dict (name, version) -> None in document order ("" when no version is declared); None when the text is not
|
|
178
|
+
XML or declares entities.
|
|
179
|
+
"""
|
|
180
|
+
if "<!ENTITY" in text:
|
|
181
|
+
return None
|
|
182
|
+
try:
|
|
183
|
+
root = ElementTree.fromstring(text)
|
|
184
|
+
except ElementTree.ParseError:
|
|
185
|
+
return None
|
|
186
|
+
local = lambda element: element.tag.rpartition("}")[2] # noqa: E731
|
|
187
|
+
child = lambda element, tag: next((item.text or "" for item in element if local(item) == tag), "").strip() # noqa: E731
|
|
188
|
+
properties = {f"project.{tag}": child(root, tag) for tag in ("groupId", "artifactId", "version")}
|
|
189
|
+
properties.update({local(item): (item.text or "").strip()
|
|
190
|
+
for block in root if local(block) == "properties" for item in block})
|
|
191
|
+
resolve = lambda value: _PROPERTY_RE.sub(lambda match: properties.get(match.group(1)) or match.group(0), value) # noqa: E731
|
|
192
|
+
return {(resolve(f"{child(item, 'groupId')}:{child(item, 'artifactId')}"), resolve(child(item, "version"))): None
|
|
193
|
+
for item in root.iter() if local(item) == "dependency"}
|
|
194
|
+
|
|
195
|
+
|
|
105
196
|
def ecosystem_of(path):
|
|
106
|
-
"""The
|
|
197
|
+
"""The ecosystem key of a manifest file name; `_ECOSYSTEMS[key].osv` is the name OSV knows it by.
|
|
107
198
|
|
|
108
199
|
Parameters: path (str) file path.
|
|
109
|
-
Returns: "npm" for package.json, "PyPI" for requirements*.txt, "Go" for go.mod
|
|
200
|
+
Returns: "npm" for package.json, "PyPI" for requirements*.txt, "pyproject" for pyproject.toml, "Go" for go.mod, "Cargo" for
|
|
201
|
+
Cargo.toml, "Maven" for pom.xml; None for any other file.
|
|
110
202
|
"""
|
|
111
203
|
name = os.path.basename(path)
|
|
112
204
|
if name == "package.json":
|
|
113
205
|
return "npm"
|
|
114
206
|
if name == "go.mod":
|
|
115
207
|
return "Go"
|
|
208
|
+
if name == "pyproject.toml":
|
|
209
|
+
return "pyproject"
|
|
210
|
+
if name == "Cargo.toml":
|
|
211
|
+
return "Cargo"
|
|
212
|
+
if name == "pom.xml":
|
|
213
|
+
return "Maven"
|
|
116
214
|
return "PyPI" if _REQUIREMENTS_FILE_RE.fullmatch(name) else None
|
|
117
215
|
|
|
118
216
|
|
|
@@ -133,7 +231,7 @@ def changed_dependencies(before, after, ecosystem="npm"):
|
|
|
133
231
|
else:
|
|
134
232
|
old = parse(before)
|
|
135
233
|
if old is None:
|
|
136
|
-
raise DependencyCheckError("current
|
|
234
|
+
raise DependencyCheckError(f"current {_ECOSYSTEMS[ecosystem].file} is not valid {_ECOSYSTEMS[ecosystem].syntax}")
|
|
137
235
|
return [pair for pair in now if pair not in old]
|
|
138
236
|
|
|
139
237
|
|
|
@@ -169,6 +267,21 @@ def _go_version(specifier):
|
|
|
169
267
|
return specifier if _GO_VERSION_RE.fullmatch(specifier) else None
|
|
170
268
|
|
|
171
269
|
|
|
270
|
+
def _cargo_version(specifier):
|
|
271
|
+
"""The Cargo case of `queried_version`: the lower bound of a bare, `^`, `~`, `=`, `>=` or `>` requirement padded to
|
|
272
|
+
three parts; None for `*`, wildcards, `<`, compound requirements and anything else."""
|
|
273
|
+
match = _SPECIFIER_RE.fullmatch(specifier.strip())
|
|
274
|
+
if not match:
|
|
275
|
+
return None
|
|
276
|
+
_operator, major, minor, patch, tail = match.groups()
|
|
277
|
+
return f"{major}.{minor or 0}.{patch or 0}{tail or ''}"
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def _maven_version(specifier):
|
|
281
|
+
"""The Maven case of `queried_version`: a plain version; None for ranges, unresolved properties and no version."""
|
|
282
|
+
return specifier if _MAVEN_VERSION_RE.fullmatch(specifier) else None
|
|
283
|
+
|
|
284
|
+
|
|
172
285
|
def queried_version(specifier, ecosystem="npm"):
|
|
173
286
|
"""The version to ask OSV about for a specifier.
|
|
174
287
|
|
|
@@ -178,12 +291,20 @@ def queried_version(specifier, ecosystem="npm"):
|
|
|
178
291
|
return _ECOSYSTEMS[ecosystem].version(specifier)
|
|
179
292
|
|
|
180
293
|
|
|
181
|
-
_Ecosystem = collections.namedtuple("_Ecosystem", "parse version label exact")
|
|
294
|
+
_Ecosystem = collections.namedtuple("_Ecosystem", "parse version label exact osv file syntax")
|
|
295
|
+
_PIP_LABEL = lambda name, spec: f"{name}{spec}" # noqa: E731
|
|
296
|
+
_PIP_EXACT = lambda spec, version: spec == "==" + version # noqa: E731
|
|
182
297
|
_ECOSYSTEMS = {
|
|
183
|
-
"npm": _Ecosystem(_npm_pairs, _npm_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec == version
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
"
|
|
298
|
+
"npm": _Ecosystem(_npm_pairs, _npm_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec == version,
|
|
299
|
+
"npm", "package.json", "JSON"),
|
|
300
|
+
"PyPI": _Ecosystem(_requirements_pairs, _pip_version, _PIP_LABEL, _PIP_EXACT, "PyPI", "requirements file", "text"),
|
|
301
|
+
"pyproject": _Ecosystem(_pyproject_pairs, _pip_version, _PIP_LABEL, _PIP_EXACT, "PyPI", "pyproject.toml", "TOML"),
|
|
302
|
+
"Go": _Ecosystem(_go_pairs, _go_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec == version,
|
|
303
|
+
"Go", "go.mod", "text"),
|
|
304
|
+
"Cargo": _Ecosystem(_cargo_pairs, _cargo_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec in (
|
|
305
|
+
version, "=" + version), "crates.io", "Cargo.toml", "TOML"),
|
|
306
|
+
"Maven": _Ecosystem(_pom_pairs, _maven_version, lambda name, spec: f"{name}@{spec}" if spec else name,
|
|
307
|
+
lambda spec, version: spec == version, "Maven", "pom.xml", "XML"),
|
|
187
308
|
}
|
|
188
309
|
ECOSYSTEMS = tuple(_ECOSYSTEMS)
|
|
189
310
|
|
|
@@ -215,7 +336,7 @@ def _fresh_ids(entry, now):
|
|
|
215
336
|
def _advisories(ecosystem, queries):
|
|
216
337
|
"""Advisory IDs for (name, version) pairs, from the cache and one OSV batch for the rest.
|
|
217
338
|
|
|
218
|
-
Parameters: ecosystem (str)
|
|
339
|
+
Parameters: ecosystem (str) the OSV ecosystem name (`_Ecosystem.osv`); queries (list of (name, version)).
|
|
219
340
|
Returns: tuple (list of IDs or None per query in order, str cache-write problem or "").
|
|
220
341
|
Raises: DependencyCheckError when the cache is corrupt or OSV cannot answer.
|
|
221
342
|
"""
|
|
@@ -226,7 +347,9 @@ def _advisories(ecosystem, queries):
|
|
|
226
347
|
if not missing:
|
|
227
348
|
return [found[query] for query in queries], ""
|
|
228
349
|
try:
|
|
229
|
-
answered =
|
|
350
|
+
answered = []
|
|
351
|
+
for start in range(0, len(missing), BATCH_SIZE):
|
|
352
|
+
answered += freshness.advisories_batch(ecosystem, missing[start:start + BATCH_SIZE], OSV_TIMEOUT_S)
|
|
230
353
|
except RuntimeError as exc:
|
|
231
354
|
raise DependencyCheckError(str(exc)) from exc
|
|
232
355
|
for query, ids in zip(missing, answered):
|
|
@@ -256,14 +379,14 @@ def notice(before, after, ecosystem="npm"):
|
|
|
256
379
|
handled, extra = changed[:MAX_PACKAGES], len(changed) - MAX_PACKAGES
|
|
257
380
|
lines, queries = {}, []
|
|
258
381
|
for name, specifier in handled:
|
|
259
|
-
version = spec.version(specifier) if freshness.NAME_RES[
|
|
382
|
+
version = spec.version(specifier) if freshness.NAME_RES[spec.osv].fullmatch(name) else None
|
|
260
383
|
if version is None:
|
|
261
384
|
lines[(name, specifier)] = f"`{spec.label(name, specifier)}` specifier not verified."
|
|
262
385
|
else:
|
|
263
386
|
queries.append((name, specifier, version))
|
|
264
387
|
problem = ""
|
|
265
388
|
if queries:
|
|
266
|
-
ids, problem = _advisories(
|
|
389
|
+
ids, problem = _advisories(spec.osv, [(name, version) for name, _specifier, version in queries])
|
|
267
390
|
for (name, specifier, version), advisories in zip(queries, ids):
|
|
268
391
|
label = f"`{spec.label(name, specifier)}`" + ("" if spec.exact(specifier, version) else f" (checked {version})")
|
|
269
392
|
if advisories is None:
|
|
@@ -278,3 +401,128 @@ def notice(before, after, ecosystem="npm"):
|
|
|
278
401
|
if problem and parts:
|
|
279
402
|
parts.append(f"({problem})")
|
|
280
403
|
return PREFIX + " ".join(parts) if parts else ""
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def manifests(root):
|
|
407
|
+
"""Project-relative paths of the dependency manifests under root (package.json, requirements*.txt, pyproject.toml,
|
|
408
|
+
go.mod, Cargo.toml, pom.xml), at most
|
|
409
|
+
AUDIT_MAX_MANIFESTS and AUDIT_MAX_DEPTH folders deep, skipping hidden and dependency folders."""
|
|
410
|
+
found = []
|
|
411
|
+
base_depth = os.path.abspath(root).count(os.sep)
|
|
412
|
+
for folder, dirs, files in os.walk(root):
|
|
413
|
+
dirs[:] = sorted(name for name in dirs if not name.startswith(".") and name not in AUDIT_SKIP_DIRS)
|
|
414
|
+
if folder.count(os.sep) - base_depth >= AUDIT_MAX_DEPTH:
|
|
415
|
+
dirs[:] = []
|
|
416
|
+
found += [os.path.relpath(os.path.join(folder, name), root).replace(os.sep, "/")
|
|
417
|
+
for name in sorted(files) if ecosystem_of(name)]
|
|
418
|
+
return found[:AUDIT_MAX_MANIFESTS]
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _locked_versions(folder):
|
|
422
|
+
"""The versions a package-lock.json next to a package.json resolved, by package name.
|
|
423
|
+
|
|
424
|
+
Parameters: folder (str) the folder of the package.json.
|
|
425
|
+
Returns: dict name -> version of the top-level installs (lockfile v2 and v3 `packages`, v1 `dependencies`); empty
|
|
426
|
+
when there is no lockfile or it cannot be read.
|
|
427
|
+
"""
|
|
428
|
+
try:
|
|
429
|
+
with open(os.path.join(folder, "package-lock.json"), encoding="utf-8") as stream:
|
|
430
|
+
data = json.load(stream)
|
|
431
|
+
except (OSError, UnicodeDecodeError, ValueError):
|
|
432
|
+
return {}
|
|
433
|
+
if not isinstance(data, dict):
|
|
434
|
+
return {}
|
|
435
|
+
found = {}
|
|
436
|
+
for name, entry in (data.get("dependencies") if isinstance(data.get("dependencies"), dict) else {}).items():
|
|
437
|
+
if isinstance(entry, dict) and isinstance(entry.get("version"), str):
|
|
438
|
+
found[name] = entry["version"]
|
|
439
|
+
prefix = "node_modules/"
|
|
440
|
+
for key, entry in (data.get("packages") if isinstance(data.get("packages"), dict) else {}).items():
|
|
441
|
+
if key.startswith(prefix) and "/node_modules/" not in key and isinstance(entry, dict) \
|
|
442
|
+
and isinstance(entry.get("version"), str):
|
|
443
|
+
found[key[len(prefix):]] = entry["version"]
|
|
444
|
+
return found
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def _locked_version(specifier, locked):
|
|
448
|
+
"""The lockfile version to ask about for an npm specifier, or None when the lock does not apply to it.
|
|
449
|
+
|
|
450
|
+
Parameters: specifier (str) the value in the dependency table; locked (str or None) the resolved version.
|
|
451
|
+
Returns: `locked` when it satisfies the specifier (an exact pin, `^` on the same major, `~` on the same minor, `>=` and
|
|
452
|
+
`>` at or above the bound) or the specifier is a dist-tag, `*` or an x-range; None for a stale lock, a lock that is not
|
|
453
|
+
a plain version and specifiers that point elsewhere (URLs, git, file:, workspace:, aliases, compound ranges).
|
|
454
|
+
"""
|
|
455
|
+
resolved = _LOCKED_RE.match(locked or "")
|
|
456
|
+
if not resolved:
|
|
457
|
+
return None
|
|
458
|
+
if _TAG_RE.fullmatch(specifier.strip()):
|
|
459
|
+
return locked
|
|
460
|
+
match = _SPECIFIER_RE.fullmatch(specifier.strip())
|
|
461
|
+
if not match:
|
|
462
|
+
return None
|
|
463
|
+
operator, major, minor, patch, _tail = match.groups()
|
|
464
|
+
have = tuple(int(part) for part in resolved.groups())
|
|
465
|
+
bound = tuple(int(part or 0) for part in (major, minor, patch))
|
|
466
|
+
if operator in (">=", ">"):
|
|
467
|
+
ok = have >= bound
|
|
468
|
+
elif operator == "^":
|
|
469
|
+
ok = have >= bound and have[0] == bound[0]
|
|
470
|
+
elif operator == "~":
|
|
471
|
+
ok = have >= bound and have[:2] == bound[:2]
|
|
472
|
+
else:
|
|
473
|
+
ok = None not in (minor, patch) and have == bound
|
|
474
|
+
return locked if ok else None
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def audit(root):
|
|
478
|
+
"""Advisories of every dependency the manifests of a project declare, not only the ones an edit changed.
|
|
479
|
+
|
|
480
|
+
Parameters: root (str) project folder.
|
|
481
|
+
Returns: dict {"checked": packages asked about, "manifests": [{"path", "ecosystem", "packages", "vulnerable":
|
|
482
|
+
[{"package", "specifier", "version", "advisories"}], "unverified": [str]}], "note": str}. A manifest that cannot be
|
|
483
|
+
read or parsed is listed with its reason in `unverified`. For a package.json with a package-lock.json beside it, the
|
|
484
|
+
version asked about is the locked one when it satisfies the range (reported in `version`). One OSV `querybatch` per
|
|
485
|
+
BATCH_SIZE packages and ecosystem; the 6-hour advisory cache is shared with the edit notice.
|
|
486
|
+
Raises: DependencyCheckError when OSV or the cache cannot answer.
|
|
487
|
+
"""
|
|
488
|
+
report, plans, queries = [], [], {}
|
|
489
|
+
for rel in manifests(root):
|
|
490
|
+
spec = _ECOSYSTEMS[ecosystem_of(rel)]
|
|
491
|
+
ecosystem = spec.osv
|
|
492
|
+
entry = {"path": rel, "ecosystem": ecosystem, "packages": 0, "vulnerable": [], "unverified": []}
|
|
493
|
+
report.append(entry)
|
|
494
|
+
try:
|
|
495
|
+
with open(os.path.join(root, rel), encoding="utf-8") as stream:
|
|
496
|
+
pairs = spec.parse(stream.read())
|
|
497
|
+
except (OSError, UnicodeDecodeError) as exc:
|
|
498
|
+
entry["unverified"].append(f"manifest not read: {type(exc).__name__}")
|
|
499
|
+
continue
|
|
500
|
+
if pairs is None:
|
|
501
|
+
entry["unverified"].append(f"manifest is not valid {spec.syntax}")
|
|
502
|
+
continue
|
|
503
|
+
entry["packages"] = len(pairs)
|
|
504
|
+
locks = _locked_versions(os.path.dirname(os.path.join(root, rel))) if ecosystem == "npm" else {}
|
|
505
|
+
for name, specifier in pairs:
|
|
506
|
+
version = None
|
|
507
|
+
if freshness.NAME_RES[ecosystem].fullmatch(name):
|
|
508
|
+
version = _locked_version(specifier, locks.get(name)) or spec.version(specifier)
|
|
509
|
+
if version is None:
|
|
510
|
+
entry["unverified"].append(f"{spec.label(name, specifier)}: specifier not verified")
|
|
511
|
+
else:
|
|
512
|
+
plans.append((entry, ecosystem, name, specifier, version, spec.label))
|
|
513
|
+
queries.setdefault(ecosystem, {})[(name, version)] = None
|
|
514
|
+
answers, problem = {}, ""
|
|
515
|
+
for ecosystem, wanted in queries.items():
|
|
516
|
+
keys = list(wanted)
|
|
517
|
+
ids, problem = _advisories(ecosystem, keys)
|
|
518
|
+
answers[ecosystem] = dict(zip(keys, ids))
|
|
519
|
+
for entry, ecosystem, name, specifier, version, describe in plans:
|
|
520
|
+
advisories = answers[ecosystem][(name, version)]
|
|
521
|
+
label = describe(name, specifier)
|
|
522
|
+
if advisories is None:
|
|
523
|
+
entry["unverified"].append(f"{label}: advisory check incomplete")
|
|
524
|
+
elif advisories:
|
|
525
|
+
entry["vulnerable"].append({"package": name, "specifier": specifier, "version": version,
|
|
526
|
+
"advisories": advisories})
|
|
527
|
+
total = sum(len(wanted) for wanted in queries.values())
|
|
528
|
+
return {"checked": total, "manifests": report, "note": problem}
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""Optional external converter for what the native reader of document_read refuses: a file type it does not read and a
|
|
2
|
+
PDF with no text layer. Off unless the `document_converter` setting turns it on (see config.document_converter).
|
|
3
|
+
|
|
4
|
+
The engine (Kreuzberg first, Docling as the drop-in) is never imported here: it runs in a separate interpreter,
|
|
5
|
+
isolated with `-I`, under a time cap and a memory cap, and hands its pages back through a file. Whatever it returns
|
|
6
|
+
that is empty or thinner than MIN_CHARS_PER_PAGE is an error, never a document."""
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import shutil
|
|
11
|
+
import subprocess
|
|
12
|
+
import sys
|
|
13
|
+
import time
|
|
14
|
+
|
|
15
|
+
import atomic_io
|
|
16
|
+
import config
|
|
17
|
+
import document_text
|
|
18
|
+
import process_limits
|
|
19
|
+
import version
|
|
20
|
+
|
|
21
|
+
EXTENSIONS = ('.png', '.jpg', '.jpeg', '.tif', '.tiff', '.bmp', '.gif', '.webp', '.doc', '.xls', '.ppt', '.odt', '.ods',
|
|
22
|
+
'.odp', '.rtf')
|
|
23
|
+
TIMEOUT_S = 600
|
|
24
|
+
MEMORY_BYTES = 4 * 1024 ** 3
|
|
25
|
+
MIN_CHARS_PER_PAGE = 40
|
|
26
|
+
DETECT_TIMEOUT_S = 30
|
|
27
|
+
LOCK_POLL_S = 0.25
|
|
28
|
+
STALE_LOCK_S = TIMEOUT_S + 60
|
|
29
|
+
|
|
30
|
+
KREUZBERG_SCRIPT = r"""
|
|
31
|
+
import json, re, sys
|
|
32
|
+
import kreuzberg
|
|
33
|
+
source, target, force_ocr = sys.argv[1], sys.argv[2], sys.argv[3] == "1"
|
|
34
|
+
settings = kreuzberg.ExtractionConfig(output_format="markdown", force_ocr=force_ocr,
|
|
35
|
+
pages=kreuzberg.PageConfig(extract_pages=True, insert_page_markers=True))
|
|
36
|
+
result = kreuzberg.extract_file_sync(source, config=settings)
|
|
37
|
+
split = re.split(r"\\?<\\?!-- PAGE \d+ --\\?>", result.content)
|
|
38
|
+
pages = [page.strip() for page in split[1:]] if len(split) > 1 else [result.content]
|
|
39
|
+
with open(target, "w", encoding="utf-8") as stream:
|
|
40
|
+
json.dump({"pages": pages}, stream)
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
DOCLING_SCRIPT = """
|
|
44
|
+
import json, sys
|
|
45
|
+
from docling.document_converter import DocumentConverter
|
|
46
|
+
source, target = sys.argv[1], sys.argv[2]
|
|
47
|
+
marker = "<!-- page break -->"
|
|
48
|
+
text = DocumentConverter().convert(source).document.export_to_markdown(page_break_placeholder=marker)
|
|
49
|
+
with open(target, "w", encoding="utf-8") as stream:
|
|
50
|
+
json.dump({"pages": text.split(marker)}, stream)
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
ENGINES = {'kreuzberg': ('kreuzberg', KREUZBERG_SCRIPT), 'docling': ('docling', DOCLING_SCRIPT)}
|
|
54
|
+
_found = set()
|
|
55
|
+
_swept = set()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Converted:
|
|
59
|
+
"""The pages an engine returned for a file: the engine's name and the text of each page."""
|
|
60
|
+
|
|
61
|
+
def __init__(self, engine, pages):
|
|
62
|
+
self.engine = engine
|
|
63
|
+
self.pages = pages
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def enabled():
|
|
67
|
+
"""Whether the `document_converter` setting is anything but off."""
|
|
68
|
+
return config.document_converter()['mode'] != 'off'
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _installed(python, module):
|
|
72
|
+
"""Whether `module` can be imported by `python`; a positive answer is kept, a negative one is asked again."""
|
|
73
|
+
if (python, module) in _found:
|
|
74
|
+
return True
|
|
75
|
+
probe = 'import importlib.util, sys; sys.exit(0 if importlib.util.find_spec(sys.argv[1]) else 1)'
|
|
76
|
+
try:
|
|
77
|
+
done = subprocess.run([python, '-I', '-c', probe, module], capture_output=True, timeout=DETECT_TIMEOUT_S,
|
|
78
|
+
creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0))
|
|
79
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
80
|
+
return False
|
|
81
|
+
if done.returncode == 0:
|
|
82
|
+
_found.add((python, module))
|
|
83
|
+
return done.returncode == 0
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def locate():
|
|
87
|
+
"""The engine to use and the interpreter that has it.
|
|
88
|
+
|
|
89
|
+
Returns: `(name, python)`. Raises: DocumentError naming what was looked for and where when the setting is off, names
|
|
90
|
+
an engine that is not installed, or `auto` finds none."""
|
|
91
|
+
setting = config.document_converter()
|
|
92
|
+
python = setting['python'] or sys.executable
|
|
93
|
+
wanted = list(ENGINES) if setting['mode'] == 'auto' else [setting['mode']]
|
|
94
|
+
if wanted == ['off']:
|
|
95
|
+
raise document_text.DocumentError('The document converter is off.')
|
|
96
|
+
for name in wanted:
|
|
97
|
+
if _installed(python, ENGINES[name][0]):
|
|
98
|
+
return name, python
|
|
99
|
+
raise document_text.DocumentError(
|
|
100
|
+
f"The document converter is set to {setting['mode']} but {' or '.join(wanted)} is not installed in {python}.")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _checked(engine, pages, expected_pages):
|
|
104
|
+
"""The pages if the engine returned enough text, else a DocumentError: under MIN_CHARS_PER_PAGE for each page of the
|
|
105
|
+
document is a failed read, not a document."""
|
|
106
|
+
if not isinstance(pages, list) or not all(isinstance(page, str) for page in pages):
|
|
107
|
+
raise document_text.DocumentError(f'{engine} returned an answer that is not a list of pages.')
|
|
108
|
+
found = sum(len(page.strip()) for page in pages)
|
|
109
|
+
needed = MIN_CHARS_PER_PAGE * max(1, expected_pages or 1)
|
|
110
|
+
if found < needed:
|
|
111
|
+
raise document_text.DocumentError(
|
|
112
|
+
f'{engine} returned {found} characters for {max(1, expected_pages or 1)} page(s), under the {needed} '
|
|
113
|
+
'it takes to count as text: the file was not read.')
|
|
114
|
+
return pages
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def cache_folder():
|
|
118
|
+
"""The one folder of this version's converted pages: `%LOCALAPPDATA%\\SmartTool\\cache\\<version>\\convert`, created
|
|
119
|
+
here; the folders of other versions are deleted the first time it is asked for in a process.
|
|
120
|
+
|
|
121
|
+
Raises: RuntimeError when LOCALAPPDATA is not set."""
|
|
122
|
+
local = os.environ.get("LOCALAPPDATA")
|
|
123
|
+
if not local:
|
|
124
|
+
raise RuntimeError("LOCALAPPDATA is not set: there is no per-user folder for the converter's cache.")
|
|
125
|
+
root = os.path.join(local, "SmartTool", "cache")
|
|
126
|
+
folder = os.path.join(root, version.VERSION, "convert")
|
|
127
|
+
os.makedirs(folder, exist_ok=True)
|
|
128
|
+
if root not in _swept:
|
|
129
|
+
_swept.add(root)
|
|
130
|
+
for name in os.listdir(root):
|
|
131
|
+
if name != version.VERSION:
|
|
132
|
+
shutil.rmtree(os.path.join(root, name), ignore_errors=True)
|
|
133
|
+
return folder
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _key(path, engine, force_ocr):
|
|
137
|
+
"""The name of the result for this content, engine and OCR flag: a SHA-256 over them, never the file's name or place."""
|
|
138
|
+
digest = hashlib.sha256(f"{engine}\0{int(force_ocr)}\0".encode())
|
|
139
|
+
with open(path, "rb") as stream:
|
|
140
|
+
for block in iter(lambda: stream.read(1 << 20), b""):
|
|
141
|
+
digest.update(block)
|
|
142
|
+
return digest.hexdigest()[:40]
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _lock(path):
|
|
146
|
+
"""Takes the lock file `path` (created exclusively), waiting for another run that holds it; a lock older than
|
|
147
|
+
STALE_LOCK_S belongs to a run that died and is taken over."""
|
|
148
|
+
deadline = time.monotonic() + STALE_LOCK_S
|
|
149
|
+
while True:
|
|
150
|
+
try:
|
|
151
|
+
os.close(os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY))
|
|
152
|
+
return
|
|
153
|
+
except FileExistsError:
|
|
154
|
+
try:
|
|
155
|
+
if time.time() - os.path.getmtime(path) > STALE_LOCK_S:
|
|
156
|
+
os.remove(path)
|
|
157
|
+
continue
|
|
158
|
+
except OSError:
|
|
159
|
+
continue
|
|
160
|
+
if time.monotonic() > deadline:
|
|
161
|
+
raise document_text.DocumentError(f"Another conversion of the same file held {path} for over {STALE_LOCK_S}s.")
|
|
162
|
+
time.sleep(LOCK_POLL_S)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _cached(result):
|
|
166
|
+
"""The pages saved at `result`, or None when there is no such file or it is not a list of pages."""
|
|
167
|
+
try:
|
|
168
|
+
with open(result, encoding="utf-8") as stream:
|
|
169
|
+
pages = json.load(stream)["pages"]
|
|
170
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
171
|
+
return None
|
|
172
|
+
return pages if isinstance(pages, list) and all(isinstance(page, str) for page in pages) else None
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def convert(path, expected_pages=None, force_ocr=False):
|
|
176
|
+
"""The text of the file at `path` through the engine the setting picks, in an isolated subprocess.
|
|
177
|
+
|
|
178
|
+
Parameters: path (absolute path of the file), expected_pages (page count of the document when known, for the minimum
|
|
179
|
+
of text; None counts one page), force_ocr (read pages as images even when they carry a text layer; used for a PDF
|
|
180
|
+
the native reader found without one). Returns: a Converted. Raises: DocumentError with the reason when no engine is
|
|
181
|
+
installed, the engine fails, is stopped at TIMEOUT_S or MEMORY_BYTES, or returns empty or thin text."""
|
|
182
|
+
engine, python = locate()
|
|
183
|
+
folder = cache_folder()
|
|
184
|
+
stem = os.path.join(folder, _key(path, engine, force_ocr))
|
|
185
|
+
pages = _cached(stem + ".json")
|
|
186
|
+
if pages is None:
|
|
187
|
+
_lock(stem + ".lock")
|
|
188
|
+
try:
|
|
189
|
+
pages = _cached(stem + ".json")
|
|
190
|
+
if pages is None:
|
|
191
|
+
pages = _checked(engine, _engine_pages(engine, python, path, stem + ".work", force_ocr), expected_pages)
|
|
192
|
+
atomic_io.write_text(stem + ".json", json.dumps({"pages": pages}))
|
|
193
|
+
finally:
|
|
194
|
+
os.remove(stem + ".lock")
|
|
195
|
+
return Converted(engine, _checked(engine, pages, expected_pages))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _engine_pages(engine, python, path, workdir, force_ocr):
|
|
199
|
+
"""The pages the engine writes for `path`, run in `workdir` (made and removed here).
|
|
200
|
+
|
|
201
|
+
Raises: DocumentError when the engine is stopped, fails or returns no pages."""
|
|
202
|
+
os.makedirs(workdir, exist_ok=True)
|
|
203
|
+
target = os.path.join(workdir, 'pages.json')
|
|
204
|
+
try:
|
|
205
|
+
try:
|
|
206
|
+
code, tail = process_limits.run_capped(
|
|
207
|
+
[python, '-I', '-c', ENGINES[engine][1], path, target, '1' if force_ocr else '0'], workdir, TIMEOUT_S,
|
|
208
|
+
MEMORY_BYTES)
|
|
209
|
+
except subprocess.TimeoutExpired:
|
|
210
|
+
raise document_text.DocumentError(f'{engine} did not finish in {TIMEOUT_S}s and was stopped.') from None
|
|
211
|
+
except OSError as exc:
|
|
212
|
+
raise document_text.DocumentError(f'{engine} could not be started: {exc}') from None
|
|
213
|
+
if code:
|
|
214
|
+
raise document_text.DocumentError(f'{engine} failed (exit {code}): {tail[-300:] or "no error text"}')
|
|
215
|
+
try:
|
|
216
|
+
with open(target, encoding='utf-8') as stream:
|
|
217
|
+
return json.load(stream)['pages']
|
|
218
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
219
|
+
raise document_text.DocumentError(f'{engine} finished without returning its pages.') from None
|
|
220
|
+
finally:
|
|
221
|
+
shutil.rmtree(workdir, ignore_errors=True)
|