@allansantos-dev/smart-tool 0.9.78 → 0.9.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,11 @@
1
1
  """Vulnerable-dependency notice for an edit of a dependency manifest, built before the edit runs.
2
2
 
3
- Manifests: package.json (OSV ecosystem npm), requirements*.txt (PyPI: `==` pins and the lower bound of `>=` and `~=`)
4
- and go.mod (Go: `require` lines with a plain vX.Y.Z). Compares the dependencies of the file before and after the edit,
3
+ Manifests: package.json (OSV ecosystem npm), requirements*.txt and pyproject.toml (PyPI: `==` pins and the lower bound
4
+ of `>=` and `~=`; pyproject reads `project.dependencies` and `project.optional-dependencies`, not Poetry tables),
5
+ go.mod (Go: `require` lines with a plain vX.Y.Z), Cargo.toml (crates.io: registry dependencies in every dependency table,
6
+ asked at the lower bound of the requirement) and pom.xml (Maven: `groupId:artifactId` with a plain or property-resolved
7
+ version, no ranges and no version inherited from a parent or a BOM). The audit asks about the version a sibling package-lock.json resolved
8
+ when it satisfies the range, else about the lower bound. Compares the dependencies of the file before and after the edit,
5
9
  asks OSV (one `querybatch` call, short timeout) about the added or changed ones only, and answers from a disk cache of
6
10
  (ecosystem, package, version) -> advisory IDs for 6 hours. Only package names and versions leave the machine. The caller (`hook_decision`) turns a `DependencyCheckError` into
7
11
  "dependency check skipped: <reason>" and lets the edit go on.
@@ -11,6 +15,8 @@ import json
11
15
  import os
12
16
  import re
13
17
  import time
18
+ import tomllib
19
+ import xml.etree.ElementTree as ElementTree
14
20
 
15
21
  import atomic_io
16
22
  import freshness
@@ -20,6 +26,10 @@ CACHE_PATH = os.path.join(paths.DATA_DIR, "dependency_advisories.json")
20
26
  CACHE_TTL_S = 6 * 3600
21
27
  OSV_TIMEOUT_S = 2.5
22
28
  MAX_PACKAGES = 5
29
+ BATCH_SIZE = 100
30
+ AUDIT_MAX_MANIFESTS = 50
31
+ AUDIT_MAX_DEPTH = 4
32
+ AUDIT_SKIP_DIRS = frozenset(("node_modules", "vendor", "dist", "build", "target", "__pycache__", "venv", "site-packages"))
23
33
  MAX_IDS_SHOWN = 5
24
34
  TABLES = ("dependencies", "devDependencies", "peerDependencies", "optionalDependencies")
25
35
  PREFIX = "Smart Tool dependencies: "
@@ -29,6 +39,11 @@ _REQUIREMENTS_FILE_RE = re.compile(r"requirements[A-Za-z0-9._-]*\.txt")
29
39
  _REQUIREMENT_RE = re.compile(r"([A-Za-z0-9][A-Za-z0-9._-]*)\s*(?:\[[^\]]*\])?\s*([^;]*)")
30
40
  _PIP_VERSION_RE = re.compile(r"[0-9][0-9A-Za-z.+!_-]*")
31
41
  _GO_VERSION_RE = re.compile(r"v\d+\.\d+\.\d+")
42
+ _LOCKED_RE = re.compile(r"(\d+)\.(\d+)\.(\d+)")
43
+ _MAVEN_VERSION_RE = re.compile(r"[0-9][0-9A-Za-z.+_-]*")
44
+ _PROPERTY_RE = re.compile(r"\$\{([^}]+)\}")
45
+ _CARGO_TABLES = ("dependencies", "dev-dependencies", "build-dependencies")
46
+ _TAG_RE = re.compile(r"\*|[xX]|[A-Za-z][A-Za-z0-9._-]*")
32
47
 
33
48
 
34
49
  class DependencyCheckError(RuntimeError):
@@ -102,17 +117,100 @@ def _go_pairs(text):
102
117
  return pairs
103
118
 
104
119
 
120
+ def _pyproject_pairs(text):
121
+ """The (name, specifier) pairs of `project.dependencies` and `project.optional-dependencies` in a pyproject.toml text.
122
+
123
+ Parameters: text (str) file content.
124
+ Returns: dict (name, specifier) -> None in file order, read as requirement lines; None when the text is not TOML.
125
+ """
126
+ try:
127
+ project = tomllib.loads(text).get("project")
128
+ except tomllib.TOMLDecodeError:
129
+ return None
130
+ if not isinstance(project, dict):
131
+ return {}
132
+ lines = [item for item in project.get("dependencies") or [] if isinstance(item, str)]
133
+ optional = project.get("optional-dependencies")
134
+ for group in optional.values() if isinstance(optional, dict) else ():
135
+ lines += [item for item in group if isinstance(item, str)]
136
+ return _requirements_pairs("\n".join(lines))
137
+
138
+
139
+ def _cargo_pairs(text):
140
+ """The (crate, requirement) pairs of the registry dependencies of a Cargo.toml text: the dependency, dev-dependency and
141
+ build-dependency tables, the `target.*` tables and `workspace.dependencies`. Path, git and `workspace = true` entries
142
+ have no registry version and are skipped; a `package` key renames the crate.
143
+
144
+ Parameters: text (str) file content.
145
+ Returns: dict (crate, requirement) -> None in file order ("" when a registry entry names no version); None when the
146
+ text is not TOML.
147
+ """
148
+ try:
149
+ data = tomllib.loads(text)
150
+ except tomllib.TOMLDecodeError:
151
+ return None
152
+ groups = [data]
153
+ targets = data.get("target")
154
+ groups += [group for group in targets.values() if isinstance(group, dict)] if isinstance(targets, dict) else []
155
+ workspace = data.get("workspace")
156
+ groups += [workspace] if isinstance(workspace, dict) else []
157
+ pairs = {}
158
+ for group in groups:
159
+ for table in _CARGO_TABLES:
160
+ entries = group.get(table)
161
+ for name, value in entries.items() if isinstance(entries, dict) else ():
162
+ if isinstance(value, str):
163
+ pairs[(name, value)] = None
164
+ elif isinstance(value, dict) and not {"path", "git", "workspace"} & value.keys():
165
+ version = value.get("version")
166
+ pairs[(value.get("package") if isinstance(value.get("package"), str) else name,
167
+ version if isinstance(version, str) else "")] = None
168
+ return pairs
169
+
170
+
171
+ def _pom_pairs(text):
172
+ """The (`groupId:artifactId`, version) pairs of every `dependency` element of a pom.xml text, dependency management
173
+ included. `${property}` versions take the value of the pom's `properties` and of `project.groupId|artifactId|version`;
174
+ one that stays unresolved is kept as written, so it reports as not verified.
175
+
176
+ Parameters: text (str) file content.
177
+ Returns: dict (name, version) -> None in document order ("" when no version is declared); None when the text is not
178
+ XML or declares entities.
179
+ """
180
+ if "<!ENTITY" in text:
181
+ return None
182
+ try:
183
+ root = ElementTree.fromstring(text)
184
+ except ElementTree.ParseError:
185
+ return None
186
+ local = lambda element: element.tag.rpartition("}")[2] # noqa: E731
187
+ child = lambda element, tag: next((item.text or "" for item in element if local(item) == tag), "").strip() # noqa: E731
188
+ properties = {f"project.{tag}": child(root, tag) for tag in ("groupId", "artifactId", "version")}
189
+ properties.update({local(item): (item.text or "").strip()
190
+ for block in root if local(block) == "properties" for item in block})
191
+ resolve = lambda value: _PROPERTY_RE.sub(lambda match: properties.get(match.group(1)) or match.group(0), value) # noqa: E731
192
+ return {(resolve(f"{child(item, 'groupId')}:{child(item, 'artifactId')}"), resolve(child(item, "version"))): None
193
+ for item in root.iter() if local(item) == "dependency"}
194
+
195
+
105
196
  def ecosystem_of(path):
106
- """The OSV ecosystem of a manifest file name.
197
+ """The ecosystem key of a manifest file name; `_ECOSYSTEMS[key].osv` is the name OSV knows it by.
107
198
 
108
199
  Parameters: path (str) file path.
109
- Returns: "npm" for package.json, "PyPI" for requirements*.txt, "Go" for go.mod; None for any other file.
200
+ Returns: "npm" for package.json, "PyPI" for requirements*.txt, "pyproject" for pyproject.toml, "Go" for go.mod, "Cargo" for
201
+ Cargo.toml, "Maven" for pom.xml; None for any other file.
110
202
  """
111
203
  name = os.path.basename(path)
112
204
  if name == "package.json":
113
205
  return "npm"
114
206
  if name == "go.mod":
115
207
  return "Go"
208
+ if name == "pyproject.toml":
209
+ return "pyproject"
210
+ if name == "Cargo.toml":
211
+ return "Cargo"
212
+ if name == "pom.xml":
213
+ return "Maven"
116
214
  return "PyPI" if _REQUIREMENTS_FILE_RE.fullmatch(name) else None
117
215
 
118
216
 
@@ -133,7 +231,7 @@ def changed_dependencies(before, after, ecosystem="npm"):
133
231
  else:
134
232
  old = parse(before)
135
233
  if old is None:
136
- raise DependencyCheckError("current package.json is not valid JSON")
234
+ raise DependencyCheckError(f"current {_ECOSYSTEMS[ecosystem].file} is not valid {_ECOSYSTEMS[ecosystem].syntax}")
137
235
  return [pair for pair in now if pair not in old]
138
236
 
139
237
 
@@ -169,6 +267,21 @@ def _go_version(specifier):
169
267
  return specifier if _GO_VERSION_RE.fullmatch(specifier) else None
170
268
 
171
269
 
270
+ def _cargo_version(specifier):
271
+ """The Cargo case of `queried_version`: the lower bound of a bare, `^`, `~`, `=`, `>=` or `>` requirement padded to
272
+ three parts; None for `*`, wildcards, `<`, compound requirements and anything else."""
273
+ match = _SPECIFIER_RE.fullmatch(specifier.strip())
274
+ if not match:
275
+ return None
276
+ _operator, major, minor, patch, tail = match.groups()
277
+ return f"{major}.{minor or 0}.{patch or 0}{tail or ''}"
278
+
279
+
280
+ def _maven_version(specifier):
281
+ """The Maven case of `queried_version`: a plain version; None for ranges, unresolved properties and no version."""
282
+ return specifier if _MAVEN_VERSION_RE.fullmatch(specifier) else None
283
+
284
+
172
285
  def queried_version(specifier, ecosystem="npm"):
173
286
  """The version to ask OSV about for a specifier.
174
287
 
@@ -178,12 +291,20 @@ def queried_version(specifier, ecosystem="npm"):
178
291
  return _ECOSYSTEMS[ecosystem].version(specifier)
179
292
 
180
293
 
181
- _Ecosystem = collections.namedtuple("_Ecosystem", "parse version label exact")
294
+ _Ecosystem = collections.namedtuple("_Ecosystem", "parse version label exact osv file syntax")
295
+ _PIP_LABEL = lambda name, spec: f"{name}{spec}" # noqa: E731
296
+ _PIP_EXACT = lambda spec, version: spec == "==" + version # noqa: E731
182
297
  _ECOSYSTEMS = {
183
- "npm": _Ecosystem(_npm_pairs, _npm_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec == version),
184
- "PyPI": _Ecosystem(_requirements_pairs, _pip_version, lambda name, spec: f"{name}{spec}",
185
- lambda spec, version: spec == "==" + version),
186
- "Go": _Ecosystem(_go_pairs, _go_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec == version),
298
+ "npm": _Ecosystem(_npm_pairs, _npm_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec == version,
299
+ "npm", "package.json", "JSON"),
300
+ "PyPI": _Ecosystem(_requirements_pairs, _pip_version, _PIP_LABEL, _PIP_EXACT, "PyPI", "requirements file", "text"),
301
+ "pyproject": _Ecosystem(_pyproject_pairs, _pip_version, _PIP_LABEL, _PIP_EXACT, "PyPI", "pyproject.toml", "TOML"),
302
+ "Go": _Ecosystem(_go_pairs, _go_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec == version,
303
+ "Go", "go.mod", "text"),
304
+ "Cargo": _Ecosystem(_cargo_pairs, _cargo_version, lambda name, spec: f"{name}@{spec}", lambda spec, version: spec in (
305
+ version, "=" + version), "crates.io", "Cargo.toml", "TOML"),
306
+ "Maven": _Ecosystem(_pom_pairs, _maven_version, lambda name, spec: f"{name}@{spec}" if spec else name,
307
+ lambda spec, version: spec == version, "Maven", "pom.xml", "XML"),
187
308
  }
188
309
  ECOSYSTEMS = tuple(_ECOSYSTEMS)
189
310
 
@@ -215,7 +336,7 @@ def _fresh_ids(entry, now):
215
336
  def _advisories(ecosystem, queries):
216
337
  """Advisory IDs for (name, version) pairs, from the cache and one OSV batch for the rest.
217
338
 
218
- Parameters: ecosystem (str) one of ECOSYSTEMS; queries (list of (name, version)).
339
+ Parameters: ecosystem (str) the OSV ecosystem name (`_Ecosystem.osv`); queries (list of (name, version)).
219
340
  Returns: tuple (list of IDs or None per query in order, str cache-write problem or "").
220
341
  Raises: DependencyCheckError when the cache is corrupt or OSV cannot answer.
221
342
  """
@@ -226,7 +347,9 @@ def _advisories(ecosystem, queries):
226
347
  if not missing:
227
348
  return [found[query] for query in queries], ""
228
349
  try:
229
- answered = freshness.advisories_batch(ecosystem, missing, OSV_TIMEOUT_S)
350
+ answered = []
351
+ for start in range(0, len(missing), BATCH_SIZE):
352
+ answered += freshness.advisories_batch(ecosystem, missing[start:start + BATCH_SIZE], OSV_TIMEOUT_S)
230
353
  except RuntimeError as exc:
231
354
  raise DependencyCheckError(str(exc)) from exc
232
355
  for query, ids in zip(missing, answered):
@@ -256,14 +379,14 @@ def notice(before, after, ecosystem="npm"):
256
379
  handled, extra = changed[:MAX_PACKAGES], len(changed) - MAX_PACKAGES
257
380
  lines, queries = {}, []
258
381
  for name, specifier in handled:
259
- version = spec.version(specifier) if freshness.NAME_RES[ecosystem].fullmatch(name) else None
382
+ version = spec.version(specifier) if freshness.NAME_RES[spec.osv].fullmatch(name) else None
260
383
  if version is None:
261
384
  lines[(name, specifier)] = f"`{spec.label(name, specifier)}` specifier not verified."
262
385
  else:
263
386
  queries.append((name, specifier, version))
264
387
  problem = ""
265
388
  if queries:
266
- ids, problem = _advisories(ecosystem, [(name, version) for name, _specifier, version in queries])
389
+ ids, problem = _advisories(spec.osv, [(name, version) for name, _specifier, version in queries])
267
390
  for (name, specifier, version), advisories in zip(queries, ids):
268
391
  label = f"`{spec.label(name, specifier)}`" + ("" if spec.exact(specifier, version) else f" (checked {version})")
269
392
  if advisories is None:
@@ -278,3 +401,128 @@ def notice(before, after, ecosystem="npm"):
278
401
  if problem and parts:
279
402
  parts.append(f"({problem})")
280
403
  return PREFIX + " ".join(parts) if parts else ""
404
+
405
+
406
+ def manifests(root):
407
+ """Project-relative paths of the dependency manifests under root (package.json, requirements*.txt, pyproject.toml,
408
+ go.mod, Cargo.toml, pom.xml), at most
409
+ AUDIT_MAX_MANIFESTS and AUDIT_MAX_DEPTH folders deep, skipping hidden and dependency folders."""
410
+ found = []
411
+ base_depth = os.path.abspath(root).count(os.sep)
412
+ for folder, dirs, files in os.walk(root):
413
+ dirs[:] = sorted(name for name in dirs if not name.startswith(".") and name not in AUDIT_SKIP_DIRS)
414
+ if folder.count(os.sep) - base_depth >= AUDIT_MAX_DEPTH:
415
+ dirs[:] = []
416
+ found += [os.path.relpath(os.path.join(folder, name), root).replace(os.sep, "/")
417
+ for name in sorted(files) if ecosystem_of(name)]
418
+ return found[:AUDIT_MAX_MANIFESTS]
419
+
420
+
421
+ def _locked_versions(folder):
422
+ """The versions a package-lock.json next to a package.json resolved, by package name.
423
+
424
+ Parameters: folder (str) the folder of the package.json.
425
+ Returns: dict name -> version of the top-level installs (lockfile v2 and v3 `packages`, v1 `dependencies`); empty
426
+ when there is no lockfile or it cannot be read.
427
+ """
428
+ try:
429
+ with open(os.path.join(folder, "package-lock.json"), encoding="utf-8") as stream:
430
+ data = json.load(stream)
431
+ except (OSError, UnicodeDecodeError, ValueError):
432
+ return {}
433
+ if not isinstance(data, dict):
434
+ return {}
435
+ found = {}
436
+ for name, entry in (data.get("dependencies") if isinstance(data.get("dependencies"), dict) else {}).items():
437
+ if isinstance(entry, dict) and isinstance(entry.get("version"), str):
438
+ found[name] = entry["version"]
439
+ prefix = "node_modules/"
440
+ for key, entry in (data.get("packages") if isinstance(data.get("packages"), dict) else {}).items():
441
+ if key.startswith(prefix) and "/node_modules/" not in key and isinstance(entry, dict) \
442
+ and isinstance(entry.get("version"), str):
443
+ found[key[len(prefix):]] = entry["version"]
444
+ return found
445
+
446
+
447
+ def _locked_version(specifier, locked):
448
+ """The lockfile version to ask about for an npm specifier, or None when the lock does not apply to it.
449
+
450
+ Parameters: specifier (str) the value in the dependency table; locked (str or None) the resolved version.
451
+ Returns: `locked` when it satisfies the specifier (an exact pin, `^` on the same major, `~` on the same minor, `>=` and
452
+ `>` at or above the bound) or the specifier is a dist-tag, `*` or an x-range; None for a stale lock, a lock that is not
453
+ a plain version and specifiers that point elsewhere (URLs, git, file:, workspace:, aliases, compound ranges).
454
+ """
455
+ resolved = _LOCKED_RE.match(locked or "")
456
+ if not resolved:
457
+ return None
458
+ if _TAG_RE.fullmatch(specifier.strip()):
459
+ return locked
460
+ match = _SPECIFIER_RE.fullmatch(specifier.strip())
461
+ if not match:
462
+ return None
463
+ operator, major, minor, patch, _tail = match.groups()
464
+ have = tuple(int(part) for part in resolved.groups())
465
+ bound = tuple(int(part or 0) for part in (major, minor, patch))
466
+ if operator in (">=", ">"):
467
+ ok = have >= bound
468
+ elif operator == "^":
469
+ ok = have >= bound and have[0] == bound[0]
470
+ elif operator == "~":
471
+ ok = have >= bound and have[:2] == bound[:2]
472
+ else:
473
+ ok = None not in (minor, patch) and have == bound
474
+ return locked if ok else None
475
+
476
+
477
+ def audit(root):
478
+ """Advisories of every dependency the manifests of a project declare, not only the ones an edit changed.
479
+
480
+ Parameters: root (str) project folder.
481
+ Returns: dict {"checked": packages asked about, "manifests": [{"path", "ecosystem", "packages", "vulnerable":
482
+ [{"package", "specifier", "version", "advisories"}], "unverified": [str]}], "note": str}. A manifest that cannot be
483
+ read or parsed is listed with its reason in `unverified`. For a package.json with a package-lock.json beside it, the
484
+ version asked about is the locked one when it satisfies the range (reported in `version`). One OSV `querybatch` per
485
+ BATCH_SIZE packages and ecosystem; the 6-hour advisory cache is shared with the edit notice.
486
+ Raises: DependencyCheckError when OSV or the cache cannot answer.
487
+ """
488
+ report, plans, queries = [], [], {}
489
+ for rel in manifests(root):
490
+ spec = _ECOSYSTEMS[ecosystem_of(rel)]
491
+ ecosystem = spec.osv
492
+ entry = {"path": rel, "ecosystem": ecosystem, "packages": 0, "vulnerable": [], "unverified": []}
493
+ report.append(entry)
494
+ try:
495
+ with open(os.path.join(root, rel), encoding="utf-8") as stream:
496
+ pairs = spec.parse(stream.read())
497
+ except (OSError, UnicodeDecodeError) as exc:
498
+ entry["unverified"].append(f"manifest not read: {type(exc).__name__}")
499
+ continue
500
+ if pairs is None:
501
+ entry["unverified"].append(f"manifest is not valid {spec.syntax}")
502
+ continue
503
+ entry["packages"] = len(pairs)
504
+ locks = _locked_versions(os.path.dirname(os.path.join(root, rel))) if ecosystem == "npm" else {}
505
+ for name, specifier in pairs:
506
+ version = None
507
+ if freshness.NAME_RES[ecosystem].fullmatch(name):
508
+ version = _locked_version(specifier, locks.get(name)) or spec.version(specifier)
509
+ if version is None:
510
+ entry["unverified"].append(f"{spec.label(name, specifier)}: specifier not verified")
511
+ else:
512
+ plans.append((entry, ecosystem, name, specifier, version, spec.label))
513
+ queries.setdefault(ecosystem, {})[(name, version)] = None
514
+ answers, problem = {}, ""
515
+ for ecosystem, wanted in queries.items():
516
+ keys = list(wanted)
517
+ ids, problem = _advisories(ecosystem, keys)
518
+ answers[ecosystem] = dict(zip(keys, ids))
519
+ for entry, ecosystem, name, specifier, version, describe in plans:
520
+ advisories = answers[ecosystem][(name, version)]
521
+ label = describe(name, specifier)
522
+ if advisories is None:
523
+ entry["unverified"].append(f"{label}: advisory check incomplete")
524
+ elif advisories:
525
+ entry["vulnerable"].append({"package": name, "specifier": specifier, "version": version,
526
+ "advisories": advisories})
527
+ total = sum(len(wanted) for wanted in queries.values())
528
+ return {"checked": total, "manifests": report, "note": problem}
package/doc_scope.py CHANGED
@@ -690,7 +690,7 @@ def generate(root, view_id=None, limit=30, model=None):
690
690
  summary["failed_checks"] = [c for c in validation["checks"] if c["status"] == "fail"]
691
691
  return {"validation": summary, "folder": folder, "written": len(files), "removed": [f"{DOCS_DIR}/{r}" for r in removed], **counts,
692
692
  "prose_pending_total": len(pending), "prose_pending": pending[:limit], "prose_dropped": dropped[:limit],
693
- "truncated": bool(model.graph.get("truncated")),
693
+ "truncated": bool(model.graph.get("truncated")), "left_out": model.graph["left_out"],
694
694
  "extras": sorted(f"{DOCS_DIR}/{rel}" for rel in extras), "index_file": index_file if block is not None else None,
695
695
  "guidance": (f"Validation FAILED: {summary['claims_failed']} claim(s) do not hold; read {validation['report']} "
696
696
  "and fix the generator or the code before trusting these pages. " if summary["verdict"] == "fail"
@@ -0,0 +1,221 @@
1
+ """Optional external converter for what the native reader of document_read refuses: a file type it does not read and a
2
+ PDF with no text layer. Off unless the `document_converter` setting turns it on (see config.document_converter).
3
+
4
+ The engine (Kreuzberg first, Docling as the drop-in) is never imported here: it runs in a separate interpreter,
5
+ isolated with `-I`, under a time cap and a memory cap, and hands its pages back through a file. Whatever it returns
6
+ that is empty or thinner than MIN_CHARS_PER_PAGE is an error, never a document."""
7
+ import hashlib
8
+ import json
9
+ import os
10
+ import shutil
11
+ import subprocess
12
+ import sys
13
+ import time
14
+
15
+ import atomic_io
16
+ import config
17
+ import document_text
18
+ import process_limits
19
+ import version
20
+
21
+ EXTENSIONS = ('.png', '.jpg', '.jpeg', '.tif', '.tiff', '.bmp', '.gif', '.webp', '.doc', '.xls', '.ppt', '.odt', '.ods',
22
+ '.odp', '.rtf')
23
+ TIMEOUT_S = 600
24
+ MEMORY_BYTES = 4 * 1024 ** 3
25
+ MIN_CHARS_PER_PAGE = 40
26
+ DETECT_TIMEOUT_S = 30
27
+ LOCK_POLL_S = 0.25
28
+ STALE_LOCK_S = TIMEOUT_S + 60
29
+
30
+ KREUZBERG_SCRIPT = r"""
31
+ import json, re, sys
32
+ import kreuzberg
33
+ source, target, force_ocr = sys.argv[1], sys.argv[2], sys.argv[3] == "1"
34
+ settings = kreuzberg.ExtractionConfig(output_format="markdown", force_ocr=force_ocr,
35
+ pages=kreuzberg.PageConfig(extract_pages=True, insert_page_markers=True))
36
+ result = kreuzberg.extract_file_sync(source, config=settings)
37
+ split = re.split(r"\\?<\\?!-- PAGE \d+ --\\?>", result.content)
38
+ pages = [page.strip() for page in split[1:]] if len(split) > 1 else [result.content]
39
+ with open(target, "w", encoding="utf-8") as stream:
40
+ json.dump({"pages": pages}, stream)
41
+ """
42
+
43
+ DOCLING_SCRIPT = """
44
+ import json, sys
45
+ from docling.document_converter import DocumentConverter
46
+ source, target = sys.argv[1], sys.argv[2]
47
+ marker = "<!-- page break -->"
48
+ text = DocumentConverter().convert(source).document.export_to_markdown(page_break_placeholder=marker)
49
+ with open(target, "w", encoding="utf-8") as stream:
50
+ json.dump({"pages": text.split(marker)}, stream)
51
+ """
52
+
53
+ ENGINES = {'kreuzberg': ('kreuzberg', KREUZBERG_SCRIPT), 'docling': ('docling', DOCLING_SCRIPT)}
54
+ _found = set()
55
+ _swept = set()
56
+
57
+
58
+ class Converted:
59
+ """The pages an engine returned for a file: the engine's name and the text of each page."""
60
+
61
+ def __init__(self, engine, pages):
62
+ self.engine = engine
63
+ self.pages = pages
64
+
65
+
66
+ def enabled():
67
+ """Whether the `document_converter` setting is anything but off."""
68
+ return config.document_converter()['mode'] != 'off'
69
+
70
+
71
+ def _installed(python, module):
72
+ """Whether `module` can be imported by `python`; a positive answer is kept, a negative one is asked again."""
73
+ if (python, module) in _found:
74
+ return True
75
+ probe = 'import importlib.util, sys; sys.exit(0 if importlib.util.find_spec(sys.argv[1]) else 1)'
76
+ try:
77
+ done = subprocess.run([python, '-I', '-c', probe, module], capture_output=True, timeout=DETECT_TIMEOUT_S,
78
+ creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0))
79
+ except (OSError, subprocess.TimeoutExpired):
80
+ return False
81
+ if done.returncode == 0:
82
+ _found.add((python, module))
83
+ return done.returncode == 0
84
+
85
+
86
+ def locate():
87
+ """The engine to use and the interpreter that has it.
88
+
89
+ Returns: `(name, python)`. Raises: DocumentError naming what was looked for and where when the setting is off, names
90
+ an engine that is not installed, or `auto` finds none."""
91
+ setting = config.document_converter()
92
+ python = setting['python'] or sys.executable
93
+ wanted = list(ENGINES) if setting['mode'] == 'auto' else [setting['mode']]
94
+ if wanted == ['off']:
95
+ raise document_text.DocumentError('The document converter is off.')
96
+ for name in wanted:
97
+ if _installed(python, ENGINES[name][0]):
98
+ return name, python
99
+ raise document_text.DocumentError(
100
+ f"The document converter is set to {setting['mode']} but {' or '.join(wanted)} is not installed in {python}.")
101
+
102
+
103
+ def _checked(engine, pages, expected_pages):
104
+ """The pages if the engine returned enough text, else a DocumentError: under MIN_CHARS_PER_PAGE for each page of the
105
+ document is a failed read, not a document."""
106
+ if not isinstance(pages, list) or not all(isinstance(page, str) for page in pages):
107
+ raise document_text.DocumentError(f'{engine} returned an answer that is not a list of pages.')
108
+ found = sum(len(page.strip()) for page in pages)
109
+ needed = MIN_CHARS_PER_PAGE * max(1, expected_pages or 1)
110
+ if found < needed:
111
+ raise document_text.DocumentError(
112
+ f'{engine} returned {found} characters for {max(1, expected_pages or 1)} page(s), under the {needed} '
113
+ 'it takes to count as text: the file was not read.')
114
+ return pages
115
+
116
+
117
+ def cache_folder():
118
+ """The one folder of this version's converted pages: `%LOCALAPPDATA%\\SmartTool\\cache\\<version>\\convert`, created
119
+ here; the folders of other versions are deleted the first time it is asked for in a process.
120
+
121
+ Raises: RuntimeError when LOCALAPPDATA is not set."""
122
+ local = os.environ.get("LOCALAPPDATA")
123
+ if not local:
124
+ raise RuntimeError("LOCALAPPDATA is not set: there is no per-user folder for the converter's cache.")
125
+ root = os.path.join(local, "SmartTool", "cache")
126
+ folder = os.path.join(root, version.VERSION, "convert")
127
+ os.makedirs(folder, exist_ok=True)
128
+ if root not in _swept:
129
+ _swept.add(root)
130
+ for name in os.listdir(root):
131
+ if name != version.VERSION:
132
+ shutil.rmtree(os.path.join(root, name), ignore_errors=True)
133
+ return folder
134
+
135
+
136
+ def _key(path, engine, force_ocr):
137
+ """The name of the result for this content, engine and OCR flag: a SHA-256 over them, never the file's name or place."""
138
+ digest = hashlib.sha256(f"{engine}\0{int(force_ocr)}\0".encode())
139
+ with open(path, "rb") as stream:
140
+ for block in iter(lambda: stream.read(1 << 20), b""):
141
+ digest.update(block)
142
+ return digest.hexdigest()[:40]
143
+
144
+
145
+ def _lock(path):
146
+ """Takes the lock file `path` (created exclusively), waiting for another run that holds it; a lock older than
147
+ STALE_LOCK_S belongs to a run that died and is taken over."""
148
+ deadline = time.monotonic() + STALE_LOCK_S
149
+ while True:
150
+ try:
151
+ os.close(os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY))
152
+ return
153
+ except FileExistsError:
154
+ try:
155
+ if time.time() - os.path.getmtime(path) > STALE_LOCK_S:
156
+ os.remove(path)
157
+ continue
158
+ except OSError:
159
+ continue
160
+ if time.monotonic() > deadline:
161
+ raise document_text.DocumentError(f"Another conversion of the same file held {path} for over {STALE_LOCK_S}s.")
162
+ time.sleep(LOCK_POLL_S)
163
+
164
+
165
+ def _cached(result):
166
+ """The pages saved at `result`, or None when there is no such file or it is not a list of pages."""
167
+ try:
168
+ with open(result, encoding="utf-8") as stream:
169
+ pages = json.load(stream)["pages"]
170
+ except (OSError, ValueError, KeyError, TypeError):
171
+ return None
172
+ return pages if isinstance(pages, list) and all(isinstance(page, str) for page in pages) else None
173
+
174
+
175
+ def convert(path, expected_pages=None, force_ocr=False):
176
+ """The text of the file at `path` through the engine the setting picks, in an isolated subprocess.
177
+
178
+ Parameters: path (absolute path of the file), expected_pages (page count of the document when known, for the minimum
179
+ of text; None counts one page), force_ocr (read pages as images even when they carry a text layer; used for a PDF
180
+ the native reader found without one). Returns: a Converted. Raises: DocumentError with the reason when no engine is
181
+ installed, the engine fails, is stopped at TIMEOUT_S or MEMORY_BYTES, or returns empty or thin text."""
182
+ engine, python = locate()
183
+ folder = cache_folder()
184
+ stem = os.path.join(folder, _key(path, engine, force_ocr))
185
+ pages = _cached(stem + ".json")
186
+ if pages is None:
187
+ _lock(stem + ".lock")
188
+ try:
189
+ pages = _cached(stem + ".json")
190
+ if pages is None:
191
+ pages = _checked(engine, _engine_pages(engine, python, path, stem + ".work", force_ocr), expected_pages)
192
+ atomic_io.write_text(stem + ".json", json.dumps({"pages": pages}))
193
+ finally:
194
+ os.remove(stem + ".lock")
195
+ return Converted(engine, _checked(engine, pages, expected_pages))
196
+
197
+
198
+ def _engine_pages(engine, python, path, workdir, force_ocr):
199
+ """The pages the engine writes for `path`, run in `workdir` (made and removed here).
200
+
201
+ Raises: DocumentError when the engine is stopped, fails or returns no pages."""
202
+ os.makedirs(workdir, exist_ok=True)
203
+ target = os.path.join(workdir, 'pages.json')
204
+ try:
205
+ try:
206
+ code, tail = process_limits.run_capped(
207
+ [python, '-I', '-c', ENGINES[engine][1], path, target, '1' if force_ocr else '0'], workdir, TIMEOUT_S,
208
+ MEMORY_BYTES)
209
+ except subprocess.TimeoutExpired:
210
+ raise document_text.DocumentError(f'{engine} did not finish in {TIMEOUT_S}s and was stopped.') from None
211
+ except OSError as exc:
212
+ raise document_text.DocumentError(f'{engine} could not be started: {exc}') from None
213
+ if code:
214
+ raise document_text.DocumentError(f'{engine} failed (exit {code}): {tail[-300:] or "no error text"}')
215
+ try:
216
+ with open(target, encoding='utf-8') as stream:
217
+ return json.load(stream)['pages']
218
+ except (OSError, ValueError, KeyError, TypeError):
219
+ raise document_text.DocumentError(f'{engine} finished without returning its pages.') from None
220
+ finally:
221
+ shutil.rmtree(workdir, ignore_errors=True)