@allansantos-dev/smart-tool 0.9.79 → 0.9.80
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/README.md +65 -18
- package/browser_control.py +10 -3
- package/browser_service_client.py +3 -32
- package/chromium_flags.py +30 -0
- package/code_health.py +180 -0
- package/code_impact.py +13 -1
- package/config.py +41 -8
- package/credential_check.py +38 -0
- package/dependency_notice.py +262 -14
- package/document_converter.py +221 -0
- package/document_read.py +100 -25
- package/document_text.py +194 -6
- package/dossier.py +35 -8
- package/embedding_cache.py +32 -8
- package/freshness.py +3 -2
- package/hook_decision.py +65 -14
- package/indexer.py +92 -3
- package/layer_check.py +57 -0
- package/local_embedder.py +114 -57
- package/locales/pt-BR.json +52 -17
- package/low_priority.py +41 -0
- package/package.json +1 -1
- package/pr_impact.py +111 -8
- package/process_limits.py +108 -0
- package/project_monitor.py +33 -7
- package/project_scheduler.py +12 -4
- package/scout.py +6 -2
- package/setup_audit.py +34 -1
- package/setup_ui.py +210 -39
- package/smart_tool_daemon.py +398 -91
- package/update_check.py +18 -0
- package/usage_meter.py +78 -0
- package/version.py +1 -1
- package/web_adapters/browser/crawl4ai_search.py +7 -3
- package/web_fetch.py +24 -0
- package/web_search_health.py +31 -0
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""Optional external converter for what the native reader of document_read refuses: a file type it does not read and a
|
|
2
|
+
PDF with no text layer. Off unless the `document_converter` setting turns it on (see config.document_converter).
|
|
3
|
+
|
|
4
|
+
The engine (Kreuzberg first, Docling as the drop-in) is never imported here: it runs in a separate interpreter,
|
|
5
|
+
isolated with `-I`, under a time cap and a memory cap, and hands its pages back through a file. Whatever it returns
|
|
6
|
+
that is empty or thinner than MIN_CHARS_PER_PAGE is an error, never a document."""
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import shutil
|
|
11
|
+
import subprocess
|
|
12
|
+
import sys
|
|
13
|
+
import time
|
|
14
|
+
|
|
15
|
+
import atomic_io
|
|
16
|
+
import config
|
|
17
|
+
import document_text
|
|
18
|
+
import process_limits
|
|
19
|
+
import version
|
|
20
|
+
|
|
21
|
+
EXTENSIONS = ('.png', '.jpg', '.jpeg', '.tif', '.tiff', '.bmp', '.gif', '.webp', '.doc', '.xls', '.ppt', '.odt', '.ods',
|
|
22
|
+
'.odp', '.rtf')
|
|
23
|
+
TIMEOUT_S = 600
|
|
24
|
+
MEMORY_BYTES = 4 * 1024 ** 3
|
|
25
|
+
MIN_CHARS_PER_PAGE = 40
|
|
26
|
+
DETECT_TIMEOUT_S = 30
|
|
27
|
+
LOCK_POLL_S = 0.25
|
|
28
|
+
STALE_LOCK_S = TIMEOUT_S + 60
|
|
29
|
+
|
|
30
|
+
KREUZBERG_SCRIPT = r"""
|
|
31
|
+
import json, re, sys
|
|
32
|
+
import kreuzberg
|
|
33
|
+
source, target, force_ocr = sys.argv[1], sys.argv[2], sys.argv[3] == "1"
|
|
34
|
+
settings = kreuzberg.ExtractionConfig(output_format="markdown", force_ocr=force_ocr,
|
|
35
|
+
pages=kreuzberg.PageConfig(extract_pages=True, insert_page_markers=True))
|
|
36
|
+
result = kreuzberg.extract_file_sync(source, config=settings)
|
|
37
|
+
split = re.split(r"\\?<\\?!-- PAGE \d+ --\\?>", result.content)
|
|
38
|
+
pages = [page.strip() for page in split[1:]] if len(split) > 1 else [result.content]
|
|
39
|
+
with open(target, "w", encoding="utf-8") as stream:
|
|
40
|
+
json.dump({"pages": pages}, stream)
|
|
41
|
+
"""
|
|
42
|
+
|
|
43
|
+
DOCLING_SCRIPT = """
|
|
44
|
+
import json, sys
|
|
45
|
+
from docling.document_converter import DocumentConverter
|
|
46
|
+
source, target = sys.argv[1], sys.argv[2]
|
|
47
|
+
marker = "<!-- page break -->"
|
|
48
|
+
text = DocumentConverter().convert(source).document.export_to_markdown(page_break_placeholder=marker)
|
|
49
|
+
with open(target, "w", encoding="utf-8") as stream:
|
|
50
|
+
json.dump({"pages": text.split(marker)}, stream)
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
ENGINES = {'kreuzberg': ('kreuzberg', KREUZBERG_SCRIPT), 'docling': ('docling', DOCLING_SCRIPT)}
|
|
54
|
+
_found = set()
|
|
55
|
+
_swept = set()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Converted:
|
|
59
|
+
"""The pages an engine returned for a file: the engine's name and the text of each page."""
|
|
60
|
+
|
|
61
|
+
def __init__(self, engine, pages):
|
|
62
|
+
self.engine = engine
|
|
63
|
+
self.pages = pages
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def enabled():
|
|
67
|
+
"""Whether the `document_converter` setting is anything but off."""
|
|
68
|
+
return config.document_converter()['mode'] != 'off'
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def _installed(python, module):
|
|
72
|
+
"""Whether `module` can be imported by `python`; a positive answer is kept, a negative one is asked again."""
|
|
73
|
+
if (python, module) in _found:
|
|
74
|
+
return True
|
|
75
|
+
probe = 'import importlib.util, sys; sys.exit(0 if importlib.util.find_spec(sys.argv[1]) else 1)'
|
|
76
|
+
try:
|
|
77
|
+
done = subprocess.run([python, '-I', '-c', probe, module], capture_output=True, timeout=DETECT_TIMEOUT_S,
|
|
78
|
+
creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0))
|
|
79
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
80
|
+
return False
|
|
81
|
+
if done.returncode == 0:
|
|
82
|
+
_found.add((python, module))
|
|
83
|
+
return done.returncode == 0
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def locate():
|
|
87
|
+
"""The engine to use and the interpreter that has it.
|
|
88
|
+
|
|
89
|
+
Returns: `(name, python)`. Raises: DocumentError naming what was looked for and where when the setting is off, names
|
|
90
|
+
an engine that is not installed, or `auto` finds none."""
|
|
91
|
+
setting = config.document_converter()
|
|
92
|
+
python = setting['python'] or sys.executable
|
|
93
|
+
wanted = list(ENGINES) if setting['mode'] == 'auto' else [setting['mode']]
|
|
94
|
+
if wanted == ['off']:
|
|
95
|
+
raise document_text.DocumentError('The document converter is off.')
|
|
96
|
+
for name in wanted:
|
|
97
|
+
if _installed(python, ENGINES[name][0]):
|
|
98
|
+
return name, python
|
|
99
|
+
raise document_text.DocumentError(
|
|
100
|
+
f"The document converter is set to {setting['mode']} but {' or '.join(wanted)} is not installed in {python}.")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _checked(engine, pages, expected_pages):
|
|
104
|
+
"""The pages if the engine returned enough text, else a DocumentError: under MIN_CHARS_PER_PAGE for each page of the
|
|
105
|
+
document is a failed read, not a document."""
|
|
106
|
+
if not isinstance(pages, list) or not all(isinstance(page, str) for page in pages):
|
|
107
|
+
raise document_text.DocumentError(f'{engine} returned an answer that is not a list of pages.')
|
|
108
|
+
found = sum(len(page.strip()) for page in pages)
|
|
109
|
+
needed = MIN_CHARS_PER_PAGE * max(1, expected_pages or 1)
|
|
110
|
+
if found < needed:
|
|
111
|
+
raise document_text.DocumentError(
|
|
112
|
+
f'{engine} returned {found} characters for {max(1, expected_pages or 1)} page(s), under the {needed} '
|
|
113
|
+
'it takes to count as text: the file was not read.')
|
|
114
|
+
return pages
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def cache_folder():
|
|
118
|
+
"""The one folder of this version's converted pages: `%LOCALAPPDATA%\\SmartTool\\cache\\<version>\\convert`, created
|
|
119
|
+
here; the folders of other versions are deleted the first time it is asked for in a process.
|
|
120
|
+
|
|
121
|
+
Raises: RuntimeError when LOCALAPPDATA is not set."""
|
|
122
|
+
local = os.environ.get("LOCALAPPDATA")
|
|
123
|
+
if not local:
|
|
124
|
+
raise RuntimeError("LOCALAPPDATA is not set: there is no per-user folder for the converter's cache.")
|
|
125
|
+
root = os.path.join(local, "SmartTool", "cache")
|
|
126
|
+
folder = os.path.join(root, version.VERSION, "convert")
|
|
127
|
+
os.makedirs(folder, exist_ok=True)
|
|
128
|
+
if root not in _swept:
|
|
129
|
+
_swept.add(root)
|
|
130
|
+
for name in os.listdir(root):
|
|
131
|
+
if name != version.VERSION:
|
|
132
|
+
shutil.rmtree(os.path.join(root, name), ignore_errors=True)
|
|
133
|
+
return folder
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _key(path, engine, force_ocr):
|
|
137
|
+
"""The name of the result for this content, engine and OCR flag: a SHA-256 over them, never the file's name or place."""
|
|
138
|
+
digest = hashlib.sha256(f"{engine}\0{int(force_ocr)}\0".encode())
|
|
139
|
+
with open(path, "rb") as stream:
|
|
140
|
+
for block in iter(lambda: stream.read(1 << 20), b""):
|
|
141
|
+
digest.update(block)
|
|
142
|
+
return digest.hexdigest()[:40]
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _lock(path):
|
|
146
|
+
"""Takes the lock file `path` (created exclusively), waiting for another run that holds it; a lock older than
|
|
147
|
+
STALE_LOCK_S belongs to a run that died and is taken over."""
|
|
148
|
+
deadline = time.monotonic() + STALE_LOCK_S
|
|
149
|
+
while True:
|
|
150
|
+
try:
|
|
151
|
+
os.close(os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY))
|
|
152
|
+
return
|
|
153
|
+
except FileExistsError:
|
|
154
|
+
try:
|
|
155
|
+
if time.time() - os.path.getmtime(path) > STALE_LOCK_S:
|
|
156
|
+
os.remove(path)
|
|
157
|
+
continue
|
|
158
|
+
except OSError:
|
|
159
|
+
continue
|
|
160
|
+
if time.monotonic() > deadline:
|
|
161
|
+
raise document_text.DocumentError(f"Another conversion of the same file held {path} for over {STALE_LOCK_S}s.")
|
|
162
|
+
time.sleep(LOCK_POLL_S)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _cached(result):
|
|
166
|
+
"""The pages saved at `result`, or None when there is no such file or it is not a list of pages."""
|
|
167
|
+
try:
|
|
168
|
+
with open(result, encoding="utf-8") as stream:
|
|
169
|
+
pages = json.load(stream)["pages"]
|
|
170
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
171
|
+
return None
|
|
172
|
+
return pages if isinstance(pages, list) and all(isinstance(page, str) for page in pages) else None
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def convert(path, expected_pages=None, force_ocr=False):
|
|
176
|
+
"""The text of the file at `path` through the engine the setting picks, in an isolated subprocess.
|
|
177
|
+
|
|
178
|
+
Parameters: path (absolute path of the file), expected_pages (page count of the document when known, for the minimum
|
|
179
|
+
of text; None counts one page), force_ocr (read pages as images even when they carry a text layer; used for a PDF
|
|
180
|
+
the native reader found without one). Returns: a Converted. Raises: DocumentError with the reason when no engine is
|
|
181
|
+
installed, the engine fails, is stopped at TIMEOUT_S or MEMORY_BYTES, or returns empty or thin text."""
|
|
182
|
+
engine, python = locate()
|
|
183
|
+
folder = cache_folder()
|
|
184
|
+
stem = os.path.join(folder, _key(path, engine, force_ocr))
|
|
185
|
+
pages = _cached(stem + ".json")
|
|
186
|
+
if pages is None:
|
|
187
|
+
_lock(stem + ".lock")
|
|
188
|
+
try:
|
|
189
|
+
pages = _cached(stem + ".json")
|
|
190
|
+
if pages is None:
|
|
191
|
+
pages = _checked(engine, _engine_pages(engine, python, path, stem + ".work", force_ocr), expected_pages)
|
|
192
|
+
atomic_io.write_text(stem + ".json", json.dumps({"pages": pages}))
|
|
193
|
+
finally:
|
|
194
|
+
os.remove(stem + ".lock")
|
|
195
|
+
return Converted(engine, _checked(engine, pages, expected_pages))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _engine_pages(engine, python, path, workdir, force_ocr):
|
|
199
|
+
"""The pages the engine writes for `path`, run in `workdir` (made and removed here).
|
|
200
|
+
|
|
201
|
+
Raises: DocumentError when the engine is stopped, fails or returns no pages."""
|
|
202
|
+
os.makedirs(workdir, exist_ok=True)
|
|
203
|
+
target = os.path.join(workdir, 'pages.json')
|
|
204
|
+
try:
|
|
205
|
+
try:
|
|
206
|
+
code, tail = process_limits.run_capped(
|
|
207
|
+
[python, '-I', '-c', ENGINES[engine][1], path, target, '1' if force_ocr else '0'], workdir, TIMEOUT_S,
|
|
208
|
+
MEMORY_BYTES)
|
|
209
|
+
except subprocess.TimeoutExpired:
|
|
210
|
+
raise document_text.DocumentError(f'{engine} did not finish in {TIMEOUT_S}s and was stopped.') from None
|
|
211
|
+
except OSError as exc:
|
|
212
|
+
raise document_text.DocumentError(f'{engine} could not be started: {exc}') from None
|
|
213
|
+
if code:
|
|
214
|
+
raise document_text.DocumentError(f'{engine} failed (exit {code}): {tail[-300:] or "no error text"}')
|
|
215
|
+
try:
|
|
216
|
+
with open(target, encoding='utf-8') as stream:
|
|
217
|
+
return json.load(stream)['pages']
|
|
218
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
219
|
+
raise document_text.DocumentError(f'{engine} finished without returning its pages.') from None
|
|
220
|
+
finally:
|
|
221
|
+
shutil.rmtree(workdir, ignore_errors=True)
|
package/document_read.py
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
"""read_document: PDF, Word, PowerPoint and
|
|
1
|
+
"""read_document: PDF, Word, PowerPoint, Excel, HTML, CSV and EPUB files read as compact text, one page, slide, sheet,
|
|
2
|
+
part or chapter range at a time.
|
|
2
3
|
|
|
3
4
|
The agent gets the file facts first (type, size, pages, slides or sheets, estimated tokens) and then only the range it
|
|
4
5
|
asks for, so a 200-page PDF is never loaded whole. Extraction is the one of document_text (pypdf in an isolated
|
|
5
|
-
subprocess for PDF; zipfile and ElementTree with the shared guards for docx, pptx and
|
|
6
|
-
split, the markdown shape, the compaction of PDF line wrapping and the range arithmetic."""
|
|
6
|
+
subprocess for PDF; zipfile and ElementTree with the shared guards for docx, pptx, xlsx and epub; the standard HTML
|
|
7
|
+
and CSV parsers for html and csv); this module adds the unit split, the markdown shape, the compaction of PDF line wrapping and the range arithmetic."""
|
|
7
8
|
import collections
|
|
8
9
|
import math
|
|
9
10
|
import os
|
|
@@ -11,10 +12,14 @@ import re
|
|
|
11
12
|
import threading
|
|
12
13
|
import unicodedata
|
|
13
14
|
|
|
15
|
+
import document_converter
|
|
14
16
|
import document_text
|
|
15
17
|
|
|
16
|
-
KINDS = {'.pdf': 'pdf', '.docx': 'docx', '.pptx': 'pptx', '.xlsx': 'xlsx'
|
|
17
|
-
|
|
18
|
+
KINDS = {'.pdf': 'pdf', '.docx': 'docx', '.pptx': 'pptx', '.xlsx': 'xlsx', '.xlsm': 'xlsx', '.html': 'html', '.htm': 'html',
|
|
19
|
+
'.csv': 'csv', '.epub': 'epub'}
|
|
20
|
+
UNIT_NAMES = {'pdf': 'page', 'docx': 'part', 'pptx': 'slide', 'xlsx': 'sheet', 'html': 'part', 'csv': 'sheet',
|
|
21
|
+
'epub': 'chapter', 'converted': 'part'}
|
|
22
|
+
LABELLED_KINDS = ('pdf', 'docx', 'html', 'epub', 'converted')
|
|
18
23
|
DEFAULT_MAX_CHARS = 20_000
|
|
19
24
|
MAX_CHARS_LIMIT = 100_000
|
|
20
25
|
DOCX_PART_CHARS = 6_000
|
|
@@ -26,6 +31,14 @@ _cache = collections.OrderedDict()
|
|
|
26
31
|
_cache_lock = threading.Lock()
|
|
27
32
|
|
|
28
33
|
|
|
34
|
+
class NoTextLayer(document_text.DocumentError):
|
|
35
|
+
"""A PDF whose pages carry no text: `pages` is its page count."""
|
|
36
|
+
|
|
37
|
+
def __init__(self, pages):
|
|
38
|
+
super().__init__(f'PDF of {pages} pages has no text layer (scanned pages are not read; there is no OCR).')
|
|
39
|
+
self.pages = pages
|
|
40
|
+
|
|
41
|
+
|
|
29
42
|
class Unit:
|
|
30
43
|
"""One readable unit of a document: a PDF page, a docx part, a slide or a sheet."""
|
|
31
44
|
|
|
@@ -74,13 +87,12 @@ def compact_pdf_text(text):
|
|
|
74
87
|
|
|
75
88
|
|
|
76
89
|
def _pdf_units(data):
|
|
77
|
-
"""Units and facts of a PDF, one unit per page; raises DocumentError for a password
|
|
78
|
-
|
|
90
|
+
"""Units and facts of a PDF, one unit per page; raises DocumentError for a password or a corrupt file, and NoTextLayer
|
|
91
|
+
for a file with no text layer at all."""
|
|
79
92
|
pages, count = document_text.pdf_pages(data, document_text.PDF_TIMEOUT_S, document_text.PDF_MAX_PAGES,
|
|
80
93
|
document_text.PDF_MAX_CHARS)
|
|
81
94
|
if not any(pages):
|
|
82
|
-
raise
|
|
83
|
-
f'PDF of {count} pages has no text layer (scanned pages are not read; there is no OCR).')
|
|
95
|
+
raise NoTextLayer(count)
|
|
84
96
|
units = [Unit(number, compact_pdf_text(page) or '(no text on this page)') for number, page in enumerate(pages, 1)]
|
|
85
97
|
facts = {'pages': count, 'pages without text': sum(1 for page in pages if not page)}
|
|
86
98
|
if count > len(pages):
|
|
@@ -89,21 +101,47 @@ def _pdf_units(data):
|
|
|
89
101
|
return units, facts
|
|
90
102
|
|
|
91
103
|
|
|
92
|
-
def
|
|
93
|
-
"""
|
|
94
|
-
split (a docx has no fixed pages, they depend on the renderer)."""
|
|
104
|
+
def _parts(blocks):
|
|
105
|
+
"""The markdown blocks grouped in parts of about DOCX_PART_CHARS characters, a block never split."""
|
|
95
106
|
parts, current, size = [], [], 0
|
|
96
|
-
for block in
|
|
107
|
+
for block in blocks:
|
|
97
108
|
if current and size + len(block) > DOCX_PART_CHARS:
|
|
98
109
|
parts.append('\n'.join(current))
|
|
99
110
|
current, size = [], 0
|
|
100
111
|
current.append(block)
|
|
101
112
|
size += len(block) + 1
|
|
102
113
|
parts.append('\n'.join(current))
|
|
114
|
+
return parts
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _docx_units(data):
|
|
118
|
+
"""Units and facts of a docx: the markdown blocks grouped in parts (a docx has no fixed pages, they depend on the
|
|
119
|
+
renderer)."""
|
|
120
|
+
parts = _parts(document_text.docx_blocks(data))
|
|
103
121
|
return [Unit(number, text) for number, text in enumerate(parts, 1)], {
|
|
104
122
|
'parts': len(parts), 'note': f'a docx has no fixed pages; a part is about {DOCX_PART_CHARS:,} characters'}
|
|
105
123
|
|
|
106
124
|
|
|
125
|
+
def _html_units(data):
|
|
126
|
+
"""Units and facts of an html file: the markdown blocks grouped in parts, as a docx (an html file has no pages)."""
|
|
127
|
+
parts = _parts(document_text.html_blocks(data))
|
|
128
|
+
return [Unit(number, text) for number, text in enumerate(parts, 1)], {
|
|
129
|
+
'parts': len(parts), 'note': f'a part is about {DOCX_PART_CHARS:,} characters of the page text'}
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _csv_units(data):
|
|
133
|
+
"""Units and facts of a csv file: one unit, the sheet `csv`, one tab-separated line per row."""
|
|
134
|
+
rows = document_text.csv_rows(data)
|
|
135
|
+
return [Unit(1, '\n'.join(['[sheet csv]'] + rows), 'csv')], {'rows': len(rows)}
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _epub_units(data):
|
|
139
|
+
"""Units and facts of an epub: one unit per chapter of the reading order."""
|
|
140
|
+
chapters = document_text.epub_chapters(data)
|
|
141
|
+
units = [Unit(number, '\n'.join(blocks) or '(no text)') for number, blocks in enumerate(chapters, 1)]
|
|
142
|
+
return units, {'chapters without text': sum(1 for blocks in chapters if not blocks)}
|
|
143
|
+
|
|
144
|
+
|
|
107
145
|
def _pptx_units(data):
|
|
108
146
|
"""Units and facts of a pptx: one unit per slide under `[slide N]`, the speaker notes under `[notes N]`."""
|
|
109
147
|
units = []
|
|
@@ -121,14 +159,51 @@ def _xlsx_units(data):
|
|
|
121
159
|
return units, {'sheet names': ', '.join(f'{u.number}={u.name}' for u in units)}
|
|
122
160
|
|
|
123
161
|
|
|
124
|
-
|
|
162
|
+
def _converted_pdf_units(path, count):
|
|
163
|
+
"""Units and facts of a PDF with no text layer, read by the external converter as page images."""
|
|
164
|
+
result = document_converter.convert(path, expected_pages=count, force_ocr=True)
|
|
165
|
+
units = [Unit(number, page.strip() or '(no text on this page)') for number, page in enumerate(result.pages, 1)]
|
|
166
|
+
facts = {'pages without text': sum(1 for page in result.pages if not page.strip()),
|
|
167
|
+
'read by': f'{result.engine} (OCR of the page images)'}
|
|
168
|
+
if len(units) != count:
|
|
169
|
+
facts['note'] = f'the converter returned {len(units)} pages for a file of {count}'
|
|
170
|
+
return units, facts
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def _converted_units(path):
|
|
174
|
+
"""Units and facts of a file the native reader has no type for, read by the external converter: its text grouped in
|
|
175
|
+
parts as a docx."""
|
|
176
|
+
result = document_converter.convert(path)
|
|
177
|
+
text = '\n\n'.join(page.strip() for page in result.pages if page.strip())
|
|
178
|
+
parts = _parts(text.split('\n'))
|
|
179
|
+
return [Unit(number, part) for number, part in enumerate(parts, 1)], {'parts': len(parts),
|
|
180
|
+
'read by': result.engine}
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def _build(path, kind):
|
|
184
|
+
"""Units and facts of the file at `path`: its native builder, or, for a kind the native reader has none for and for a
|
|
185
|
+
PDF without a text layer, the external converter when the setting turns it on."""
|
|
186
|
+
if kind == 'converted':
|
|
187
|
+
return _converted_units(path)
|
|
188
|
+
with open(path, 'rb') as stream:
|
|
189
|
+
data = stream.read()
|
|
190
|
+
try:
|
|
191
|
+
return _BUILDERS[kind](data)
|
|
192
|
+
except NoTextLayer as exc:
|
|
193
|
+
if not document_converter.enabled():
|
|
194
|
+
raise
|
|
195
|
+
return _converted_pdf_units(path, exc.pages)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
_BUILDERS = {'pdf': _pdf_units, 'docx': _docx_units, 'pptx': _pptx_units, 'xlsx': _xlsx_units, 'html': _html_units,
|
|
199
|
+
'csv': _csv_units, 'epub': _epub_units}
|
|
125
200
|
|
|
126
201
|
|
|
127
202
|
def load(path):
|
|
128
203
|
"""The document at `path`, split into units. Parsed documents are kept for a few calls (keyed by path, size and
|
|
129
204
|
modification time), so reading page after page parses the file once.
|
|
130
205
|
|
|
131
|
-
Parameters: path (absolute path of a .pdf, .docx, .pptx or .
|
|
206
|
+
Parameters: path (absolute path of a .pdf, .docx, .pptx, .xlsx, .html, .htm, .csv or .epub file). Returns: a Document. Raises: ValueError for a
|
|
132
207
|
relative path, a missing file, an unsupported type or a file over the size limit of its type; DocumentError with a
|
|
133
208
|
readable reason for a file the extractor refuses (corrupt, encrypted, hostile, scanned, without text)."""
|
|
134
209
|
if not isinstance(path, str) or not path.strip():
|
|
@@ -136,27 +211,27 @@ def load(path):
|
|
|
136
211
|
if not os.path.isabs(path):
|
|
137
212
|
raise ValueError(f'read_document needs an absolute path, got {path!r}')
|
|
138
213
|
extension = os.path.splitext(path)[1].lower()
|
|
139
|
-
|
|
214
|
+
converted = extension not in KINDS and extension in document_converter.EXTENSIONS and document_converter.enabled()
|
|
215
|
+
if extension not in KINDS and not converted:
|
|
140
216
|
raise ValueError(f"read_document reads {', '.join(KINDS)} files; {extension or 'this file'} is not supported "
|
|
141
|
-
'(use the normal Read for plain text; legacy .doc, .xls and .ppt and macro files are not read)')
|
|
217
|
+
'(use the normal Read for plain text; legacy .doc, .xls, .xlsb and .ppt and macro files other than .xlsm are not read)')
|
|
142
218
|
try:
|
|
143
219
|
info = os.stat(path)
|
|
144
220
|
except OSError as exc:
|
|
145
221
|
raise ValueError(f'cannot read {path}: {exc.strerror or type(exc).__name__}') from None
|
|
146
222
|
if not os.path.isfile(path):
|
|
147
223
|
raise ValueError(f'{path} is not a file')
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
224
|
+
limit = document_text.PDF_MAX_BYTES if converted else document_text.size_limit(path)
|
|
225
|
+
if info.st_size > limit:
|
|
226
|
+
raise ValueError(f'{path} is {info.st_size / 1048576:.1f} MiB, over the {limit // 1048576} MiB limit of a '
|
|
227
|
+
f'{extension} file')
|
|
151
228
|
key = (os.path.normcase(path), info.st_size, info.st_mtime_ns)
|
|
152
229
|
with _cache_lock:
|
|
153
230
|
if key in _cache:
|
|
154
231
|
_cache.move_to_end(key)
|
|
155
232
|
return _cache[key]
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
kind = KINDS[extension]
|
|
159
|
-
units, facts = _BUILDERS[kind](data)
|
|
233
|
+
kind = 'converted' if converted else KINDS[extension]
|
|
234
|
+
units, facts = _build(path, kind)
|
|
160
235
|
document = Document(path, kind, info.st_size, units, facts)
|
|
161
236
|
with _cache_lock:
|
|
162
237
|
_cache[key] = document
|
|
@@ -241,7 +316,7 @@ def read(path, range_spec=None, max_chars=DEFAULT_MAX_CHARS, info_only=False):
|
|
|
241
316
|
body, shown, size, cut = [], [], 0, None
|
|
242
317
|
for number in numbers:
|
|
243
318
|
text = document.units[number - 1].text
|
|
244
|
-
label = f'[{unit} {number}]\n' if document.kind in
|
|
319
|
+
label = f'[{unit} {number}]\n' if document.kind in LABELLED_KINDS else ''
|
|
245
320
|
piece = label + text
|
|
246
321
|
if shown and size + len(piece) > max_chars:
|
|
247
322
|
break
|