@allansantos-dev/smart-tool 0.9.79 → 0.9.80

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,221 @@
1
+ """Optional external converter for what the native reader of document_read refuses: a file type it does not read and a
2
+ PDF with no text layer. Off unless the `document_converter` setting turns it on (see config.document_converter).
3
+
4
+ The engine (Kreuzberg first, Docling as the drop-in) is never imported here: it runs in a separate interpreter,
5
+ isolated with `-I`, under a time cap and a memory cap, and hands its pages back through a file. Whatever it returns
6
+ that is empty or thinner than MIN_CHARS_PER_PAGE is an error, never a document."""
7
+ import hashlib
8
+ import json
9
+ import os
10
+ import shutil
11
+ import subprocess
12
+ import sys
13
+ import time
14
+
15
+ import atomic_io
16
+ import config
17
+ import document_text
18
+ import process_limits
19
+ import version
20
+
21
+ EXTENSIONS = ('.png', '.jpg', '.jpeg', '.tif', '.tiff', '.bmp', '.gif', '.webp', '.doc', '.xls', '.ppt', '.odt', '.ods',
22
+ '.odp', '.rtf')
23
+ TIMEOUT_S = 600
24
+ MEMORY_BYTES = 4 * 1024 ** 3
25
+ MIN_CHARS_PER_PAGE = 40
26
+ DETECT_TIMEOUT_S = 30
27
+ LOCK_POLL_S = 0.25
28
+ STALE_LOCK_S = TIMEOUT_S + 60
29
+
30
+ KREUZBERG_SCRIPT = r"""
31
+ import json, re, sys
32
+ import kreuzberg
33
+ source, target, force_ocr = sys.argv[1], sys.argv[2], sys.argv[3] == "1"
34
+ settings = kreuzberg.ExtractionConfig(output_format="markdown", force_ocr=force_ocr,
35
+ pages=kreuzberg.PageConfig(extract_pages=True, insert_page_markers=True))
36
+ result = kreuzberg.extract_file_sync(source, config=settings)
37
+ split = re.split(r"\\?<\\?!-- PAGE \d+ --\\?>", result.content)
38
+ pages = [page.strip() for page in split[1:]] if len(split) > 1 else [result.content]
39
+ with open(target, "w", encoding="utf-8") as stream:
40
+ json.dump({"pages": pages}, stream)
41
+ """
42
+
43
+ DOCLING_SCRIPT = """
44
+ import json, sys
45
+ from docling.document_converter import DocumentConverter
46
+ source, target = sys.argv[1], sys.argv[2]
47
+ marker = "<!-- page break -->"
48
+ text = DocumentConverter().convert(source).document.export_to_markdown(page_break_placeholder=marker)
49
+ with open(target, "w", encoding="utf-8") as stream:
50
+ json.dump({"pages": text.split(marker)}, stream)
51
+ """
52
+
53
+ ENGINES = {'kreuzberg': ('kreuzberg', KREUZBERG_SCRIPT), 'docling': ('docling', DOCLING_SCRIPT)}
54
+ _found = set()
55
+ _swept = set()
56
+
57
+
58
+ class Converted:
59
+ """The pages an engine returned for a file: the engine's name and the text of each page."""
60
+
61
+ def __init__(self, engine, pages):
62
+ self.engine = engine
63
+ self.pages = pages
64
+
65
+
66
+ def enabled():
67
+ """Whether the `document_converter` setting is anything but off."""
68
+ return config.document_converter()['mode'] != 'off'
69
+
70
+
71
+ def _installed(python, module):
72
+ """Whether `module` can be imported by `python`; a positive answer is kept, a negative one is asked again."""
73
+ if (python, module) in _found:
74
+ return True
75
+ probe = 'import importlib.util, sys; sys.exit(0 if importlib.util.find_spec(sys.argv[1]) else 1)'
76
+ try:
77
+ done = subprocess.run([python, '-I', '-c', probe, module], capture_output=True, timeout=DETECT_TIMEOUT_S,
78
+ creationflags=getattr(subprocess, 'CREATE_NO_WINDOW', 0))
79
+ except (OSError, subprocess.TimeoutExpired):
80
+ return False
81
+ if done.returncode == 0:
82
+ _found.add((python, module))
83
+ return done.returncode == 0
84
+
85
+
86
+ def locate():
87
+ """The engine to use and the interpreter that has it.
88
+
89
+ Returns: `(name, python)`. Raises: DocumentError naming what was looked for and where when the setting is off, names
90
+ an engine that is not installed, or `auto` finds none."""
91
+ setting = config.document_converter()
92
+ python = setting['python'] or sys.executable
93
+ wanted = list(ENGINES) if setting['mode'] == 'auto' else [setting['mode']]
94
+ if wanted == ['off']:
95
+ raise document_text.DocumentError('The document converter is off.')
96
+ for name in wanted:
97
+ if _installed(python, ENGINES[name][0]):
98
+ return name, python
99
+ raise document_text.DocumentError(
100
+ f"The document converter is set to {setting['mode']} but {' or '.join(wanted)} is not installed in {python}.")
101
+
102
+
103
+ def _checked(engine, pages, expected_pages):
104
+ """The pages if the engine returned enough text, else a DocumentError: under MIN_CHARS_PER_PAGE for each page of the
105
+ document is a failed read, not a document."""
106
+ if not isinstance(pages, list) or not all(isinstance(page, str) for page in pages):
107
+ raise document_text.DocumentError(f'{engine} returned an answer that is not a list of pages.')
108
+ found = sum(len(page.strip()) for page in pages)
109
+ needed = MIN_CHARS_PER_PAGE * max(1, expected_pages or 1)
110
+ if found < needed:
111
+ raise document_text.DocumentError(
112
+ f'{engine} returned {found} characters for {max(1, expected_pages or 1)} page(s), under the {needed} '
113
+ 'it takes to count as text: the file was not read.')
114
+ return pages
115
+
116
+
117
+ def cache_folder():
118
+ """The one folder of this version's converted pages: `%LOCALAPPDATA%\\SmartTool\\cache\\<version>\\convert`, created
119
+ here; the folders of other versions are deleted the first time it is asked for in a process.
120
+
121
+ Raises: RuntimeError when LOCALAPPDATA is not set."""
122
+ local = os.environ.get("LOCALAPPDATA")
123
+ if not local:
124
+ raise RuntimeError("LOCALAPPDATA is not set: there is no per-user folder for the converter's cache.")
125
+ root = os.path.join(local, "SmartTool", "cache")
126
+ folder = os.path.join(root, version.VERSION, "convert")
127
+ os.makedirs(folder, exist_ok=True)
128
+ if root not in _swept:
129
+ _swept.add(root)
130
+ for name in os.listdir(root):
131
+ if name != version.VERSION:
132
+ shutil.rmtree(os.path.join(root, name), ignore_errors=True)
133
+ return folder
134
+
135
+
136
+ def _key(path, engine, force_ocr):
137
+ """The name of the result for this content, engine and OCR flag: a SHA-256 over them, never the file's name or place."""
138
+ digest = hashlib.sha256(f"{engine}\0{int(force_ocr)}\0".encode())
139
+ with open(path, "rb") as stream:
140
+ for block in iter(lambda: stream.read(1 << 20), b""):
141
+ digest.update(block)
142
+ return digest.hexdigest()[:40]
143
+
144
+
145
+ def _lock(path):
146
+ """Takes the lock file `path` (created exclusively), waiting for another run that holds it; a lock older than
147
+ STALE_LOCK_S belongs to a run that died and is taken over."""
148
+ deadline = time.monotonic() + STALE_LOCK_S
149
+ while True:
150
+ try:
151
+ os.close(os.open(path, os.O_CREAT | os.O_EXCL | os.O_WRONLY))
152
+ return
153
+ except FileExistsError:
154
+ try:
155
+ if time.time() - os.path.getmtime(path) > STALE_LOCK_S:
156
+ os.remove(path)
157
+ continue
158
+ except OSError:
159
+ continue
160
+ if time.monotonic() > deadline:
161
+ raise document_text.DocumentError(f"Another conversion of the same file held {path} for over {STALE_LOCK_S}s.")
162
+ time.sleep(LOCK_POLL_S)
163
+
164
+
165
+ def _cached(result):
166
+ """The pages saved at `result`, or None when there is no such file or it is not a list of pages."""
167
+ try:
168
+ with open(result, encoding="utf-8") as stream:
169
+ pages = json.load(stream)["pages"]
170
+ except (OSError, ValueError, KeyError, TypeError):
171
+ return None
172
+ return pages if isinstance(pages, list) and all(isinstance(page, str) for page in pages) else None
173
+
174
+
175
+ def convert(path, expected_pages=None, force_ocr=False):
176
+ """The text of the file at `path` through the engine the setting picks, in an isolated subprocess.
177
+
178
+ Parameters: path (absolute path of the file), expected_pages (page count of the document when known, for the minimum
179
+ of text; None counts one page), force_ocr (read pages as images even when they carry a text layer; used for a PDF
180
+ the native reader found without one). Returns: a Converted. Raises: DocumentError with the reason when no engine is
181
+ installed, the engine fails, is stopped at TIMEOUT_S or MEMORY_BYTES, or returns empty or thin text."""
182
+ engine, python = locate()
183
+ folder = cache_folder()
184
+ stem = os.path.join(folder, _key(path, engine, force_ocr))
185
+ pages = _cached(stem + ".json")
186
+ if pages is None:
187
+ _lock(stem + ".lock")
188
+ try:
189
+ pages = _cached(stem + ".json")
190
+ if pages is None:
191
+ pages = _checked(engine, _engine_pages(engine, python, path, stem + ".work", force_ocr), expected_pages)
192
+ atomic_io.write_text(stem + ".json", json.dumps({"pages": pages}))
193
+ finally:
194
+ os.remove(stem + ".lock")
195
+ return Converted(engine, _checked(engine, pages, expected_pages))
196
+
197
+
198
+ def _engine_pages(engine, python, path, workdir, force_ocr):
199
+ """The pages the engine writes for `path`, run in `workdir` (made and removed here).
200
+
201
+ Raises: DocumentError when the engine is stopped, fails or returns no pages."""
202
+ os.makedirs(workdir, exist_ok=True)
203
+ target = os.path.join(workdir, 'pages.json')
204
+ try:
205
+ try:
206
+ code, tail = process_limits.run_capped(
207
+ [python, '-I', '-c', ENGINES[engine][1], path, target, '1' if force_ocr else '0'], workdir, TIMEOUT_S,
208
+ MEMORY_BYTES)
209
+ except subprocess.TimeoutExpired:
210
+ raise document_text.DocumentError(f'{engine} did not finish in {TIMEOUT_S}s and was stopped.') from None
211
+ except OSError as exc:
212
+ raise document_text.DocumentError(f'{engine} could not be started: {exc}') from None
213
+ if code:
214
+ raise document_text.DocumentError(f'{engine} failed (exit {code}): {tail[-300:] or "no error text"}')
215
+ try:
216
+ with open(target, encoding='utf-8') as stream:
217
+ return json.load(stream)['pages']
218
+ except (OSError, ValueError, KeyError, TypeError):
219
+ raise document_text.DocumentError(f'{engine} finished without returning its pages.') from None
220
+ finally:
221
+ shutil.rmtree(workdir, ignore_errors=True)
package/document_read.py CHANGED
@@ -1,9 +1,10 @@
1
- """read_document: PDF, Word, PowerPoint and Excel files read as compact text, one page, slide or sheet range at a time.
1
+ """read_document: PDF, Word, PowerPoint, Excel, HTML, CSV and EPUB files read as compact text, one page, slide, sheet,
2
+ part or chapter range at a time.
2
3
 
3
4
  The agent gets the file facts first (type, size, pages, slides or sheets, estimated tokens) and then only the range it
4
5
  asks for, so a 200-page PDF is never loaded whole. Extraction is the one of document_text (pypdf in an isolated
5
- subprocess for PDF; zipfile and ElementTree with the shared guards for docx, pptx and xlsx); this module adds the unit
6
- split, the markdown shape, the compaction of PDF line wrapping and the range arithmetic."""
6
+ subprocess for PDF; zipfile and ElementTree with the shared guards for docx, pptx, xlsx and epub; the standard HTML
7
+ and CSV parsers for html and csv); this module adds the unit split, the markdown shape, the compaction of PDF line wrapping and the range arithmetic."""
7
8
  import collections
8
9
  import math
9
10
  import os
@@ -11,10 +12,14 @@ import re
11
12
  import threading
12
13
  import unicodedata
13
14
 
15
+ import document_converter
14
16
  import document_text
15
17
 
16
- KINDS = {'.pdf': 'pdf', '.docx': 'docx', '.pptx': 'pptx', '.xlsx': 'xlsx'}
17
- UNIT_NAMES = {'pdf': 'page', 'docx': 'part', 'pptx': 'slide', 'xlsx': 'sheet'}
18
+ KINDS = {'.pdf': 'pdf', '.docx': 'docx', '.pptx': 'pptx', '.xlsx': 'xlsx', '.xlsm': 'xlsx', '.html': 'html', '.htm': 'html',
19
+ '.csv': 'csv', '.epub': 'epub'}
20
+ UNIT_NAMES = {'pdf': 'page', 'docx': 'part', 'pptx': 'slide', 'xlsx': 'sheet', 'html': 'part', 'csv': 'sheet',
21
+ 'epub': 'chapter', 'converted': 'part'}
22
+ LABELLED_KINDS = ('pdf', 'docx', 'html', 'epub', 'converted')
18
23
  DEFAULT_MAX_CHARS = 20_000
19
24
  MAX_CHARS_LIMIT = 100_000
20
25
  DOCX_PART_CHARS = 6_000
@@ -26,6 +31,14 @@ _cache = collections.OrderedDict()
26
31
  _cache_lock = threading.Lock()
27
32
 
28
33
 
34
+ class NoTextLayer(document_text.DocumentError):
35
+ """A PDF whose pages carry no text: `pages` is its page count."""
36
+
37
+ def __init__(self, pages):
38
+ super().__init__(f'PDF of {pages} pages has no text layer (scanned pages are not read; there is no OCR).')
39
+ self.pages = pages
40
+
41
+
29
42
  class Unit:
30
43
  """One readable unit of a document: a PDF page, a docx part, a slide or a sheet."""
31
44
 
@@ -74,13 +87,12 @@ def compact_pdf_text(text):
74
87
 
75
88
 
76
89
  def _pdf_units(data):
77
- """Units and facts of a PDF, one unit per page; raises DocumentError for a password, a corrupt file or a file with no
78
- text layer at all (scanned pages are not read, there is no OCR)."""
90
+ """Units and facts of a PDF, one unit per page; raises DocumentError for a password or a corrupt file, and NoTextLayer
91
+ for a file with no text layer at all."""
79
92
  pages, count = document_text.pdf_pages(data, document_text.PDF_TIMEOUT_S, document_text.PDF_MAX_PAGES,
80
93
  document_text.PDF_MAX_CHARS)
81
94
  if not any(pages):
82
- raise document_text.DocumentError(
83
- f'PDF of {count} pages has no text layer (scanned pages are not read; there is no OCR).')
95
+ raise NoTextLayer(count)
84
96
  units = [Unit(number, compact_pdf_text(page) or '(no text on this page)') for number, page in enumerate(pages, 1)]
85
97
  facts = {'pages': count, 'pages without text': sum(1 for page in pages if not page)}
86
98
  if count > len(pages):
@@ -89,21 +101,47 @@ def _pdf_units(data):
89
101
  return units, facts
90
102
 
91
103
 
92
- def _docx_units(data):
93
- """Units and facts of a docx: the markdown blocks grouped in parts of about DOCX_PART_CHARS characters, a block never
94
- split (a docx has no fixed pages, they depend on the renderer)."""
104
+ def _parts(blocks):
105
+ """The markdown blocks grouped in parts of about DOCX_PART_CHARS characters, a block never split."""
95
106
  parts, current, size = [], [], 0
96
- for block in document_text.docx_blocks(data):
107
+ for block in blocks:
97
108
  if current and size + len(block) > DOCX_PART_CHARS:
98
109
  parts.append('\n'.join(current))
99
110
  current, size = [], 0
100
111
  current.append(block)
101
112
  size += len(block) + 1
102
113
  parts.append('\n'.join(current))
114
+ return parts
115
+
116
+
117
+ def _docx_units(data):
118
+ """Units and facts of a docx: the markdown blocks grouped in parts (a docx has no fixed pages, they depend on the
119
+ renderer)."""
120
+ parts = _parts(document_text.docx_blocks(data))
103
121
  return [Unit(number, text) for number, text in enumerate(parts, 1)], {
104
122
  'parts': len(parts), 'note': f'a docx has no fixed pages; a part is about {DOCX_PART_CHARS:,} characters'}
105
123
 
106
124
 
125
+ def _html_units(data):
126
+ """Units and facts of an html file: the markdown blocks grouped in parts, as a docx (an html file has no pages)."""
127
+ parts = _parts(document_text.html_blocks(data))
128
+ return [Unit(number, text) for number, text in enumerate(parts, 1)], {
129
+ 'parts': len(parts), 'note': f'a part is about {DOCX_PART_CHARS:,} characters of the page text'}
130
+
131
+
132
+ def _csv_units(data):
133
+ """Units and facts of a csv file: one unit, the sheet `csv`, one tab-separated line per row."""
134
+ rows = document_text.csv_rows(data)
135
+ return [Unit(1, '\n'.join(['[sheet csv]'] + rows), 'csv')], {'rows': len(rows)}
136
+
137
+
138
+ def _epub_units(data):
139
+ """Units and facts of an epub: one unit per chapter of the reading order."""
140
+ chapters = document_text.epub_chapters(data)
141
+ units = [Unit(number, '\n'.join(blocks) or '(no text)') for number, blocks in enumerate(chapters, 1)]
142
+ return units, {'chapters without text': sum(1 for blocks in chapters if not blocks)}
143
+
144
+
107
145
  def _pptx_units(data):
108
146
  """Units and facts of a pptx: one unit per slide under `[slide N]`, the speaker notes under `[notes N]`."""
109
147
  units = []
@@ -121,14 +159,51 @@ def _xlsx_units(data):
121
159
  return units, {'sheet names': ', '.join(f'{u.number}={u.name}' for u in units)}
122
160
 
123
161
 
124
- _BUILDERS = {'pdf': _pdf_units, 'docx': _docx_units, 'pptx': _pptx_units, 'xlsx': _xlsx_units}
162
+ def _converted_pdf_units(path, count):
163
+ """Units and facts of a PDF with no text layer, read by the external converter as page images."""
164
+ result = document_converter.convert(path, expected_pages=count, force_ocr=True)
165
+ units = [Unit(number, page.strip() or '(no text on this page)') for number, page in enumerate(result.pages, 1)]
166
+ facts = {'pages without text': sum(1 for page in result.pages if not page.strip()),
167
+ 'read by': f'{result.engine} (OCR of the page images)'}
168
+ if len(units) != count:
169
+ facts['note'] = f'the converter returned {len(units)} pages for a file of {count}'
170
+ return units, facts
171
+
172
+
173
+ def _converted_units(path):
174
+ """Units and facts of a file the native reader has no type for, read by the external converter: its text grouped in
175
+ parts as a docx."""
176
+ result = document_converter.convert(path)
177
+ text = '\n\n'.join(page.strip() for page in result.pages if page.strip())
178
+ parts = _parts(text.split('\n'))
179
+ return [Unit(number, part) for number, part in enumerate(parts, 1)], {'parts': len(parts),
180
+ 'read by': result.engine}
181
+
182
+
183
+ def _build(path, kind):
184
+ """Units and facts of the file at `path`: its native builder, or, for a kind the native reader has none for and for a
185
+ PDF without a text layer, the external converter when the setting turns it on."""
186
+ if kind == 'converted':
187
+ return _converted_units(path)
188
+ with open(path, 'rb') as stream:
189
+ data = stream.read()
190
+ try:
191
+ return _BUILDERS[kind](data)
192
+ except NoTextLayer as exc:
193
+ if not document_converter.enabled():
194
+ raise
195
+ return _converted_pdf_units(path, exc.pages)
196
+
197
+
198
+ _BUILDERS = {'pdf': _pdf_units, 'docx': _docx_units, 'pptx': _pptx_units, 'xlsx': _xlsx_units, 'html': _html_units,
199
+ 'csv': _csv_units, 'epub': _epub_units}
125
200
 
126
201
 
127
202
  def load(path):
128
203
  """The document at `path`, split into units. Parsed documents are kept for a few calls (keyed by path, size and
129
204
  modification time), so reading page after page parses the file once.
130
205
 
131
- Parameters: path (absolute path of a .pdf, .docx, .pptx or .xlsx file). Returns: a Document. Raises: ValueError for a
206
+ Parameters: path (absolute path of a .pdf, .docx, .pptx, .xlsx, .html, .htm, .csv or .epub file). Returns: a Document. Raises: ValueError for a
132
207
  relative path, a missing file, an unsupported type or a file over the size limit of its type; DocumentError with a
133
208
  readable reason for a file the extractor refuses (corrupt, encrypted, hostile, scanned, without text)."""
134
209
  if not isinstance(path, str) or not path.strip():
@@ -136,27 +211,27 @@ def load(path):
136
211
  if not os.path.isabs(path):
137
212
  raise ValueError(f'read_document needs an absolute path, got {path!r}')
138
213
  extension = os.path.splitext(path)[1].lower()
139
- if extension not in KINDS:
214
+ converted = extension not in KINDS and extension in document_converter.EXTENSIONS and document_converter.enabled()
215
+ if extension not in KINDS and not converted:
140
216
  raise ValueError(f"read_document reads {', '.join(KINDS)} files; {extension or 'this file'} is not supported "
141
- '(use the normal Read for plain text; legacy .doc, .xls and .ppt and macro files are not read)')
217
+ '(use the normal Read for plain text; legacy .doc, .xls, .xlsb and .ppt and macro files other than .xlsm are not read)')
142
218
  try:
143
219
  info = os.stat(path)
144
220
  except OSError as exc:
145
221
  raise ValueError(f'cannot read {path}: {exc.strerror or type(exc).__name__}') from None
146
222
  if not os.path.isfile(path):
147
223
  raise ValueError(f'{path} is not a file')
148
- if info.st_size > document_text.size_limit(path):
149
- raise ValueError(f'{path} is {info.st_size / 1048576:.1f} MiB, over the '
150
- f'{document_text.size_limit(path) // 1048576} MiB limit of a {extension} file')
224
+ limit = document_text.PDF_MAX_BYTES if converted else document_text.size_limit(path)
225
+ if info.st_size > limit:
226
+ raise ValueError(f'{path} is {info.st_size / 1048576:.1f} MiB, over the {limit // 1048576} MiB limit of a '
227
+ f'{extension} file')
151
228
  key = (os.path.normcase(path), info.st_size, info.st_mtime_ns)
152
229
  with _cache_lock:
153
230
  if key in _cache:
154
231
  _cache.move_to_end(key)
155
232
  return _cache[key]
156
- with open(path, 'rb') as stream:
157
- data = stream.read()
158
- kind = KINDS[extension]
159
- units, facts = _BUILDERS[kind](data)
233
+ kind = 'converted' if converted else KINDS[extension]
234
+ units, facts = _build(path, kind)
160
235
  document = Document(path, kind, info.st_size, units, facts)
161
236
  with _cache_lock:
162
237
  _cache[key] = document
@@ -241,7 +316,7 @@ def read(path, range_spec=None, max_chars=DEFAULT_MAX_CHARS, info_only=False):
241
316
  body, shown, size, cut = [], [], 0, None
242
317
  for number in numbers:
243
318
  text = document.units[number - 1].text
244
- label = f'[{unit} {number}]\n' if document.kind in ('pdf', 'docx') else ''
319
+ label = f'[{unit} {number}]\n' if document.kind in LABELLED_KINDS else ''
245
320
  piece = label + text
246
321
  if shown and size + len(piece) > max_chars:
247
322
  break