@allansantos-dev/smart-tool 0.9.42 → 0.9.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/README.md +17 -2
- package/browser_control.py +71 -15
- package/client_hooks.py +2 -2
- package/document_read.py +272 -0
- package/document_text.py +212 -54
- package/hook_decision.py +9 -1
- package/index_scope.py +2 -2
- package/lazy_mcp_proxy.py +185 -0
- package/package.json +1 -1
- package/requester.py +76 -0
- package/requirements.txt +1 -1
- package/research_metrics.py +5 -3
- package/smart_tool_daemon.py +68 -10
- package/tool_redirect.py +2 -1
- package/version.py +1 -1
- package/web_search_health.py +5 -2
package/document_text.py
CHANGED
|
@@ -115,39 +115,157 @@ def _local(tag):
|
|
|
115
115
|
return tag.rsplit('}', 1)[-1]
|
|
116
116
|
|
|
117
117
|
|
|
118
|
+
def _docx_namespace(root):
|
|
119
|
+
"""The WordprocessingML namespace prefix (transitional or strict) of a parsed part."""
|
|
120
|
+
return W if root.tag.startswith(W) else '{http://purl.oclc.org/ooxml/wordprocessingml/main}'
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _docx_text(paragraph, namespace):
|
|
124
|
+
"""Text of one `w:p`: runs, tabs and line breaks; a nested paragraph (text box) is not part of it."""
|
|
125
|
+
pending = list(reversed(paragraph))
|
|
126
|
+
pieces = []
|
|
127
|
+
while pending:
|
|
128
|
+
child = pending.pop()
|
|
129
|
+
if child.tag == namespace + 'p':
|
|
130
|
+
continue
|
|
131
|
+
if child.tag == namespace + 't':
|
|
132
|
+
pieces.append(child.text or '')
|
|
133
|
+
elif child.tag == namespace + 'tab':
|
|
134
|
+
pieces.append('\t')
|
|
135
|
+
elif child.tag in (namespace + 'br', namespace + 'cr'):
|
|
136
|
+
pieces.append('\n')
|
|
137
|
+
else:
|
|
138
|
+
pending.extend(reversed(child))
|
|
139
|
+
return ''.join(pieces)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _docx_extra_parts(package):
|
|
143
|
+
"""Names of the header, footer, footnote and endnote parts of a .docx, in a stable order."""
|
|
144
|
+
return sorted(name for name in package.entries if re.fullmatch(
|
|
145
|
+
r'word/(?:header\d+|footer\d+|footnotes|endnotes)\.xml', name))
|
|
146
|
+
|
|
147
|
+
|
|
118
148
|
def extract_docx(data):
|
|
119
149
|
"""Text of a .docx: body, headers, footers, footnotes and endnotes, one line per non-empty paragraph.
|
|
120
150
|
|
|
121
151
|
Parameters: data (the file bytes). Returns: UTF-8 text bytes. Raises: DocumentError with a readable reason for a
|
|
122
152
|
corrupt, encrypted or oversized package, or one with a DOCTYPE or entity."""
|
|
123
153
|
with _open_package(data, 'DOCX', 'word/document.xml') as package:
|
|
124
|
-
names = ['word/document.xml'] + sorted(name for name in package.entries if re.fullmatch(
|
|
125
|
-
r'word/(?:header\d+|footer\d+|footnotes|endnotes)\.xml', name))
|
|
126
154
|
paragraphs = _Lines('DOCX')
|
|
127
|
-
for name in
|
|
155
|
+
for name in ['word/document.xml'] + _docx_extra_parts(package):
|
|
128
156
|
root = package.parse(name)
|
|
129
|
-
namespace =
|
|
157
|
+
namespace = _docx_namespace(root)
|
|
130
158
|
for paragraph in root.iter(namespace + 'p'):
|
|
131
|
-
|
|
132
|
-
pending = list(reversed(node))
|
|
133
|
-
while pending:
|
|
134
|
-
child = pending.pop()
|
|
135
|
-
if child.tag == namespace + 'p':
|
|
136
|
-
continue
|
|
137
|
-
if child.tag == namespace + 't':
|
|
138
|
-
yield child.text or ''
|
|
139
|
-
elif child.tag == namespace + 'tab':
|
|
140
|
-
yield '\t'
|
|
141
|
-
elif child.tag in (namespace + 'br', namespace + 'cr'):
|
|
142
|
-
yield '\n'
|
|
143
|
-
else:
|
|
144
|
-
pending.extend(reversed(child))
|
|
145
|
-
text = ''.join(parts(paragraph))
|
|
159
|
+
text = _docx_text(paragraph, namespace)
|
|
146
160
|
if text.strip():
|
|
147
161
|
paragraphs.add(text)
|
|
148
162
|
return ('\n'.join(paragraphs) + '\n' if paragraphs else '').encode('utf-8')
|
|
149
163
|
|
|
150
164
|
|
|
165
|
+
def _docx_heading_levels(package, namespace):
|
|
166
|
+
"""`{styleId: level}` of the heading styles of word/styles.xml (the style name is `heading N` in every language,
|
|
167
|
+
`Title` is level 1); empty when the part is missing."""
|
|
168
|
+
if 'word/styles.xml' not in package.entries:
|
|
169
|
+
return {}
|
|
170
|
+
levels = {}
|
|
171
|
+
for style in package.parse('word/styles.xml').iter(namespace + 'style'):
|
|
172
|
+
name = next((node.get(namespace + 'val', '') for node in style if node.tag == namespace + 'name'), '').lower()
|
|
173
|
+
found = re.fullmatch(r'heading (\d)', name)
|
|
174
|
+
if found or name == 'title':
|
|
175
|
+
levels[style.get(namespace + 'styleId')] = int(found.group(1)) if found else 1
|
|
176
|
+
return levels
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _docx_paragraph_line(paragraph, namespace, levels):
|
|
180
|
+
"""Markdown line of a paragraph: `#` heading by style or outline level, `- ` item (indented by level) for a list
|
|
181
|
+
paragraph, else the plain text; an empty string for a paragraph without text."""
|
|
182
|
+
text = ' '.join(_docx_text(paragraph, namespace).split())
|
|
183
|
+
if not text:
|
|
184
|
+
return ''
|
|
185
|
+
properties = next((node for node in paragraph if node.tag == namespace + 'pPr'), None)
|
|
186
|
+
if properties is None:
|
|
187
|
+
return text
|
|
188
|
+
style = next((node.get(namespace + 'val') for node in properties if node.tag == namespace + 'pStyle'), None)
|
|
189
|
+
outline = next((node.get(namespace + 'val') for node in properties if node.tag == namespace + 'outlineLvl'), None)
|
|
190
|
+
level = levels.get(style) or (int(outline) + 1 if outline and outline.isdigit() and int(outline) < 6 else 0)
|
|
191
|
+
if level:
|
|
192
|
+
return '#' * min(level, 6) + ' ' + text
|
|
193
|
+
numbering = next((node for node in properties if node.tag == namespace + 'numPr'), None)
|
|
194
|
+
if numbering is not None:
|
|
195
|
+
depth = next((node.get(namespace + 'val') for node in numbering if node.tag == namespace + 'ilvl'), '0')
|
|
196
|
+
return ' ' * (int(depth) if depth.isdigit() else 0) + '- ' + text
|
|
197
|
+
return text
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _docx_table_lines(table, namespace):
|
|
201
|
+
"""GFM table lines of a `w:tbl`: one row per `w:tr`, the first row as the header, cell text on one line."""
|
|
202
|
+
rows = []
|
|
203
|
+
for row in table:
|
|
204
|
+
if row.tag != namespace + 'tr':
|
|
205
|
+
continue
|
|
206
|
+
cells = []
|
|
207
|
+
for cell in row:
|
|
208
|
+
if cell.tag == namespace + 'tc':
|
|
209
|
+
text = ' '.join(' '.join(_docx_text(p, namespace).split()) for p in cell.iter(namespace + 'p'))
|
|
210
|
+
cells.append(' '.join(text.split()).replace('|', '\\|'))
|
|
211
|
+
if any(cells):
|
|
212
|
+
rows.append('| ' + ' | '.join(cells) + ' |')
|
|
213
|
+
if len(rows) > 1:
|
|
214
|
+
rows.insert(1, '|' + ' --- |' * (rows[0].count(' | ') + 1))
|
|
215
|
+
return rows
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def _docx_body_items(node, namespace):
|
|
219
|
+
"""The paragraphs and tables of a body (or of a content control inside it) in document order."""
|
|
220
|
+
for child in node:
|
|
221
|
+
if child.tag in (namespace + 'p', namespace + 'tbl'):
|
|
222
|
+
yield child
|
|
223
|
+
elif child.tag == namespace + 'sdt':
|
|
224
|
+
for content in child:
|
|
225
|
+
if content.tag == namespace + 'sdtContent':
|
|
226
|
+
yield from _docx_body_items(content, namespace)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def docx_blocks(data):
|
|
230
|
+
"""Markdown blocks of a .docx in document order, for reading: headings as `#`, list items as `- `, tables as GFM
|
|
231
|
+
tables, other paragraphs as text. Footnotes, endnotes and the distinct header and footer lines follow under
|
|
232
|
+
`## Notes` and `## Headers and footers`.
|
|
233
|
+
|
|
234
|
+
Parameters: data (the file bytes). Returns: a list of blocks (a block is a paragraph line or a whole table, lines
|
|
235
|
+
joined by newline). Raises: DocumentError with a readable reason for a corrupt, encrypted, oversized or hostile
|
|
236
|
+
package, or one without text."""
|
|
237
|
+
with _open_package(data, 'DOCX', 'word/document.xml') as package:
|
|
238
|
+
root = package.parse('word/document.xml')
|
|
239
|
+
namespace = _docx_namespace(root)
|
|
240
|
+
levels = _docx_heading_levels(package, namespace)
|
|
241
|
+
blocks = _Lines('DOCX')
|
|
242
|
+
body = next((node for node in root if node.tag == namespace + 'body'), root)
|
|
243
|
+
for item in _docx_body_items(body, namespace):
|
|
244
|
+
if item.tag == namespace + 'tbl':
|
|
245
|
+
table = _docx_table_lines(item, namespace)
|
|
246
|
+
if table:
|
|
247
|
+
blocks.add('\n'.join(table))
|
|
248
|
+
else:
|
|
249
|
+
line = _docx_paragraph_line(item, namespace, levels)
|
|
250
|
+
if line:
|
|
251
|
+
blocks.add(line)
|
|
252
|
+
notes, frame = [], []
|
|
253
|
+
for name in _docx_extra_parts(package):
|
|
254
|
+
part = package.parse(name)
|
|
255
|
+
target = frame if re.search(r'header|footer', name) else notes
|
|
256
|
+
for paragraph in part.iter(_docx_namespace(part) + 'p'):
|
|
257
|
+
text = ' '.join(_docx_text(paragraph, _docx_namespace(part)).split())
|
|
258
|
+
if text and text not in target:
|
|
259
|
+
target.append(text)
|
|
260
|
+
if notes:
|
|
261
|
+
blocks.add('## Notes', *notes)
|
|
262
|
+
if frame:
|
|
263
|
+
blocks.add('## Headers and footers', *frame)
|
|
264
|
+
if not blocks:
|
|
265
|
+
raise DocumentError('DOCX has no text.')
|
|
266
|
+
return list(blocks)
|
|
267
|
+
|
|
268
|
+
|
|
151
269
|
def _drawing_paragraphs(root):
|
|
152
270
|
"""Non-empty paragraphs (`a:p`) of a slide or notes part, from the `a:t` runs; the slide-number field is not text."""
|
|
153
271
|
lines = []
|
|
@@ -173,26 +291,40 @@ def _drawing_paragraphs(root):
|
|
|
173
291
|
return lines
|
|
174
292
|
|
|
175
293
|
|
|
176
|
-
def
|
|
177
|
-
"""
|
|
294
|
+
def pptx_slides(data):
|
|
295
|
+
"""The slides of a .pptx in slide order, for extraction and for reading.
|
|
178
296
|
|
|
179
|
-
Parameters: data (the file bytes). Returns:
|
|
180
|
-
|
|
297
|
+
Parameters: data (the file bytes). Returns: a list of `(number, body lines, notes lines)`, a slide without text
|
|
298
|
+
included with empty lists. Raises: DocumentError with a readable reason for a corrupt, encrypted, oversized or hostile
|
|
299
|
+
package or one over MAX_TEXT_BYTES of text."""
|
|
181
300
|
def has_slides(names):
|
|
182
301
|
return any(re.fullmatch(r'ppt/slides/slide\d+\.xml', name) for name in names)
|
|
183
302
|
|
|
184
303
|
with _open_package(data, 'PPTX', has_slides) as package:
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
304
|
+
numbered = sorted((int(m.group(1)), name) for name in package.entries
|
|
305
|
+
if (m := re.fullmatch(r'ppt/slides/slide(\d+)\.xml', name)))
|
|
306
|
+
budget = _Lines('PPTX')
|
|
307
|
+
slides = []
|
|
308
|
+
for number, name in numbered:
|
|
189
309
|
body = _drawing_paragraphs(package.parse(name))
|
|
190
|
-
if body:
|
|
191
|
-
lines.add(f'[slide {number}]', *body)
|
|
192
310
|
notes_name = f'ppt/notesSlides/notesSlide{number}.xml'
|
|
193
311
|
notes = _drawing_paragraphs(package.parse(notes_name)) if notes_name in package.entries else []
|
|
194
|
-
|
|
195
|
-
|
|
312
|
+
budget.add(*body, *notes)
|
|
313
|
+
slides.append((number, body, notes))
|
|
314
|
+
return slides
|
|
315
|
+
|
|
316
|
+
|
|
317
|
+
def extract_pptx(data):
|
|
318
|
+
"""Text of a .pptx: each slide under `[slide N]`, then its speaker notes under `[notes N]`, in slide order.
|
|
319
|
+
|
|
320
|
+
Parameters: data (the file bytes). Returns: UTF-8 text bytes. Raises: DocumentError with a readable reason for a
|
|
321
|
+
corrupt, encrypted, oversized or hostile package, or a deck without any text."""
|
|
322
|
+
lines = _Lines('PPTX')
|
|
323
|
+
for number, body, notes in pptx_slides(data):
|
|
324
|
+
if body:
|
|
325
|
+
lines.add(f'[slide {number}]', *body)
|
|
326
|
+
if notes:
|
|
327
|
+
lines.add(f'[notes {number}]', *notes)
|
|
196
328
|
if not lines:
|
|
197
329
|
raise DocumentError('PPTX has no text.')
|
|
198
330
|
return ('\n'.join(lines) + '\n').encode('utf-8')
|
|
@@ -264,14 +396,13 @@ def _sheet_rows(root, strings):
|
|
|
264
396
|
yield '\t'.join(values)
|
|
265
397
|
|
|
266
398
|
|
|
267
|
-
def
|
|
268
|
-
"""
|
|
269
|
-
|
|
270
|
-
value, nothing is calculated.
|
|
399
|
+
def xlsx_sheets(data):
|
|
400
|
+
"""The sheets of an .xlsx in workbook order, for extraction and for reading. Shared strings are resolved; numbers
|
|
401
|
+
and inline strings are kept; a formula contributes its cached value, nothing is calculated.
|
|
271
402
|
|
|
272
|
-
Parameters: data (the file bytes). Returns:
|
|
273
|
-
|
|
274
|
-
|
|
403
|
+
Parameters: data (the file bytes). Returns: a list of `(name, rows)`, `rows` being the tab-joined text of each
|
|
404
|
+
non-empty row (empty for a sheet without text). Raises: DocumentError with a readable reason for a corrupt,
|
|
405
|
+
encrypted, oversized or hostile package, or a workbook over XLSX_MAX_ROWS rows or MAX_TEXT_BYTES of text."""
|
|
275
406
|
with _open_package(data, 'XLSX', 'xl/workbook.xml') as package:
|
|
276
407
|
workbook = package.parse('xl/workbook.xml')
|
|
277
408
|
targets = {}
|
|
@@ -281,22 +412,40 @@ def extract_xlsx(data):
|
|
|
281
412
|
targets[rel.get('Id')] = posixpath.normpath(target.lstrip('/') if target.startswith('/')
|
|
282
413
|
else posixpath.join('xl', target))
|
|
283
414
|
strings = _shared_strings(package)
|
|
284
|
-
|
|
285
|
-
|
|
415
|
+
budget = _Lines('XLSX')
|
|
416
|
+
sheets = []
|
|
417
|
+
total = 0
|
|
286
418
|
for sheet in (node for node in workbook.iter() if _local(node.tag) == 'sheet'):
|
|
287
419
|
rid = next((value for key, value in sheet.attrib.items() if _local(key) == 'id'), None)
|
|
288
420
|
part = targets.get(rid)
|
|
289
421
|
if part not in package.entries:
|
|
290
422
|
raise DocumentError('Invalid XLSX or excessive structure.')
|
|
291
|
-
|
|
423
|
+
name = ' '.join(sheet.get('name', '').split())
|
|
424
|
+
rows = []
|
|
292
425
|
for line in _sheet_rows(package.parse(part), strings):
|
|
293
|
-
|
|
294
|
-
if
|
|
426
|
+
total += 1
|
|
427
|
+
if total > XLSX_MAX_ROWS:
|
|
295
428
|
raise DocumentError(f'XLSX over {XLSX_MAX_ROWS} rows.')
|
|
296
|
-
if
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
429
|
+
if not rows:
|
|
430
|
+
budget.add(f'[sheet {name}]')
|
|
431
|
+
budget.add(line)
|
|
432
|
+
rows.append(line)
|
|
433
|
+
sheets.append((name, rows))
|
|
434
|
+
return sheets
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def extract_xlsx(data):
|
|
438
|
+
"""Text of an .xlsx: each sheet, in workbook order, under `[sheet NAME]`, then one line per row with the cells
|
|
439
|
+
separated by tabs. Shared strings are resolved; numbers and inline strings are kept; a formula contributes its cached
|
|
440
|
+
value, nothing is calculated.
|
|
441
|
+
|
|
442
|
+
Parameters: data (the file bytes). Returns: UTF-8 text bytes. Raises: DocumentError with a readable reason for a
|
|
443
|
+
corrupt, encrypted, oversized or hostile package, a workbook over XLSX_MAX_ROWS rows or MAX_TEXT_BYTES of text, or
|
|
444
|
+
one without any text."""
|
|
445
|
+
lines = _Lines('XLSX')
|
|
446
|
+
for name, rows in xlsx_sheets(data):
|
|
447
|
+
if rows:
|
|
448
|
+
lines.add(f'[sheet {name}]', *rows)
|
|
300
449
|
if not lines:
|
|
301
450
|
raise DocumentError('XLSX has no text.')
|
|
302
451
|
return ('\n'.join(lines) + '\n').encode('utf-8')
|
|
@@ -320,14 +469,15 @@ print(json.dumps({"pages": pages, "count": len(reader.pages)}))
|
|
|
320
469
|
"""
|
|
321
470
|
|
|
322
471
|
|
|
323
|
-
def
|
|
324
|
-
"""Text of a PDF (pypdf, pure Python)
|
|
325
|
-
|
|
472
|
+
def pdf_pages(data, timeout, max_pages, max_chars):
|
|
473
|
+
"""Text of each page of a PDF (pypdf, pure Python). A hostile PDF can make pypdf parse for minutes, so it runs in an
|
|
474
|
+
isolated subprocess (`-I`) killed at the deadline.
|
|
326
475
|
|
|
327
476
|
Parameters: data (PDF bytes), timeout (seconds before the subprocess is killed), max_pages, max_chars (cap on the
|
|
328
|
-
text read, over all pages). Returns: the
|
|
329
|
-
|
|
330
|
-
the last line pypdf reported), all
|
|
477
|
+
text read, over all pages). Returns: `(pages, count)`: the text of the pages read (an empty string for a page
|
|
478
|
+
without text) and the page count of the file, which is larger when a cap cut the reading. Raises: PdfTimeout,
|
|
479
|
+
PdfEncrypted (a password is needed), PdfUnreadable (corrupt; `detail` has the last line pypdf reported), all
|
|
480
|
+
DocumentError."""
|
|
331
481
|
try:
|
|
332
482
|
done = subprocess.run([sys.executable, "-I", "-c", PDF_SCRIPT, str(max_pages), str(max_chars)],
|
|
333
483
|
input=data, capture_output=True, timeout=timeout,
|
|
@@ -341,11 +491,19 @@ def run_pdf(data, timeout, max_pages, max_chars):
|
|
|
341
491
|
result = json.loads(done.stdout)
|
|
342
492
|
if result.get("encrypted"):
|
|
343
493
|
raise PdfEncrypted("PDF protected by a password.")
|
|
344
|
-
|
|
494
|
+
return result["pages"], result["count"]
|
|
345
495
|
except (ValueError, KeyError, AttributeError) as exc:
|
|
346
496
|
if isinstance(exc, PdfEncrypted):
|
|
347
497
|
raise
|
|
348
498
|
raise PdfUnreadable("PDF could not be read.", f"unexpected extractor output ({type(exc).__name__})") from None
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
def run_pdf(data, timeout, max_pages, max_chars):
|
|
502
|
+
"""Text of a PDF through pdf_pages, with `[page N]` markers.
|
|
503
|
+
|
|
504
|
+
Parameters: as pdf_pages. Returns: the marked text, with a `[pages A-B not read]` line when a cap cut it; empty
|
|
505
|
+
pages are left out. Raises: as pdf_pages."""
|
|
506
|
+
pages, count = pdf_pages(data, timeout, max_pages, max_chars)
|
|
349
507
|
text = "\n\n".join(f"[page {number}]\n{page}" for number, page in enumerate(pages, 1) if page)
|
|
350
508
|
if count > len(pages):
|
|
351
509
|
text += f"\n\n[pages {len(pages) + 1}-{count} not read]"
|
package/hook_decision.py
CHANGED
|
@@ -25,6 +25,7 @@ import index_scope
|
|
|
25
25
|
import endpoint_sync
|
|
26
26
|
import project_store
|
|
27
27
|
import redirect_rule
|
|
28
|
+
import requester
|
|
28
29
|
import router
|
|
29
30
|
import tool_redirect
|
|
30
31
|
|
|
@@ -298,6 +299,8 @@ def decide(payload, client, web_route, jobs):
|
|
|
298
299
|
router.CLIENT.set(client)
|
|
299
300
|
router.SESSION.set(payload.get("session_id"))
|
|
300
301
|
router._log_decision(SMART_SEARCH_TOOL, {}, "allow", "Smart Tool search, logged to measure followed redirects")
|
|
302
|
+
requester.note_hook(payload["tool_name"].removeprefix(f"mcp__{SMART_TOOL_SERVER}__"), payload.get("tool_input"),
|
|
303
|
+
payload.get("session_id"), payload.get("cwd"))
|
|
301
304
|
return {}
|
|
302
305
|
mode = config.hook_mode(cfg)
|
|
303
306
|
if mode == "off" or not isinstance(payload, dict):
|
|
@@ -321,7 +324,12 @@ def _redirect(payload, client, plan, jobs):
|
|
|
321
324
|
else runs as the agent wrote it and, in Claude Code, `after` swaps its output for the answer."""
|
|
322
325
|
key, tool_name = payload["tool_use_id"], payload["tool_name"]
|
|
323
326
|
tool_input = payload.get("tool_input") if isinstance(payload.get("tool_input"), dict) else {}
|
|
324
|
-
|
|
327
|
+
who = requester.CURRENT.set(requester.from_hook(client, payload)) if tool_name in WEB_TOOLS else None
|
|
328
|
+
try:
|
|
329
|
+
jobs.start(key, plan["kind"], plan["arguments"])
|
|
330
|
+
finally:
|
|
331
|
+
if who is not None:
|
|
332
|
+
requester.CURRENT.reset(who)
|
|
325
333
|
if tool_name not in tool_redirect.NO_OP:
|
|
326
334
|
return {}
|
|
327
335
|
text = _answer(jobs, key, PRE_WAIT_S, tool_name)
|
package/index_scope.py
CHANGED
|
@@ -387,8 +387,8 @@ def _build_scope_prompt(root, ambiguous_dirs, root_files, doc_context):
|
|
|
387
387
|
"not indexed, and an empty `include` indexes only the loose root files. "
|
|
388
388
|
"Include sources useful for search even if they are gitignored (e.g. docs/, README, "
|
|
389
389
|
"ADRs). Java, Angular, HTML/CSS, plain text and Markdown are supported sources. "
|
|
390
|
-
"DOCX documents are supported through bounded local text extraction; do not exclude "
|
|
391
|
-
"useful
|
|
390
|
+
"DOCX, PPTX, XLSX and PDF documents are supported through bounded local text extraction; do not exclude "
|
|
391
|
+
"useful Office or PDF documentation merely because its container is binary. "
|
|
392
392
|
"Exclude directories that inspection shows are generated artifacts or "
|
|
393
393
|
"dependencies (compiled binaries, minified bundles, hashed build chunks, vendored "
|
|
394
394
|
"packages)."
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Stdio MCP proxy that starts the real server only when a request needs it.
|
|
3
|
+
|
|
4
|
+
Claude Code and Codex start every registered stdio server with the session, so a Playwright MCP server (Node, about
|
|
5
|
+
97 MB of commit) sat idle in each session that never opened a browser. This proxy is what gets registered: it answers
|
|
6
|
+
`initialize`, `ping` and `tools/list` from the handshake cache browser_control wrote after probing the real server, and
|
|
7
|
+
starts the real command on the first other request (a tool call), replaying the client's `initialize` to it. From then
|
|
8
|
+
on every message passes through unchanged. When the client closes stdin the real server's process tree is killed.
|
|
9
|
+
|
|
10
|
+
Usage: lazy_mcp_proxy.py --handshake FILE -- COMMAND [ARG...]
|
|
11
|
+
The cache (`{"initialize": <result>, "tools": [...]}`) missing or unreadable is an error on stderr and exit code 2, never
|
|
12
|
+
an empty tool list.
|
|
13
|
+
"""
|
|
14
|
+
import json
|
|
15
|
+
import os
|
|
16
|
+
import queue
|
|
17
|
+
import subprocess
|
|
18
|
+
import sys
|
|
19
|
+
import threading
|
|
20
|
+
|
|
21
|
+
LAZY_ID = "smart-tool-lazy-init"
|
|
22
|
+
INIT_TIMEOUT_S = 60
|
|
23
|
+
KNOWN_PROTOCOLS = ("2024-11-05", "2025-03-26", "2025-06-18", "2025-11-25")
|
|
24
|
+
_NO_WINDOW = getattr(subprocess, "CREATE_NO_WINDOW", 0)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def load_handshake(path):
|
|
28
|
+
"""The cached initialize result and tool list; RuntimeError when the file cannot be read or has the wrong shape."""
|
|
29
|
+
try:
|
|
30
|
+
with open(path, encoding="utf-8") as stream:
|
|
31
|
+
data = json.load(stream)
|
|
32
|
+
if not isinstance(data["initialize"], dict) or not isinstance(data["tools"], list):
|
|
33
|
+
raise TypeError("initialize must be an object and tools a list")
|
|
34
|
+
except (OSError, ValueError, KeyError, TypeError) as exc:
|
|
35
|
+
raise RuntimeError(f"handshake cache {path} is unusable ({type(exc).__name__}: {exc}); run browser_control.ensure "
|
|
36
|
+
f"again (reinstall Smart Tool).") from exc
|
|
37
|
+
return data
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class Proxy:
|
|
41
|
+
"""One client conversation: local answers until a request needs the real server, then a pass-through."""
|
|
42
|
+
|
|
43
|
+
def __init__(self, handshake, command, stdin, stdout):
|
|
44
|
+
self.handshake, self.command = handshake, command
|
|
45
|
+
self.stdin, self.stdout = stdin, stdout
|
|
46
|
+
self.child = None
|
|
47
|
+
self.closing = False
|
|
48
|
+
self.client_init = None
|
|
49
|
+
self.write_lock = threading.Lock()
|
|
50
|
+
|
|
51
|
+
def send(self, message):
|
|
52
|
+
"""Writes one JSON-RPC message (dict) or one raw line (bytes) to the client."""
|
|
53
|
+
data = message if isinstance(message, bytes) else json.dumps(message, ensure_ascii=False).encode("utf-8") + b"\n"
|
|
54
|
+
with self.write_lock:
|
|
55
|
+
self.stdout.write(data)
|
|
56
|
+
self.stdout.flush()
|
|
57
|
+
|
|
58
|
+
def answer(self, request_id, result):
|
|
59
|
+
self.send({"jsonrpc": "2.0", "id": request_id, "result": result})
|
|
60
|
+
|
|
61
|
+
def local_initialize(self, params):
|
|
62
|
+
"""The cached initialize result, speaking the protocol version the client asked for when it is a known one."""
|
|
63
|
+
result = dict(self.handshake["initialize"])
|
|
64
|
+
asked = (params or {}).get("protocolVersion")
|
|
65
|
+
if asked in KNOWN_PROTOCOLS:
|
|
66
|
+
result["protocolVersion"] = asked
|
|
67
|
+
return result
|
|
68
|
+
|
|
69
|
+
def start_child(self):
|
|
70
|
+
"""Starts the real server and replays the client's initialize to it; its answer is not forwarded (the client
|
|
71
|
+
already got the cached one). RuntimeError when it exits or answers with an error."""
|
|
72
|
+
child = subprocess.Popen(self.command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=None,
|
|
73
|
+
creationflags=_NO_WINDOW)
|
|
74
|
+
params = self.client_init or {"protocolVersion": KNOWN_PROTOCOLS[2], "capabilities": {},
|
|
75
|
+
"clientInfo": {"name": "smart-tool-lazy-proxy", "version": "1"}}
|
|
76
|
+
child.stdin.write(json.dumps({"jsonrpc": "2.0", "id": LAZY_ID, "method": "initialize", "params": params}).encode()
|
|
77
|
+
+ b"\n")
|
|
78
|
+
child.stdin.flush()
|
|
79
|
+
answer = self.read_initialize(child)
|
|
80
|
+
if "error" in answer:
|
|
81
|
+
self.kill(child)
|
|
82
|
+
raise RuntimeError(f"the real server refused initialize: {answer['error']}")
|
|
83
|
+
child.stdin.write(b'{"jsonrpc":"2.0","method":"notifications/initialized"}\n')
|
|
84
|
+
child.stdin.flush()
|
|
85
|
+
self.child = child
|
|
86
|
+
threading.Thread(target=self.pump_child, args=(child,), daemon=True).start()
|
|
87
|
+
|
|
88
|
+
@classmethod
|
|
89
|
+
def read_initialize(cls, child):
|
|
90
|
+
"""The real server's answer to the replayed initialize; RuntimeError (child killed) when it exits or stays silent
|
|
91
|
+
for INIT_TIMEOUT_S."""
|
|
92
|
+
answers = queue.Queue()
|
|
93
|
+
|
|
94
|
+
def read():
|
|
95
|
+
for line in child.stdout:
|
|
96
|
+
try:
|
|
97
|
+
message = json.loads(line)
|
|
98
|
+
except ValueError:
|
|
99
|
+
continue
|
|
100
|
+
if message.get("id") == LAZY_ID:
|
|
101
|
+
answers.put(message)
|
|
102
|
+
return
|
|
103
|
+
answers.put(None)
|
|
104
|
+
threading.Thread(target=read, daemon=True).start()
|
|
105
|
+
try:
|
|
106
|
+
answer = answers.get(timeout=INIT_TIMEOUT_S)
|
|
107
|
+
except queue.Empty:
|
|
108
|
+
cls.kill(child)
|
|
109
|
+
raise RuntimeError(f"the real server did not answer initialize in {INIT_TIMEOUT_S}s") from None
|
|
110
|
+
if answer is None:
|
|
111
|
+
raise RuntimeError(f"the real server exited before answering initialize (code {child.wait(10)})")
|
|
112
|
+
return answer
|
|
113
|
+
|
|
114
|
+
def pump_child(self, child):
|
|
115
|
+
"""Child to client until the child closes its stdout; then the proxy exits with the child's code, loudly."""
|
|
116
|
+
for line in child.stdout:
|
|
117
|
+
self.send(line)
|
|
118
|
+
code = child.wait()
|
|
119
|
+
if self.closing:
|
|
120
|
+
return
|
|
121
|
+
print(f"lazy_mcp_proxy: the real server exited (code {code})", file=sys.stderr, flush=True)
|
|
122
|
+
os._exit(code or 1)
|
|
123
|
+
|
|
124
|
+
@staticmethod
|
|
125
|
+
def kill(child):
|
|
126
|
+
if child.poll() is not None:
|
|
127
|
+
return
|
|
128
|
+
if os.name == "nt":
|
|
129
|
+
subprocess.run(["taskkill", "/PID", str(child.pid), "/T", "/F"], capture_output=True, creationflags=_NO_WINDOW)
|
|
130
|
+
else:
|
|
131
|
+
child.kill()
|
|
132
|
+
|
|
133
|
+
def handle(self, line):
|
|
134
|
+
"""One client line: answered here while the real server is not needed, else passed to it."""
|
|
135
|
+
try:
|
|
136
|
+
message = json.loads(line)
|
|
137
|
+
except ValueError:
|
|
138
|
+
print(f"lazy_mcp_proxy: invalid line ignored: {line[:200]!r}", file=sys.stderr, flush=True)
|
|
139
|
+
return
|
|
140
|
+
method, request_id = message.get("method"), message.get("id")
|
|
141
|
+
if method == "initialize":
|
|
142
|
+
self.client_init = message.get("params")
|
|
143
|
+
if self.child is None and request_id is not None and method == "initialize":
|
|
144
|
+
return self.answer(request_id, self.local_initialize(message.get("params")))
|
|
145
|
+
if self.child is None and request_id is not None and method == "ping":
|
|
146
|
+
return self.answer(request_id, {})
|
|
147
|
+
if self.child is None and request_id is not None and method == "tools/list":
|
|
148
|
+
return self.answer(request_id, {"tools": self.handshake["tools"]})
|
|
149
|
+
if self.child is None and (method is None or request_id is None):
|
|
150
|
+
return # notifications and responses mean nothing to a server that is not running
|
|
151
|
+
if self.child is None:
|
|
152
|
+
try:
|
|
153
|
+
self.start_child()
|
|
154
|
+
except (OSError, RuntimeError) as exc:
|
|
155
|
+
self.send({"jsonrpc": "2.0", "id": request_id,
|
|
156
|
+
"error": {"code": -32000, "message": f"could not start the server: {exc}"[:500]}})
|
|
157
|
+
return
|
|
158
|
+
self.child.stdin.write(line if line.endswith(b"\n") else line + b"\n")
|
|
159
|
+
self.child.stdin.flush()
|
|
160
|
+
|
|
161
|
+
def run(self):
|
|
162
|
+
for line in self.stdin:
|
|
163
|
+
if line.strip():
|
|
164
|
+
self.handle(line)
|
|
165
|
+
self.closing = True
|
|
166
|
+
if self.child is not None:
|
|
167
|
+
self.kill(self.child)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def main(argv):
|
|
171
|
+
if "--" not in argv or argv[:1] != ["--handshake"] or len(argv) < 4:
|
|
172
|
+
print("usage: lazy_mcp_proxy.py --handshake FILE -- COMMAND [ARG...]", file=sys.stderr)
|
|
173
|
+
return 2
|
|
174
|
+
split = argv.index("--")
|
|
175
|
+
try:
|
|
176
|
+
handshake = load_handshake(argv[1])
|
|
177
|
+
except RuntimeError as exc:
|
|
178
|
+
print(f"lazy_mcp_proxy: {exc}", file=sys.stderr)
|
|
179
|
+
return 2
|
|
180
|
+
Proxy(handshake, argv[split + 1:], sys.stdin.buffer, sys.stdout.buffer).run()
|
|
181
|
+
return 0
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
if __name__ == "__main__":
|
|
185
|
+
sys.exit(main(sys.argv[1:]))
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@allansantos-dev/smart-tool",
|
|
3
|
-
"version": "0.9.
|
|
3
|
+
"version": "0.9.45",
|
|
4
4
|
"description": "Local MCP server that gives coding agents (Claude Code, Codex) cheaper, sharper tools than their built-in search. Windows.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Allan Santos",
|
package/requester.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Who asked for a web search or fetch, recorded next to each provider call and research job, never the query text.
|
|
2
|
+
|
|
3
|
+
An MCP client only sends `clientInfo` (name, version) and the daemon's own `Mcp-Session-Id`: no session or folder. The
|
|
4
|
+
PreToolUse hook of Claude Code does carry `session_id` and `cwd` before every tool call it matches, so the hook route
|
|
5
|
+
notes them here (note_hook) and the MCP call that follows claims them by tool and arguments (claim). The first claim
|
|
6
|
+
binds that MCP session to the Claude session and folder, so later calls of the same MCP session are attributed even
|
|
7
|
+
when the hook did not fire for them. A client without that hook (Codex, or a Claude Code whose hook predates the
|
|
8
|
+
web tools) leaves `session` and `project` empty and `source` says how far the attribution got.
|
|
9
|
+
"""
|
|
10
|
+
import collections
|
|
11
|
+
import contextvars
|
|
12
|
+
import json
|
|
13
|
+
import threading
|
|
14
|
+
import time
|
|
15
|
+
|
|
16
|
+
WEB_TOOLS = ("web_search", "web_fetch")
|
|
17
|
+
NOTE_TTL_S = 60
|
|
18
|
+
MAX_NOTES = 256
|
|
19
|
+
MAX_BOUND = 256
|
|
20
|
+
|
|
21
|
+
CURRENT = contextvars.ContextVar("requester", default=None)
|
|
22
|
+
_LOCK = threading.Lock()
|
|
23
|
+
_NOTES = collections.OrderedDict()
|
|
24
|
+
_BOUND = collections.OrderedDict()
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _key(tool, arguments):
|
|
28
|
+
return tool + "\0" + json.dumps(arguments, sort_keys=True, ensure_ascii=False, default=str)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def note_hook(tool, arguments, session_id, cwd):
|
|
32
|
+
"""Remembers who the PreToolUse hook saw about to call `tool` with `arguments` (the MCP tool name without the
|
|
33
|
+
server prefix); claim() takes it back within NOTE_TTL_S."""
|
|
34
|
+
if tool not in WEB_TOOLS or not isinstance(arguments, dict):
|
|
35
|
+
return
|
|
36
|
+
entry = (time.monotonic(), session_id if isinstance(session_id, str) else None, cwd if isinstance(cwd, str) else None)
|
|
37
|
+
with _LOCK:
|
|
38
|
+
_NOTES.setdefault(_key(tool, arguments), collections.deque()).append(entry)
|
|
39
|
+
while len(_NOTES) > MAX_NOTES:
|
|
40
|
+
_NOTES.popitem(last=False)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def claim(mcp_session, client, tool, arguments):
|
|
44
|
+
"""The requester of one MCP tool call: {client, mcp_session, session, project, source}. source is `hook` when the
|
|
45
|
+
hook noted this very call, `mcp-session` when only an earlier call of the same MCP session was attributed, `none`
|
|
46
|
+
when the client gave nothing beyond its name."""
|
|
47
|
+
now = time.monotonic()
|
|
48
|
+
found = None
|
|
49
|
+
with _LOCK:
|
|
50
|
+
queue = _NOTES.get(_key(tool, arguments))
|
|
51
|
+
while queue and now - queue[0][0] > NOTE_TTL_S:
|
|
52
|
+
queue.popleft()
|
|
53
|
+
if queue:
|
|
54
|
+
found = queue.popleft()
|
|
55
|
+
if queue is not None and not queue:
|
|
56
|
+
del _NOTES[_key(tool, arguments)]
|
|
57
|
+
if found and mcp_session:
|
|
58
|
+
_BOUND[mcp_session] = (found[1], found[2])
|
|
59
|
+
while len(_BOUND) > MAX_BOUND:
|
|
60
|
+
_BOUND.popitem(last=False)
|
|
61
|
+
bound = _BOUND.get(mcp_session)
|
|
62
|
+
session, project = (found[1:] if found else bound) or (None, None)
|
|
63
|
+
return {"client": client or None, "mcp_session": (mcp_session or "")[:12] or None, "session": session,
|
|
64
|
+
"project": project, "source": "hook" if found else ("mcp-session" if bound else "none")}
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def from_hook(client, payload):
|
|
68
|
+
"""The requester of a native WebSearch/WebFetch the hook is about to redirect: the payload itself names the session
|
|
69
|
+
and folder, no MCP call is involved."""
|
|
70
|
+
return {"client": client or None, "mcp_session": None, "session": payload.get("session_id"),
|
|
71
|
+
"project": payload.get("cwd"), "source": "hook"}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def current():
|
|
75
|
+
"""The requester of the tool call this thread (or a context copied from it) is serving, or None outside one."""
|
|
76
|
+
return CURRENT.get()
|