@senpi-ai/runtime 3.0.80 → 3.0.104
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actions/decision-engine.d.ts +16 -0
- package/dist/actions/decision-engine.d.ts.map +1 -1
- package/dist/actions/decision-engine.js +137 -1
- package/dist/actions/decision-engine.js.map +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -16
- package/dist/index.js.map +1 -1
- package/dist/runtime/runtime.d.ts +6 -0
- package/dist/runtime/runtime.d.ts.map +1 -1
- package/dist/runtime/runtime.js +26 -8
- package/dist/runtime/runtime.js.map +1 -1
- package/dist/runtime-api/intake/intake-events.d.ts +4 -2
- package/dist/runtime-api/intake/intake-events.d.ts.map +1 -1
- package/dist/runtime-api/intake/intake-events.js +4 -2
- package/dist/runtime-api/intake/intake-events.js.map +1 -1
- package/dist/runtime-api/intake/intake-mount.d.ts +2 -0
- package/dist/runtime-api/intake/intake-mount.d.ts.map +1 -1
- package/dist/runtime-api/intake/intake-mount.js +8 -20
- package/dist/runtime-api/intake/intake-mount.js.map +1 -1
- package/dist/runtime-api/intake/scaffold-error-router.d.ts +43 -0
- package/dist/runtime-api/intake/scaffold-error-router.d.ts.map +1 -0
- package/dist/runtime-api/intake/scaffold-error-router.js +63 -0
- package/dist/runtime-api/intake/scaffold-error-router.js.map +1 -0
- package/dist/scanners/external/scaffold/scaffold.py +94 -1
- package/dist/scanners/external/scaffold/senpi_mcp.py +427 -71
- package/dist/utils/event-catalog.d.ts +3 -2
- package/dist/utils/event-catalog.d.ts.map +1 -1
- package/dist/utils/event-catalog.js +1 -1
- package/dist/utils/event-catalog.js.map +1 -1
- package/openclaw.plugin.json +1 -1
- package/package.json +1 -1
|
@@ -12,13 +12,50 @@ the OFFICIAL MCP Python SDK (`mcp`), which is async-only:
|
|
|
12
12
|
await session.initialize()
|
|
13
13
|
result = await session.call_tool(name, arguments=args)
|
|
14
14
|
|
|
15
|
-
DECISION (
|
|
16
|
-
|
|
17
|
-
every call
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
15
|
+
DECISION (execution-plan B5, AMENDED 2026-09-04): use the SDK the STANDARD way —
|
|
16
|
+
ONE persistent `ClientSession` created once (and `initialize()`d once), reused
|
|
17
|
+
across every call, with no pooling toggle and no fresh-connection-per-call. The
|
|
18
|
+
async context managers must STAY OPEN across calls, so we drive them on a
|
|
19
|
+
dedicated event loop running in a background thread; `call_tool` submits work
|
|
20
|
+
onto that loop and blocks for the result (the sync↔async bridge).
|
|
21
|
+
|
|
22
|
+
AMENDMENT (RCA 2026-09-04, "External scanners go permanently blind after one
|
|
23
|
+
MCP transport failure"): B5 originally also locked "no reconnect". In
|
|
24
|
+
production the MCP server's `POST /mcp` stream is severed at the 60 s Kong
|
|
25
|
+
upstream read timeout; the SDK then closes its streams but the client kept the
|
|
26
|
+
dead session for the life of the process, so every later `call_tool` raised
|
|
27
|
+
`ClosedResourceError` in ~1 ms until a runtime restart — 16–25 scanner MIDs
|
|
28
|
+
blind per day, and authored `scan()` code swallowing the error made every tick
|
|
29
|
+
report `ok` with zero candidates. The session is therefore now REBUILT: a
|
|
30
|
+
transport-level failure (closed/broken stream, connection reset, broken pipe,
|
|
31
|
+
httpx transport error) discards the dead session and the NEXT call opens and
|
|
32
|
+
initialises a fresh one. The failing call still fails — there is no retry
|
|
33
|
+
inside a call — the point is that the failure is no longer permanent. The
|
|
34
|
+
failure is logged once as `senpi_mcp_error` with `session_rebuilt: true`.
|
|
35
|
+
|
|
36
|
+
AMENDMENT, second half (same RCA): B5 also locked "no per-call timeout".
|
|
37
|
+
`call_tool` blocked on the loop's future with no bound, so one stalled
|
|
38
|
+
`POST /mcp` hung a call until Kong severed the stream at 60 s, and a handful of
|
|
39
|
+
such calls consumed the whole 300 s tick budget (`scaffold_tick_timeout`).
|
|
40
|
+
EVERY call is now BOUNDED — session open included — by
|
|
41
|
+
`call_timeout_seconds` (launch env `SENPI_MCP_CALL_TIMEOUT_SECONDS`, default
|
|
42
|
+
`DEFAULT_CALL_TIMEOUT_SECONDS`, well under the 60 s ceiling so the scaffold,
|
|
43
|
+
not the proxy, is what gives up). A call that outlives the bound raises
|
|
44
|
+
`SenpiMcpTimeoutError` (a `TimeoutError`) and is treated exactly like a
|
|
45
|
+
transport failure: the abandoned session is discarded — the server may still
|
|
46
|
+
write to that stream — and the next call opens a fresh one. It is logged as
|
|
47
|
+
`senpi_mcp_error` with `error_type: TimeoutError`, `timeout_seconds` and
|
|
48
|
+
`elapsed_ms` so a stall is distinguishable from a severed stream in ClickStack.
|
|
49
|
+
|
|
50
|
+
AMENDMENT, third part (dev 2026-09-07): the SDK's transport is an anyio task
|
|
51
|
+
group, so a failure raised through it arrives as an `ExceptionGroup` around the
|
|
52
|
+
real cause. The client classifies and logs the LEAF, never the group: a wrapped
|
|
53
|
+
closed stream still discards the session (the group's own type names no
|
|
54
|
+
transport class, so without unwrapping the dead session would have been kept —
|
|
55
|
+
the original wedge in a new coat), `error_type` is the leaf's class (never
|
|
56
|
+
`ExceptionGroup`), and a sole-leaf group is re-raised to `scan()` as that leaf.
|
|
57
|
+
Several leaves stay a group, logged as the first leaf plus `leaf_count`, and
|
|
58
|
+
rebuild when ANY leaf is a transport failure.
|
|
22
59
|
|
|
23
60
|
Response normalization MATCHES producer.mjs `createSenpiToolClient`:
|
|
24
61
|
1. isError truthy -> RAISE with the error text (never a silent None).
|
|
@@ -42,26 +79,37 @@ The `mcp` package is LAZY-IMPORTED on first connect (NOT at module top), so
|
|
|
42
79
|
`import scaffold.senpi_mcp` works even where `mcp` is not installed. Missing creds
|
|
43
80
|
are LOUD AT FIRST USE: `call_tool` raises, construction never does.
|
|
44
81
|
|
|
82
|
+
SESSION LIFETIME: each session lives inside ONE long-lived task on the loop
|
|
83
|
+
(`_session_lifetime`), which enters the transport + ClientSession context
|
|
84
|
+
managers, publishes the initialised session, then parks until a close is
|
|
85
|
+
requested and exits the same context managers from the same task. That
|
|
86
|
+
same-task shape is required: the SDK's transport is an anyio task group, and
|
|
87
|
+
anyio refuses to exit a cancel scope from a task other than the one that
|
|
88
|
+
entered it — so a dead session cannot be closed by calling `aclose()` from the
|
|
89
|
+
`call_tool` task. A rebuild only flips the session's close event; the old
|
|
90
|
+
lifetime task unwinds by itself, and `call_tool` never blocks on that teardown,
|
|
91
|
+
so a wedged transport cannot hang a tick.
|
|
92
|
+
|
|
45
93
|
TEARDOWN (intentional, one-process-per-scanner model): there is NO explicit
|
|
46
|
-
close()/shutdown for the
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
returns; on a hard kill the OS reclaims the thread. We deliberately do NOT
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
cross-thread aclose() on a wedged transport could itself hang a path that today
|
|
56
|
-
cannot hang.
|
|
94
|
+
close()/shutdown for the client. The background loop runs on a DAEMON thread,
|
|
95
|
+
so it dies with the process. The scaffold runs exactly ONE client per process
|
|
96
|
+
for that process's whole life — the supervisor restarts by spawning a FRESH
|
|
97
|
+
child process (not by re-running run_scaffold in-place), so a single client is
|
|
98
|
+
never replaced in-process and there is no loop leak across restart. On
|
|
99
|
+
graceful stop (SIGTERM/SIGINT) run_scaffold exits its loop and the process
|
|
100
|
+
returns; on a hard kill the OS reclaims the thread. We deliberately do NOT
|
|
101
|
+
drive a close from a shutdown hook: it buys nothing over the dropped TCP
|
|
102
|
+
connection (streamable-http is request/response, no protocol goodbye).
|
|
57
103
|
"""
|
|
58
104
|
|
|
59
105
|
from __future__ import annotations
|
|
60
106
|
|
|
61
107
|
import asyncio
|
|
108
|
+
import concurrent.futures
|
|
62
109
|
import json
|
|
63
110
|
import math
|
|
64
111
|
import threading
|
|
112
|
+
import time
|
|
65
113
|
from typing import Any, Iterable, Optional
|
|
66
114
|
|
|
67
115
|
from .loop_primitives import log_event
|
|
@@ -356,13 +404,194 @@ def _cast_market_numerics(name: str, payload: Any, args: Optional[dict] = None)
|
|
|
356
404
|
return payload
|
|
357
405
|
|
|
358
406
|
|
|
407
|
+
class TickErrorTally:
|
|
408
|
+
"""Per-tick count of MCP call failures, by error type.
|
|
409
|
+
|
|
410
|
+
Authored scan() code routinely catches the MCP exception and returns [], so
|
|
411
|
+
the exception alone never reaches the scaffold — the tick would finish `ok`
|
|
412
|
+
with zero candidates while the scanner is blind. The client records every
|
|
413
|
+
failure here as it re-raises; the scaffold resets the tally at tick start and
|
|
414
|
+
reads it at tick end to decide whether a candidate-less tick was really ok.
|
|
415
|
+
Thread-safe because call_tool may be driven from author threads.
|
|
416
|
+
"""
|
|
417
|
+
|
|
418
|
+
def __init__(self) -> None:
|
|
419
|
+
self._by_type: dict[str, int] = {}
|
|
420
|
+
self._lock = threading.Lock()
|
|
421
|
+
|
|
422
|
+
def record(self, error_type: str) -> None:
|
|
423
|
+
with self._lock:
|
|
424
|
+
self._by_type[error_type] = self._by_type.get(error_type, 0) + 1
|
|
425
|
+
|
|
426
|
+
def reset(self) -> None:
|
|
427
|
+
with self._lock:
|
|
428
|
+
self._by_type = {}
|
|
429
|
+
|
|
430
|
+
def snapshot(self) -> dict[str, Any]:
|
|
431
|
+
"""{count, dominant_error_type, by_type}; count 0 / None when clean.
|
|
432
|
+
|
|
433
|
+
The dominant type is the most frequent one, ties broken by first seen —
|
|
434
|
+
so one dominant `ClosedResourceError` reads as the wedge it is even when
|
|
435
|
+
a stray other failure sits beside it.
|
|
436
|
+
"""
|
|
437
|
+
with self._lock:
|
|
438
|
+
by_type = dict(self._by_type)
|
|
439
|
+
count = sum(by_type.values())
|
|
440
|
+
dominant = max(by_type, key=by_type.__getitem__) if by_type else None
|
|
441
|
+
return {"count": count, "dominant_error_type": dominant, "by_type": by_type}
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
# --------------------------------------------------------------------------
|
|
445
|
+
# Transport-failure classification — which errors mean "the session is dead".
|
|
446
|
+
# --------------------------------------------------------------------------
|
|
447
|
+
#
|
|
448
|
+
# Matched by class NAME along the MRO so the module keeps its lazy-import
|
|
449
|
+
# contract (no `anyio`/`httpx` import at module top). anyio raises the first
|
|
450
|
+
# three when the SDK's streams are closed or broken under a session; httpx's
|
|
451
|
+
# `TransportError` is the base of every connect/read/write/protocol/timeout
|
|
452
|
+
# failure on the POST itself. OSError covers ConnectionError and its subclasses
|
|
453
|
+
# (reset, refused, aborted), BrokenPipeError and — deliberately — TimeoutError,
|
|
454
|
+
# which is what the per-call bound raises (`SenpiMcpTimeoutError`): a call that
|
|
455
|
+
# timed out at the socket or under the bound has left the session in an unknown
|
|
456
|
+
# state (the server may still write to the abandoned stream) and the next call
|
|
457
|
+
# must not inherit it.
|
|
458
|
+
# Our own pre-connect raises (PermissionError for a blocked mutation tool,
|
|
459
|
+
# RuntimeError for missing creds) happen before any session is held, so they
|
|
460
|
+
# never reach this classification.
|
|
461
|
+
_TRANSPORT_ERROR_NAMES: frozenset[str] = frozenset(
|
|
462
|
+
{
|
|
463
|
+
"ClosedResourceError",
|
|
464
|
+
"BrokenResourceError",
|
|
465
|
+
"EndOfStream",
|
|
466
|
+
"TransportError",
|
|
467
|
+
}
|
|
468
|
+
)
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
# The SDK's `streamablehttp_client` runs an anyio task group, so a failure
|
|
472
|
+
# raised through it (a closed stream, an httpx error on the POST) surfaces as an
|
|
473
|
+
# `ExceptionGroup` wrapping the real cause — observed on dev 2026-09-07 as
|
|
474
|
+
# `error_type: ExceptionGroup, error: "unhandled errors in a TaskGroup (1
|
|
475
|
+
# sub-exception)"` against a 503. Classification and the fleet fingerprint must
|
|
476
|
+
# look at the LEAVES: the group's own MRO names no transport class, so a wrapped
|
|
477
|
+
# `ClosedResourceError` on a live session would otherwise keep the dead session
|
|
478
|
+
# (the pre-rebuild wedge). Matched by name so the `exceptiongroup` backport on
|
|
479
|
+
# Python < 3.11 unwraps like the builtin.
|
|
480
|
+
_EXCEPTION_GROUP_NAME = "BaseExceptionGroup"
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def _is_exception_group(exc: BaseException) -> bool:
|
|
484
|
+
return any(klass.__name__ == _EXCEPTION_GROUP_NAME for klass in type(exc).__mro__)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _leaf_exceptions(exc: BaseException) -> list[BaseException]:
|
|
488
|
+
"""Flatten an exception-group tree to its leaves; a plain exception is its own leaf."""
|
|
489
|
+
if not _is_exception_group(exc):
|
|
490
|
+
return [exc]
|
|
491
|
+
leaves: list[BaseException] = []
|
|
492
|
+
for sub in exc.exceptions: # type: ignore[attr-defined]
|
|
493
|
+
leaves.extend(_leaf_exceptions(sub))
|
|
494
|
+
return leaves or [exc]
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _unwrap_sole_leaf(exc: BaseException) -> BaseException:
|
|
498
|
+
"""The one real cause behind a group, else `exc` itself.
|
|
499
|
+
|
|
500
|
+
A group with exactly one leaf is the anyio wrapping and nothing more, so
|
|
501
|
+
the caller gets the leaf. Several leaves are genuinely several failures
|
|
502
|
+
and stay a group — the log line then names the first leaf and carries the
|
|
503
|
+
count.
|
|
504
|
+
"""
|
|
505
|
+
leaves = _leaf_exceptions(exc)
|
|
506
|
+
return leaves[0] if len(leaves) == 1 else exc
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _is_transport_error(exc: BaseException) -> bool:
|
|
510
|
+
"""True when `exc` means the MCP session's transport is unusable.
|
|
511
|
+
|
|
512
|
+
Looks through exception groups: a group is a transport failure when ANY of
|
|
513
|
+
its leaves is.
|
|
514
|
+
"""
|
|
515
|
+
return any(_leaf_is_transport_error(leaf) for leaf in _leaf_exceptions(exc))
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _leaf_is_transport_error(exc: BaseException) -> bool:
|
|
519
|
+
if isinstance(exc, (OSError, EOFError)):
|
|
520
|
+
return True
|
|
521
|
+
return any(klass.__name__ in _TRANSPORT_ERROR_NAMES for klass in type(exc).__mro__)
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
# --------------------------------------------------------------------------
|
|
525
|
+
# Per-call bound — the scaffold gives up before the proxy does.
|
|
526
|
+
# --------------------------------------------------------------------------
|
|
527
|
+
#
|
|
528
|
+
# Kong severs a stalled `POST /mcp` at 60.000 s; a tick has 300 s. 30 s is
|
|
529
|
+
# comfortably under the ceiling — a stall is visible as OUR timeout, not as a
|
|
530
|
+
# severed stream — and still leaves ten sequential calls inside one tick. The
|
|
531
|
+
# bound covers the whole call: opening the session (initialize) when there is
|
|
532
|
+
# none, plus the tool call itself.
|
|
533
|
+
DEFAULT_CALL_TIMEOUT_SECONDS: float = 30.0
|
|
534
|
+
|
|
535
|
+
# Launch-env knob for the bound, read by scaffold._build_senpi_mcp.
|
|
536
|
+
CALL_TIMEOUT_ENV_VAR = "SENPI_MCP_CALL_TIMEOUT_SECONDS"
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def is_valid_call_timeout(seconds: Any) -> bool:
|
|
540
|
+
"""A usable bound: a positive, finite number (bools are not numbers here).
|
|
541
|
+
|
|
542
|
+
Shared by the constructor (which rejects anything else) and the launch-env
|
|
543
|
+
reader in scaffold.py (which falls back to the default instead).
|
|
544
|
+
"""
|
|
545
|
+
if isinstance(seconds, bool) or not isinstance(seconds, (int, float)):
|
|
546
|
+
return False
|
|
547
|
+
return math.isfinite(seconds) and seconds > 0
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
class SenpiMcpTimeoutError(TimeoutError):
|
|
551
|
+
"""A `call_tool` outlived the client's per-call bound.
|
|
552
|
+
|
|
553
|
+
A `TimeoutError` so authored `scan()` code can `except TimeoutError`; a
|
|
554
|
+
distinct class so the scaffold's own bound is tellable from a socket
|
|
555
|
+
timeout. Classified as a transport failure: the session it happened on is
|
|
556
|
+
discarded and the next call opens a fresh one.
|
|
557
|
+
"""
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
class _McpSession:
|
|
561
|
+
"""One live MCP session: the SDK session plus the handles that end it.
|
|
562
|
+
|
|
563
|
+
`lifetime` is the loop task that entered the SDK's context managers and
|
|
564
|
+
will exit them once `close_requested` is set. `lifetime.done()` while the
|
|
565
|
+
session is still held means the transport died on its own.
|
|
566
|
+
"""
|
|
567
|
+
|
|
568
|
+
__slots__ = ("session", "lifetime", "close_requested")
|
|
569
|
+
|
|
570
|
+
def __init__(
|
|
571
|
+
self, session: Any, lifetime: "asyncio.Task[None]", close_requested: asyncio.Event
|
|
572
|
+
) -> None:
|
|
573
|
+
self.session = session
|
|
574
|
+
self.lifetime = lifetime
|
|
575
|
+
self.close_requested = close_requested
|
|
576
|
+
|
|
577
|
+
@property
|
|
578
|
+
def alive(self) -> bool:
|
|
579
|
+
return not self.lifetime.done()
|
|
580
|
+
|
|
581
|
+
|
|
359
582
|
class SenpiMcpClient:
|
|
360
583
|
"""Sync MCP client over the official async SDK with one persistent session.
|
|
361
584
|
|
|
362
585
|
Construction is cheap and never touches the network or imports `mcp`. The
|
|
363
586
|
first `call_tool` lazily imports the SDK, spins up a private event loop on a
|
|
364
|
-
background thread,
|
|
365
|
-
it
|
|
587
|
+
background thread (once, for the client's life), opens the transport +
|
|
588
|
+
ClientSession, and initializes it once. Every subsequent call reuses that
|
|
589
|
+
same session — until a transport failure, after which the dead session is
|
|
590
|
+
discarded and the next call opens a fresh one (B5 amendment, RCA 2026-09-04).
|
|
591
|
+
|
|
592
|
+
`tick_errors` is the per-tick failure tally the scaffold loop resets and
|
|
593
|
+
reads (see TickErrorTally); every failure call_tool re-raises is recorded
|
|
594
|
+
there exactly once, whatever its kind (transport, timeout, isError, ...).
|
|
366
595
|
"""
|
|
367
596
|
|
|
368
597
|
def __init__(
|
|
@@ -370,23 +599,38 @@ class SenpiMcpClient:
|
|
|
370
599
|
api_key: Optional[str] = None,
|
|
371
600
|
mcp_url: Optional[str] = None,
|
|
372
601
|
write_allowlist: Iterable[str] = (),
|
|
602
|
+
call_timeout_seconds: Optional[float] = None,
|
|
373
603
|
) -> None:
|
|
374
604
|
self._api_key = api_key
|
|
375
605
|
self._mcp_url = mcp_url
|
|
606
|
+
if call_timeout_seconds is None:
|
|
607
|
+
call_timeout_seconds = DEFAULT_CALL_TIMEOUT_SECONDS
|
|
608
|
+
if not is_valid_call_timeout(call_timeout_seconds):
|
|
609
|
+
raise ValueError(
|
|
610
|
+
"senpi_mcp: call_timeout_seconds must be a positive number, "
|
|
611
|
+
f"got {call_timeout_seconds!r}"
|
|
612
|
+
)
|
|
613
|
+
self._call_timeout = float(call_timeout_seconds)
|
|
376
614
|
# Mutation tools the operator has explicitly permitted. EMPTY by default
|
|
377
615
|
# → every MUTATION_TOOLS entry is blocked (read-only producer boundary).
|
|
378
616
|
self._write_allowlist: frozenset[str] = frozenset(write_allowlist)
|
|
617
|
+
self.tick_errors = TickErrorTally()
|
|
379
618
|
|
|
380
|
-
# Persistent bridge state
|
|
619
|
+
# Persistent bridge state. The loop + thread are created lazily on the
|
|
620
|
+
# first call_tool and live for the client's life; the session is
|
|
621
|
+
# (re)opened lazily whenever there is no live one.
|
|
381
622
|
self._loop: Optional[asyncio.AbstractEventLoop] = None
|
|
382
623
|
self._thread: Optional[threading.Thread] = None
|
|
383
|
-
self.
|
|
384
|
-
self._stack: Any = None
|
|
385
|
-
self._connected = False
|
|
624
|
+
self._current: Optional[_McpSession] = None
|
|
386
625
|
self._connect_lock = threading.Lock()
|
|
387
626
|
|
|
388
627
|
# -- public sync surface ----------------------------------------------
|
|
389
628
|
|
|
629
|
+
@property
|
|
630
|
+
def call_timeout_seconds(self) -> float:
|
|
631
|
+
"""The bound every `call_tool` fails within (session open included)."""
|
|
632
|
+
return self._call_timeout
|
|
633
|
+
|
|
390
634
|
def call_tool(self, name: str, args: Optional[dict] = None) -> Any:
|
|
391
635
|
"""Invoke an MCP tool synchronously; return the unwrapped result.
|
|
392
636
|
|
|
@@ -396,7 +640,20 @@ class SenpiMcpClient:
|
|
|
396
640
|
scaffold event) before it is re-raised, so even when the author's scan()
|
|
397
641
|
swallows the exception in its own try/except, the failure still leaves a
|
|
398
642
|
visible record at this boundary instead of vanishing into an empty result.
|
|
643
|
+
|
|
644
|
+
A transport-level failure (see `_is_transport_error`) additionally
|
|
645
|
+
discards the session it happened on, so the NEXT call opens a fresh one
|
|
646
|
+
instead of failing forever; its log event carries `session_rebuilt: true`.
|
|
647
|
+
|
|
648
|
+
The whole call — opening a session when there is none, then the tool
|
|
649
|
+
call — must finish within `call_timeout_seconds`, else it raises
|
|
650
|
+
`SenpiMcpTimeoutError` (logged with `error_type: TimeoutError` and the
|
|
651
|
+
elapsed ms) and the abandoned session is discarded like any other
|
|
652
|
+
transport failure.
|
|
399
653
|
"""
|
|
654
|
+
used: Optional[_McpSession] = None
|
|
655
|
+
started = time.monotonic()
|
|
656
|
+
deadline = started + self._call_timeout
|
|
400
657
|
try:
|
|
401
658
|
# Read-only producer boundary, enforced PRE-CONNECT (no transport
|
|
402
659
|
# opened for a blocked write). A mutation tool passes only if its exact
|
|
@@ -411,30 +668,63 @@ class SenpiMcpClient:
|
|
|
411
668
|
"not reachable by authored producer code)."
|
|
412
669
|
)
|
|
413
670
|
|
|
414
|
-
self._ensure_connected()
|
|
415
|
-
result = self._run(
|
|
671
|
+
used = self._ensure_connected(name, deadline)
|
|
672
|
+
result = self._run(used.session.call_tool(name, arguments=args), name, deadline)
|
|
416
673
|
return _cast_market_numerics(name, self._unwrap(name, result), args)
|
|
417
674
|
except Exception as exc:
|
|
418
675
|
# Catch every failure — mutation block, missing creds, the lazy
|
|
419
|
-
# `import mcp`, transport errors, tool `isError` —
|
|
420
|
-
# to the author's scan(). Scoped to
|
|
421
|
-
# (_TickTimeout, a BaseException) still
|
|
676
|
+
# `import mcp`, transport errors, timeouts, tool `isError` — tally
|
|
677
|
+
# it, log it, then re-raise to the author's scan(). Scoped to
|
|
678
|
+
# Exception so a tick timeout (_TickTimeout, a BaseException) still
|
|
679
|
+
# unwinds the tick untouched. This is the ONE exit for every failed
|
|
680
|
+
# call, so the tally sees each failure exactly once.
|
|
681
|
+
rebuilt = used is not None and _is_transport_error(exc) and self._discard(used)
|
|
682
|
+
# Classify and log the LEAF, never the anyio ExceptionGroup around
|
|
683
|
+
# it (see _leaf_exceptions): the fingerprint has to tell a 503 from
|
|
684
|
+
# a closed stream. One leaf is the failure; several are logged as
|
|
685
|
+
# the first leaf plus `leaf_count`.
|
|
686
|
+
leaves = _leaf_exceptions(exc)
|
|
687
|
+
primary = leaves[0]
|
|
688
|
+
timed_out = isinstance(primary, SenpiMcpTimeoutError)
|
|
689
|
+
# The fleet fingerprint for the bound is the builtin category, not
|
|
690
|
+
# the scaffold subclass: ClickStack queries `TimeoutError`. The tally
|
|
691
|
+
# counts under the SAME name, so the dominant type a blind tick
|
|
692
|
+
# reports to /errors matches the senpi_mcp_error lines beside it.
|
|
693
|
+
error_type = "TimeoutError" if timed_out else type(primary).__name__
|
|
694
|
+
self.tick_errors.record(error_type)
|
|
422
695
|
log_event(
|
|
423
696
|
"senpi_mcp_error",
|
|
424
697
|
tool=name,
|
|
425
|
-
error_type=
|
|
426
|
-
error=str(
|
|
698
|
+
error_type=error_type,
|
|
699
|
+
error=str(primary)[:500],
|
|
700
|
+
elapsed_ms=round((time.monotonic() - started) * 1000, 1),
|
|
701
|
+
**({"leaf_count": len(leaves)} if len(leaves) > 1 else {}),
|
|
702
|
+
**({"timeout_seconds": self._call_timeout} if timed_out else {}),
|
|
703
|
+
**({"session_rebuilt": True} if rebuilt else {}),
|
|
427
704
|
)
|
|
705
|
+
if len(leaves) == 1 and primary is not exc:
|
|
706
|
+
# The author's scan() sees the real cause (so `except OSError`
|
|
707
|
+
# or `except TimeoutError` keeps working), not the wrapper.
|
|
708
|
+
raise primary
|
|
428
709
|
raise
|
|
429
710
|
|
|
430
711
|
# -- persistent session lifecycle -------------------------------------
|
|
431
712
|
|
|
432
|
-
def _ensure_connected(self) ->
|
|
433
|
-
|
|
434
|
-
|
|
713
|
+
def _ensure_connected(self, name: str, deadline: float) -> _McpSession:
|
|
714
|
+
"""Return the live session, opening one when there is none.
|
|
715
|
+
|
|
716
|
+
A session whose lifetime task has ended on its own (the transport died
|
|
717
|
+
between calls) counts as absent and is replaced here, before the call.
|
|
718
|
+
Opening counts against the call's `deadline`: a stalled initialize
|
|
719
|
+
raises `SenpiMcpTimeoutError` like a stalled tool call would.
|
|
720
|
+
"""
|
|
721
|
+
current = self._current
|
|
722
|
+
if current is not None and current.alive:
|
|
723
|
+
return current
|
|
435
724
|
with self._connect_lock:
|
|
436
|
-
|
|
437
|
-
|
|
725
|
+
current = self._current
|
|
726
|
+
if current is not None and current.alive:
|
|
727
|
+
return current
|
|
438
728
|
|
|
439
729
|
# Loud at first use: creds are required to reach the MCP server.
|
|
440
730
|
if not self._api_key:
|
|
@@ -448,48 +738,114 @@ class SenpiMcpClient:
|
|
|
448
738
|
"(creds are resolved at launch from SENPI_API_KEY/SENPI_MCP_URL)."
|
|
449
739
|
)
|
|
450
740
|
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
741
|
+
if self._loop is None:
|
|
742
|
+
# Dedicated event loop on a background thread, created ONCE per
|
|
743
|
+
# client. The async context managers (transport + session) must
|
|
744
|
+
# stay OPEN across calls, so we cannot use asyncio.run() per
|
|
745
|
+
# call — that would tear the session down between calls. A
|
|
746
|
+
# rebuild reuses this loop; it never spawns another thread.
|
|
747
|
+
self._loop = asyncio.new_event_loop()
|
|
748
|
+
self._thread = threading.Thread(
|
|
749
|
+
target=self._loop.run_forever,
|
|
750
|
+
name="senpi-mcp-loop",
|
|
751
|
+
daemon=True,
|
|
752
|
+
)
|
|
753
|
+
self._thread.start()
|
|
462
754
|
|
|
463
|
-
self._run(self.
|
|
464
|
-
self.
|
|
755
|
+
self._current = self._run(self._open_session(), name, deadline)
|
|
756
|
+
return self._current
|
|
465
757
|
|
|
466
|
-
|
|
467
|
-
"""
|
|
758
|
+
def _discard(self, dead: _McpSession) -> bool:
|
|
759
|
+
"""Drop `dead` so the next call opens a fresh session; ask it to close.
|
|
760
|
+
|
|
761
|
+
Returns False (and touches nothing) when `dead` is no longer the held
|
|
762
|
+
session — a concurrent caller already replaced it, and its successor
|
|
763
|
+
must not be thrown away for the predecessor's failure. The close is
|
|
764
|
+
fire-and-forget: the lifetime task exits its own context managers, and
|
|
765
|
+
the caller never waits on a possibly-wedged transport.
|
|
766
|
+
"""
|
|
767
|
+
with self._connect_lock:
|
|
768
|
+
if self._current is not dead:
|
|
769
|
+
return False
|
|
770
|
+
self._current = None
|
|
771
|
+
self._loop.call_soon_threadsafe(dead.close_requested.set)
|
|
772
|
+
return True
|
|
773
|
+
|
|
774
|
+
async def _open_session(self) -> _McpSession:
|
|
775
|
+
"""Start a session lifetime task and wait until its session is initialised.
|
|
776
|
+
|
|
777
|
+
A failure while opening propagates to the caller; the lifetime task has
|
|
778
|
+
already unwound whatever it had entered, so nothing is left half-built.
|
|
779
|
+
A cancelled open (the caller's bound ran out while initialize stalled)
|
|
780
|
+
cancels the lifetime task too, which unwinds the transport from inside
|
|
781
|
+
the task that entered it — a stalled open never leaks a session.
|
|
782
|
+
"""
|
|
783
|
+
loop = asyncio.get_running_loop()
|
|
784
|
+
ready: "asyncio.Future[Any]" = loop.create_future()
|
|
785
|
+
close_requested = asyncio.Event()
|
|
786
|
+
lifetime = loop.create_task(self._session_lifetime(ready, close_requested))
|
|
787
|
+
try:
|
|
788
|
+
session = await ready
|
|
789
|
+
except asyncio.CancelledError:
|
|
790
|
+
lifetime.cancel()
|
|
791
|
+
raise
|
|
792
|
+
return _McpSession(session, lifetime, close_requested)
|
|
793
|
+
|
|
794
|
+
async def _session_lifetime(self, ready: "asyncio.Future[Any]", close_requested: asyncio.Event) -> None:
|
|
795
|
+
"""Own one session from open to close, all within this single task.
|
|
468
796
|
|
|
469
797
|
Lazy-import the SDK here so `import scaffold.senpi_mcp` succeeds in envs
|
|
470
798
|
without `mcp` installed (tests inject a fake `mcp` into sys.modules).
|
|
471
799
|
"""
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
800
|
+
try:
|
|
801
|
+
from mcp import ClientSession
|
|
802
|
+
from mcp.client.streamable_http import streamablehttp_client
|
|
803
|
+
|
|
804
|
+
headers = {"Authorization": f"Bearer {self._api_key}"}
|
|
805
|
+
async with streamablehttp_client(self._mcp_url, headers=headers) as (read, write, _):
|
|
806
|
+
async with ClientSession(read, write) as session:
|
|
807
|
+
await session.initialize()
|
|
808
|
+
ready.set_result(session)
|
|
809
|
+
await close_requested.wait()
|
|
810
|
+
except BaseException as exc:
|
|
811
|
+
if not ready.done():
|
|
812
|
+
# Opening failed: surface it to the waiting call_tool. The
|
|
813
|
+
# transport's anyio task group wraps the cause in an
|
|
814
|
+
# ExceptionGroup; the waiter gets the sole leaf (an httpx 503,
|
|
815
|
+
# a refused connection) so it can classify and re-raise the
|
|
816
|
+
# real error. A non-Exception (task cancellation) is wrapped
|
|
817
|
+
# so the waiter gets a plain error instead of blocking forever.
|
|
818
|
+
cause = _unwrap_sole_leaf(exc)
|
|
819
|
+
ready.set_exception(
|
|
820
|
+
cause
|
|
821
|
+
if isinstance(cause, Exception)
|
|
822
|
+
else RuntimeError(f"senpi_mcp: session open aborted by {type(cause).__name__}")
|
|
823
|
+
)
|
|
824
|
+
# Otherwise the transport died under a published session (the
|
|
825
|
+
# next call_tool sees `alive` False and rebuilds) or its teardown
|
|
826
|
+
# complained after a rebuild — neither has a caller left to tell.
|
|
827
|
+
if not isinstance(exc, Exception):
|
|
828
|
+
raise
|
|
829
|
+
|
|
830
|
+
def _run(self, coro: Any, name: str, deadline: float) -> Any:
|
|
831
|
+
"""Drive a coroutine on the persistent loop, blocking until `deadline`.
|
|
832
|
+
|
|
833
|
+
Past the deadline the loop-side task is cancelled and
|
|
834
|
+
`SenpiMcpTimeoutError` is raised. If the task finished in the instant
|
|
835
|
+
between the deadline and the cancel, its outcome is taken as-is — the
|
|
836
|
+
call made it, by a hair — so a completed open never leaks its session.
|
|
837
|
+
"""
|
|
491
838
|
future = asyncio.run_coroutine_threadsafe(coro, self._loop)
|
|
492
|
-
|
|
839
|
+
try:
|
|
840
|
+
return future.result(timeout=max(0.0, deadline - time.monotonic()))
|
|
841
|
+
except concurrent.futures.TimeoutError:
|
|
842
|
+
if not future.cancel():
|
|
843
|
+
return future.result()
|
|
844
|
+
raise SenpiMcpTimeoutError(
|
|
845
|
+
f'senpi_mcp: tool "{name}" did not complete within '
|
|
846
|
+
f"{self._call_timeout:g}s ({CALL_TIMEOUT_ENV_VAR}); the session (or "
|
|
847
|
+
"the pending open) is discarded and the next call opens a fresh one"
|
|
848
|
+
) from None
|
|
493
849
|
|
|
494
850
|
# -- response normalization (matches producer.mjs createSenpiToolClient) --
|
|
495
851
|
|
|
@@ -87,7 +87,7 @@ export type EventName = (typeof EVENT_NAME)[keyof typeof EVENT_NAME];
|
|
|
87
87
|
* queryable. Extend this set (not scattered string literals) when a new coded failure is introduced.
|
|
88
88
|
*/
|
|
89
89
|
export declare const SENPI_ERROR_CODE: {
|
|
90
|
-
/** A scaffold tick that threw (or `state_persist_failed`) — `scaffold.tick_error`. */
|
|
90
|
+
/** A scaffold tick that threw (or `state_persist_failed` / `mcp_error`) — `scaffold.tick_error`. */
|
|
91
91
|
readonly scannerTickError: "E_SCANNER_TICK_ERROR";
|
|
92
92
|
/** A scaffold tick that blew its wall-clock budget — `scaffold.tick_timeout`. */
|
|
93
93
|
readonly scannerTickTimeout: "E_SCANNER_TICK_TIMEOUT";
|
|
@@ -339,7 +339,8 @@ export interface IntakeDispositionAttrs {
|
|
|
339
339
|
/**
|
|
340
340
|
* Shared shape for the two scaffold tick-failure events from the `/errors` boundary — a scanner tick
|
|
341
341
|
* that threw or exceeded its budget (so it produced no signal). `senpi.scaffold.error_type` carries
|
|
342
|
-
* the scaffold's failure class (a Python exception name
|
|
342
|
+
* the scaffold's failure class (a Python exception name — for an `mcp_error` tick the dominant MCP
|
|
343
|
+
* error class — or `state_persist_failed`) on `tick_error`;
|
|
343
344
|
* it is omitted on `tick_timeout`, where the name says it. The raw message rides `redact`.
|
|
344
345
|
*/
|
|
345
346
|
export interface ScaffoldErrorAttrs {
|