@yahyahammad/venastine 0.3.9 → 0.3.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -20,6 +20,15 @@ it wrong got it wrong in the half the other had already fixed.
20
20
  There are THREE callers now -- `fetch_url.py` joined them in ROADMAP_v3 §51 --
21
21
  and the third is why §52 gave this class a size bound as well as a time one.
22
22
 
23
+ THE THIRD THING IS THE NETWORK METER (ROADMAP_v3 §53, batch 239), and it is
24
+ here for IO2's reason: this is the one module every network-backed tool
25
+ already imports, `core/reasoning/scholar.py` included, so a second home
26
+ would be a second place to forget. Three meters -- tools, the provider API
27
+ and MCP over HTTP -- each with its own ceiling over one rolling window,
28
+ counted into a small machine-wide SQLite ledger. Its section, at the end of
29
+ this file, says what is counted exactly, what is an estimate, and what
30
+ nothing here can see.
31
+
23
32
  WHAT IS DELIBERATELY *NOT* SHARED. Each tool keeps its OWN instance, so the
24
33
  two key spaces stay separate -- a `TTLCache` shared between them would let
25
34
  an arXiv query and a web query collide on an equal key string, which is a
@@ -31,12 +40,21 @@ a factory would make the params class harder to read to save two lines.
31
40
 
32
41
  from __future__ import annotations
33
42
 
43
+ import atexit
44
+ import contextlib
45
+ import json
34
46
  import logging
47
+ import os
48
+ import sqlite3
35
49
  import threading
36
50
  import time
51
+ from dataclasses import dataclass
52
+ from functools import cache
37
53
  from typing import Any, Optional
38
54
 
55
+ import config
39
56
  from safety.policy_enforcement import redact_output_text
57
+ from security import protected_paths
40
58
 
41
59
  logger = logging.getLogger(__name__)
42
60
 
@@ -80,6 +98,17 @@ MAX_ENTRIES = 128
80
98
  MAX_URL_CHARS = 400
81
99
 
82
100
 
101
+ # Batch 240. Who this harness says it is when it fetches a page itself.
102
+ # Wikimedia answers httpx's default `python-httpx/<version>` with 403 --
103
+ # measured on en.wikipedia.org, where a descriptive agent got 200 -- and its
104
+ # policy asks a client to name itself and a way to reach its maintainers.
105
+ # The project's own public repository is that way; nothing about the user
106
+ # is in it. `fetch_url` and the scholar lookup send it; `arxiv_search` keeps
107
+ # the agent arXiv's API was already given.
108
+ USER_AGENT = ("venastine-research-harness "
109
+ "(+https://github.com/YahyaHammad/Venastine-Research-Harness)")
110
+
111
+
83
112
  class ThirdPartyText:
84
113
  """One tool call's worth of strings nobody here wrote, REDACTED and
85
114
  THEN cut.
@@ -265,3 +294,852 @@ class TTLCache:
265
294
  test's answers -- the same reason conftest isolates the stores."""
266
295
  with self._lock:
267
296
  self._entries.clear()
297
+
298
+
299
+ # ===========================================================================
300
+ # ---- The network meter (ROADMAP_v3 §53, batch 239) ------------------------
301
+ # ===========================================================================
302
+ #
303
+ # UNLIMITED BY DEFAULT, AND THEN NOTHING HERE DOES ANYTHING (IO1). Every
304
+ # ceiling in config.yaml ships at 0. A meter with no ceiling admits every
305
+ # call, counts nothing and never opens the ledger, so a harness nobody has
306
+ # configured behaves exactly as it did before this section existed and
307
+ # writes no file for it.
308
+ #
309
+ # WHAT IS COUNTED, AND HOW EXACTLY (IO3, IO4). A request and its response as
310
+ # HTTP/1.1 frames them: the request line, the headers, the body, the status
311
+ # line, the response headers and the body AS IT CAME OFF THE WIRE --
312
+ # compressed, and without chunk framing. Measured (batch 239) against a
313
+ # local server: a 50,000-byte page served gzipped counts as its 143 wire
314
+ # bytes, not its 50,000 decoded ones, because that is what an allowance is
315
+ # charged. Every redirect hop is counted, its body too.
316
+ #
317
+ # WHAT IS NOT (IO5), so a ceiling is never read as more than it is: TLS and
318
+ # TCP overhead, HTTP/2's header compression, bytes the kernel had already
319
+ # received when a read stopped early, sign-in and token-refresh traffic
320
+ # (`/connect` and the ChatGPT plan's refresh), a `stdio` MCP server's own
321
+ # sockets, and everything a shell command does. `web_search` is an
322
+ # ESTIMATE (IO15), kept apart so every figure that includes it can say so.
323
+
324
+ #: The three meters, in the order every listing shows them (IO1).
325
+ METERS = ("tools", "provider", "mcp")
326
+
327
+ #: What a person reads for each, in a refusal and in `/network`.
328
+ METER_LABELS = {"tools": "tool", "provider": "provider API", "mcp": "MCP"}
329
+
330
+ #: A ceiling is in DECIMAL megabytes (IO9), which is how a data allowance
331
+ #: is sold and how a carrier counts it.
332
+ MEGABYTE = 1_000_000
333
+
334
+ #: The ledger is written at most this often (IO12), and only when something
335
+ #: was counted. It is also how stale another process's figures may get
336
+ #: before this one reads the file again.
337
+ FLUSH_INTERVAL_S = 60.0
338
+
339
+ #: The ledger holds at most this many buckets per meter (IO10). A bucket is
340
+ #: the window divided by this, never narrower than MIN_BUCKET_S, so the
341
+ #: file's size is bounded by the bucket count and not by the traffic: 50 MB
342
+ #: a day and 50 GB a day both add to one integer per bucket.
343
+ LEDGER_BUCKETS = 1440
344
+ MIN_BUCKET_S = 60
345
+
346
+ LEDGER_FILENAME = "network.db"
347
+
348
+ #: IO15, amending IO3, the owner's choice. What one `web_search` costs on
349
+ #: the wire, per result requested. Measured (batch 239) through a counting
350
+ #: CONNECT proxy on two live queries: 85,746 bytes for five results and
351
+ #: 206,637 for ten, against 1,967 and 2,445 bytes of rows returned -- so
352
+ #: IO3's "the size of what it returned" would have undercounted the tool
353
+ #: by 44x and 85x. The higher ratio, rounded up, so the estimate leans
354
+ #: high rather than low. Re-measure when `ddgs` moves: which backends it
355
+ #: queries decides this number.
356
+ WEB_SEARCH_BYTES_PER_RESULT = 21_000
357
+
358
+ #: How long a ledger write waits on another process's lock. Short, because
359
+ #: a writer holds it for one tiny transaction.
360
+ _BUSY_TIMEOUT_S = 2.0
361
+
362
+ #: The WAL file is cut back to this after a checkpoint, so it cannot keep
363
+ #: the size of the busiest minute it ever saw.
364
+ _JOURNAL_LIMIT_BYTES = 256 * 1024
365
+
366
+ _SCHEMA = (
367
+ "CREATE TABLE IF NOT EXISTS usage ("
368
+ " meter TEXT NOT NULL,"
369
+ " bucket INTEGER NOT NULL,"
370
+ " bytes INTEGER NOT NULL,"
371
+ " estimated INTEGER NOT NULL DEFAULT 0,"
372
+ " PRIMARY KEY (meter, bucket)) WITHOUT ROWID")
373
+
374
+ # An ADD, not a replace: two processes flushing the same bucket must both
375
+ # count. This is why the ledger is SQLite rather than a JSON file written
376
+ # atomically -- an atomic replace of a read-modify-write still loses the
377
+ # other writer's increment.
378
+ _UPSERT = (
379
+ "INSERT INTO usage (meter, bucket, bytes, estimated) VALUES (?, ?, ?, ?)"
380
+ " ON CONFLICT (meter, bucket) DO UPDATE SET"
381
+ " bytes = bytes + excluded.bytes,"
382
+ " estimated = estimated + excluded.estimated")
383
+
384
+
385
+ def window_s() -> float:
386
+ """The rolling window, in seconds, read at call time (IO7, IO8)."""
387
+ return float(config.NETWORK_WINDOW_HOURS) * 3600.0
388
+
389
+
390
+ def bucket_width_s() -> int:
391
+ """How wide one ledger bucket is for the current window (IO10)."""
392
+ return max(MIN_BUCKET_S, int(window_s() // LEDGER_BUCKETS))
393
+
394
+
395
+ class NetworkLimitReached(Exception):
396
+ """A call refused because its meter has no room for it (IO6).
397
+
398
+ RAISED, NOT RETURNED, so that one boundary per consumer decides what it
399
+ becomes, and no caller has to remember to check: `registry.dispatch`
400
+ turns it into the tool result the model reads, `core/provider_errors`
401
+ makes it a failure that is never retried and is described in these
402
+ words even under an SDK that wrapped it, and `mcp_client` returns it in
403
+ its one error shape. `reason` is the sentence all three show.
404
+ """
405
+
406
+ def __init__(self, reason: str, meter: str) -> None:
407
+ super().__init__(reason)
408
+ self.reason = reason
409
+ self.meter = meter
410
+
411
+
412
+ @dataclass(frozen=True)
413
+ class MeterState:
414
+ """One meter, as `/network`, the sidebar and a refusal read it."""
415
+
416
+ name: str
417
+ label: str
418
+ ceiling: int # bytes; 0 means this meter is off
419
+ used: int # bytes in the window, estimate included
420
+ estimated: int # the part of `used` that is an estimate
421
+ window_s: float
422
+ room_at: Optional[float] # when room returns, if the meter is full
423
+ ledger_note: Optional[str] # why the ledger is not in use, if it is not
424
+
425
+ @property
426
+ def enabled(self) -> bool:
427
+ return self.ceiling > 0
428
+
429
+ @property
430
+ def full(self) -> bool:
431
+ return self.enabled and self.used >= self.ceiling
432
+
433
+
434
+ def format_bytes(n: int) -> str:
435
+ """`12.3 MB` -- decimal units, to match the ceiling's (IO9)."""
436
+ if n < 1000:
437
+ return f"{n} B"
438
+ if n < MEGABYTE:
439
+ return f"{n / 1000:.1f} kB"
440
+ if n < 1000 * MEGABYTE:
441
+ return f"{n / MEGABYTE:.1f} MB"
442
+ return f"{n / (1000 * MEGABYTE):.2f} GB"
443
+
444
+
445
+ def format_bytes_short(n: int) -> str:
446
+ """`12M` -- the sidebar's spelling, which has 22 columns to work in."""
447
+ if n < 1000:
448
+ return f"{n}"
449
+ if n < MEGABYTE:
450
+ return f"{n / 1000:.0f}k"
451
+ if n < 10 * MEGABYTE:
452
+ return f"{n / MEGABYTE:.1f}M"
453
+ if n < 1000 * MEGABYTE:
454
+ return f"{n / MEGABYTE:.0f}M"
455
+ return f"{n / (1000 * MEGABYTE):.1f}G"
456
+
457
+
458
+ def describe_window(seconds: float) -> str:
459
+ """`24 h`, or `7 days` for a whole number of days past two."""
460
+ hours = seconds / 3600.0
461
+ if hours >= 48 and hours % 24 == 0:
462
+ return f"{hours / 24:g} days"
463
+ return f"{hours:g} h"
464
+
465
+
466
+ def _clock(at: float) -> str:
467
+ """A local time, with the date only when it is not today."""
468
+ when = time.localtime(at)
469
+ if time.strftime("%Y-%m-%d", when) == time.strftime("%Y-%m-%d"):
470
+ return time.strftime("%H:%M", when)
471
+ return time.strftime("%Y-%m-%d %H:%M", when)
472
+
473
+
474
+ def _room_at(buckets: list, used: int, ceiling: int, outgoing: int,
475
+ window: float, width: int) -> Optional[float]:
476
+ """When a request sending `outgoing` bytes would next be admitted.
477
+
478
+ The oldest bytes leave first, so walk the buckets oldest to newest and
479
+ find the first whose departure brings the meter under the line. None
480
+ when the request is larger than the whole ceiling, which no wait fixes.
481
+ """
482
+ need = max(outgoing, 0)
483
+ if need > ceiling:
484
+ return None
485
+ remaining = used
486
+ for bucket, nbytes, _ in buckets:
487
+ remaining -= nbytes
488
+ if remaining < ceiling and remaining + need <= ceiling:
489
+ return bucket + width + window
490
+ return None
491
+
492
+
493
+ class Ledger:
494
+ """The machine-wide record the meters count into (IO10, IO12).
495
+
496
+ ONE FILE FOR EVERY HARNESS PROCESS, `~/.config/venastine/network.db`,
497
+ because IO7 rejects a reset per run and a restart is a run: a counter
498
+ that lived in memory would be emptied by quitting, which is the
499
+ short-run loophole IO7 names. Every subagent shares it for the same
500
+ reason -- they run in this process and count into this instance.
501
+
502
+ COUNTS, NEVER TRAFFIC, AND BOUNDED BY TIME RATHER THAN BY VOLUME. A row
503
+ is (meter, bucket start, bytes, estimated bytes). Rows older than the
504
+ window are deleted on every write and SQLite reuses the freed pages, so
505
+ the file stops at about `LEDGER_BUCKETS` rows per meter whatever the
506
+ traffic was.
507
+
508
+ WRITTEN AT MOST ONCE A MINUTE, AND NEVER WHILE IDLE (IO12, the owner's
509
+ concern about write wear). Counts gather in memory; the first one after
510
+ a flush starts a one-shot timer, and the flush it fires is one
511
+ transaction of a few rows. With nothing counted there is no timer and no
512
+ write, and with every meter off there is no file at all. A refusal
513
+ decision reads memory, never the disk; the file is read again only when
514
+ the last read is more than a minute old.
515
+
516
+ THE COSTS, STATED: another process's traffic can be up to a minute late
517
+ here, and a crash loses at most the last minute of counts -- an
518
+ UNDER-count, which the gap register records. A file that cannot be
519
+ read or written does not turn the meter off: this process's own counts
520
+ still enforce, and `note` says the ledger is not in use.
521
+ """
522
+
523
+ def __init__(self, path: Optional[str] = None, clock=time.time,
524
+ timers: bool = True) -> None:
525
+ self._path = path
526
+ self._clock = clock
527
+ self._timers = timers
528
+ # (meter, bucket) -> [bytes, estimated]. `_stored` is what the file
529
+ # said at the last read, every process's flushed counts included;
530
+ # `_pending` is this process's, not yet written. Disjoint by
531
+ # construction: a flush moves pending into the file and reads the
532
+ # file back, so nothing is ever in both.
533
+ self._stored: dict = {}
534
+ self._pending: dict = {}
535
+ self._read_at: Optional[float] = None
536
+ self._timer: Optional[threading.Timer] = None
537
+ self._note: Optional[str] = None
538
+ # NOT reentrant, for TTLCache's reason: every `_locked` method is
539
+ # called with it held and must not take it again.
540
+ self._lock = threading.Lock()
541
+
542
+ # -- reading ---------------------------------------------------------
543
+
544
+ def now(self) -> float:
545
+ return self._clock()
546
+
547
+ def path(self) -> str:
548
+ """Resolved at CALL time, so a redirected HOME is honoured."""
549
+ return self._path or os.path.join(
550
+ protected_paths.user_config_dir(), LEDGER_FILENAME)
551
+
552
+ @property
553
+ def note(self) -> Optional[str]:
554
+ return self._note
555
+
556
+ def usage(self, meter: str, now: Optional[float] = None) -> list:
557
+ """`[(bucket, bytes, estimated)]` inside the window, oldest first.
558
+
559
+ A bucket STRADDLING the window's old edge is counted whole, so the
560
+ figure errs high by at most one bucket (1/1440 of the window) and
561
+ never low -- a ceiling should refuse a moment early rather than
562
+ admit a moment late.
563
+ """
564
+ now = self._clock() if now is None else now
565
+ with self._lock:
566
+ if self._read_at is None or now - self._read_at >= FLUSH_INTERVAL_S:
567
+ self._refresh_locked(now)
568
+ edge = now - window_s() - bucket_width_s()
569
+ merged: dict = {}
570
+ for source in (self._stored, self._pending):
571
+ for (name, bucket), (nbytes, estimated) in source.items():
572
+ if name == meter and bucket > edge:
573
+ slot = merged.setdefault(bucket, [0, 0])
574
+ slot[0] += nbytes
575
+ slot[1] += estimated
576
+ return sorted((bucket, n, e) for bucket, (n, e) in merged.items())
577
+
578
+ # -- counting --------------------------------------------------------
579
+
580
+ def add(self, meter: str, nbytes: int, estimated: bool = False,
581
+ now: Optional[float] = None) -> None:
582
+ now = self._clock() if now is None else now
583
+ width = bucket_width_s()
584
+ bucket = int(now // width * width)
585
+ with self._lock:
586
+ slot = self._pending.setdefault((meter, bucket), [0, 0])
587
+ slot[0] += nbytes
588
+ if estimated:
589
+ slot[1] += nbytes
590
+ self._schedule_locked()
591
+
592
+ def flush(self) -> None:
593
+ """Write what this process counted, then read every process's back.
594
+
595
+ Called by the timer, at exit, and by tests. A failure keeps the
596
+ counts pending for the next attempt and says why, once.
597
+ """
598
+ with self._lock:
599
+ self._timer = None
600
+ if not self._pending:
601
+ return
602
+ now = self._clock()
603
+ try:
604
+ self._write_locked(now)
605
+ except (sqlite3.Error, OSError) as exc:
606
+ self._fail_locked("written", exc)
607
+ # Bounded even while the file is failing: a bucket that has
608
+ # left the window is no use to anyone.
609
+ edge = now - window_s() - 2 * bucket_width_s()
610
+ self._pending = {key: value
611
+ for key, value in self._pending.items()
612
+ if key[1] > edge}
613
+
614
+ def cancel(self) -> None:
615
+ """Stop a scheduled flush. For tests, which replace the ledger."""
616
+ with self._lock:
617
+ if self._timer is not None:
618
+ self._timer.cancel()
619
+ self._timer = None
620
+
621
+ # -- the file --------------------------------------------------------
622
+
623
+ def _schedule_locked(self) -> None:
624
+ if self._timers and self._timer is None:
625
+ timer = threading.Timer(FLUSH_INTERVAL_S, self.flush)
626
+ timer.daemon = True
627
+ self._timer = timer
628
+ timer.start()
629
+
630
+ def _connect(self, create: bool) -> Optional[sqlite3.Connection]:
631
+ path = self.path()
632
+ if not create and not os.path.exists(path):
633
+ return None
634
+ if create:
635
+ os.makedirs(os.path.dirname(path), mode=0o700, exist_ok=True)
636
+ # Autocommit, so the one transaction below is spelled out rather
637
+ # than opened implicitly by the first statement.
638
+ db = sqlite3.connect(path, timeout=_BUSY_TIMEOUT_S,
639
+ isolation_level=None)
640
+ try:
641
+ if create:
642
+ # WAL with synchronous=NORMAL: a commit appends to the log
643
+ # and does not wait on fsync; only a checkpoint does. The
644
+ # last connection to close checkpoints and removes the log.
645
+ db.execute("PRAGMA journal_mode=WAL")
646
+ db.execute("PRAGMA synchronous=NORMAL")
647
+ db.execute(f"PRAGMA journal_size_limit={_JOURNAL_LIMIT_BYTES}")
648
+ db.execute(_SCHEMA)
649
+ except BaseException:
650
+ db.close()
651
+ raise
652
+ return db
653
+
654
+ def _select_locked(self, db: sqlite3.Connection, edge: float) -> dict:
655
+ rows = db.execute(
656
+ "SELECT meter, bucket, bytes, estimated FROM usage"
657
+ " WHERE bucket > ?", (int(edge),)).fetchall()
658
+ return {(meter, bucket): [nbytes, estimated]
659
+ for meter, bucket, nbytes, estimated in rows}
660
+
661
+ def _refresh_locked(self, now: float) -> None:
662
+ # Stamped even when the read fails, so a broken file is tried once
663
+ # a minute rather than on every call.
664
+ self._read_at = now
665
+ edge = now - window_s() - bucket_width_s()
666
+ try:
667
+ db = self._connect(create=False)
668
+ if db is None:
669
+ self._stored = {}
670
+ return
671
+ try:
672
+ self._stored = self._select_locked(db, edge)
673
+ finally:
674
+ db.close()
675
+ except (sqlite3.Error, OSError) as exc:
676
+ self._fail_locked("read", exc)
677
+
678
+ def _write_locked(self, now: float) -> None:
679
+ width = bucket_width_s()
680
+ # Two buckets of margin past the window's edge: the straddling one
681
+ # `usage` still counts, and one more so a clock that has moved on
682
+ # between processes cannot delete a row another still reads.
683
+ horizon = int(now - window_s()) - 2 * width
684
+ rows = [(meter, bucket, nbytes, estimated)
685
+ for (meter, bucket), (nbytes, estimated)
686
+ in self._pending.items() if bucket > horizon]
687
+ db = self._connect(create=True)
688
+ try:
689
+ db.execute("BEGIN IMMEDIATE")
690
+ try:
691
+ db.executemany(_UPSERT, rows)
692
+ db.execute("DELETE FROM usage WHERE bucket <= ?", (horizon,))
693
+ db.execute("COMMIT")
694
+ except BaseException:
695
+ with contextlib.suppress(sqlite3.Error):
696
+ db.execute("ROLLBACK")
697
+ raise
698
+ stored = self._select_locked(db, now - window_s() - width)
699
+ finally:
700
+ db.close()
701
+ self._pending.clear()
702
+ self._stored = stored
703
+ self._read_at = now
704
+ if self._note is not None:
705
+ logger.warning("The network ledger at %s is in use again.",
706
+ self.path())
707
+ self._note = None
708
+
709
+ def _fail_locked(self, verb: str, exc: BaseException) -> None:
710
+ note = (f"the ledger at {self.path()} could not be {verb} "
711
+ f"({type(exc).__name__}: {exc}); counting this process only")
712
+ if note != self._note:
713
+ logger.warning("Network meter: %s.", note)
714
+ self._note = note
715
+
716
+
717
+ #: The one ledger every meter in this process counts into. Replaced, never
718
+ #: mutated, by the tests that turn a meter on (tests/conftest.py).
719
+ LEDGER = Ledger()
720
+
721
+
722
+ def _ledger() -> Ledger:
723
+ """The CURRENT ledger, looked up at call time so a test's swap is seen."""
724
+ return LEDGER
725
+
726
+
727
+ @atexit.register
728
+ def _flush_at_exit() -> None:
729
+ """The last minute's counts, written as the process leaves (IO12)."""
730
+ with contextlib.suppress(Exception):
731
+ _ledger().flush()
732
+
733
+
734
+ class Meter:
735
+ """One of the three (IO1): a ceiling, and the arithmetic against it.
736
+
737
+ THE CEILING IS READ AT CALL TIME from `config`, like every other
738
+ setting a test may monkeypatch; nothing here caches it.
739
+
740
+ ADMISSION, NEVER TRUNCATION (IO6, IO11). `admit` is asked before a call
741
+ starts, with the size of what the call will SEND when that is known,
742
+ and refuses when the meter is full or the send would pass the ceiling.
743
+ A response already in flight is never cut: it completes and is counted,
744
+ so a meter can end a window over its ceiling by at most one response --
745
+ and a half-read body is the spliced-document failure PG5 exists to
746
+ prevent.
747
+ """
748
+
749
+ def __init__(self, name: str) -> None:
750
+ self.name = name
751
+ self.label = METER_LABELS[name]
752
+ # Whether the last decision was a refusal, so a run of them warns
753
+ # once (the transcript line IO6 owes) and not once per call.
754
+ self._refusing = False
755
+ self._lock = threading.Lock()
756
+
757
+ def ceiling(self) -> int:
758
+ megabytes = getattr(config, f"NETWORK_LIMIT_{self.name.upper()}_MB", 0)
759
+ # Rounded, not truncated: 2.01 MB is 2,009,999.9999999998 bytes in
760
+ # binary floating point, and a ceiling must not shrink in the
761
+ # conversion.
762
+ return int(round(float(megabytes or 0) * MEGABYTE))
763
+
764
+ def enabled(self) -> bool:
765
+ return self.ceiling() > 0
766
+
767
+ def _measure(self) -> tuple:
768
+ ceiling = self.ceiling()
769
+ window = window_s()
770
+ if not ceiling:
771
+ return MeterState(self.name, self.label, 0, 0, 0, window,
772
+ None, None), []
773
+ ledger = _ledger()
774
+ buckets = ledger.usage(self.name)
775
+ used = sum(nbytes for _, nbytes, _ in buckets)
776
+ estimated = sum(part for _, _, part in buckets)
777
+ room = (_room_at(buckets, used, ceiling, 0, window, bucket_width_s())
778
+ if used >= ceiling else None)
779
+ return MeterState(self.name, self.label, ceiling, used, estimated,
780
+ window, room, ledger.note), buckets
781
+
782
+ def state(self) -> MeterState:
783
+ return self._measure()[0]
784
+
785
+ def admit(self, outgoing: int = 0) -> None:
786
+ """Refuse, by raising NetworkLimitReached, or return and admit."""
787
+ if not self.enabled():
788
+ return
789
+ state, buckets = self._measure()
790
+ if state.used < state.ceiling and state.used + outgoing <= state.ceiling:
791
+ with self._lock:
792
+ self._refusing = False
793
+ return
794
+ reason = self._refusal(state, buckets, outgoing)
795
+ with self._lock:
796
+ first, self._refusing = not self._refusing, True
797
+ if first:
798
+ logger.warning("Network limit: %s", reason)
799
+ raise NetworkLimitReached(reason, self.name)
800
+
801
+ def record(self, nbytes: int, estimated: bool = False) -> None:
802
+ if nbytes > 0 and self.enabled():
803
+ _ledger().add(self.name, int(nbytes), estimated)
804
+
805
+ def _refusal(self, state: MeterState, buckets: list, outgoing: int) -> str:
806
+ ceiling, window = state.ceiling, describe_window(state.window_s)
807
+ if outgoing > ceiling:
808
+ text = (f"The {self.label} network limit cannot admit this "
809
+ f"request: it sends {format_bytes(outgoing)}, more than "
810
+ f"the whole {format_bytes(ceiling)} allowed per {window}.")
811
+ elif state.used >= ceiling:
812
+ text = (f"The {self.label} network limit is reached: "
813
+ f"{format_bytes(state.used)} of {format_bytes(ceiling)} "
814
+ f"used in the last {window}.")
815
+ else:
816
+ text = (f"The {self.label} network limit would be passed: this "
817
+ f"request sends {format_bytes(outgoing)} and "
818
+ f"{format_bytes(ceiling - state.used)} of "
819
+ f"{format_bytes(ceiling)} is left for the last {window}.")
820
+ if state.estimated:
821
+ text += (f" {format_bytes(state.estimated)} of the use is an "
822
+ f"estimate (web_search).")
823
+ room = _room_at(buckets, state.used, ceiling, outgoing,
824
+ state.window_s, bucket_width_s())
825
+ if room is not None:
826
+ text += f" Room returns at about {_clock(room)}."
827
+ text += (f" The ceiling is network_limit_{self.name}_mb in "
828
+ f"config.yaml.")
829
+ if self.name != "provider":
830
+ # The model reads this one. The limit is the user's, and the
831
+ # shell is where an agent would reach next (gap register).
832
+ text += " It is the user's limit: do not work around it."
833
+ return text
834
+
835
+
836
+ TOOLS = Meter("tools")
837
+ PROVIDER = Meter("provider")
838
+ MCP = Meter("mcp")
839
+ _BY_NAME = {meter.name: meter for meter in (TOOLS, PROVIDER, MCP)}
840
+
841
+
842
+ def meter(name: str) -> Meter:
843
+ return _BY_NAME[name]
844
+
845
+
846
+ def states() -> list:
847
+ """Every meter's state, in METERS order."""
848
+ return [_BY_NAME[name].state() for name in METERS]
849
+
850
+
851
+ # ---- Sizing an exchange ----------------------------------------------------
852
+ #
853
+ # DUCK-TYPED over httpx and httpx2, which share these attribute names: the
854
+ # tools' own calls are httpx, the provider SDKs and MCP are httpx2. Total
855
+ # over a wrong shape -- every accessor is guarded, and a part that cannot
856
+ # be read counts as zero -- because sizing a request must never be the
857
+ # reason it fails.
858
+
859
+ _HTTP11_REQUEST_TAIL = len(" HTTP/1.1\r\n")
860
+
861
+
862
+ def _headers_bytes(headers) -> int:
863
+ raw = getattr(headers, "raw", None)
864
+ if raw is not None:
865
+ pairs = list(raw)
866
+ elif hasattr(headers, "items"):
867
+ pairs = list(headers.items())
868
+ else:
869
+ return 0
870
+ total = 2 # the blank line
871
+ for key, value in pairs:
872
+ total += len(key) + len(value) + 4 # ": " and CRLF
873
+ return total
874
+
875
+
876
+ def request_bytes(request) -> int:
877
+ """One request as HTTP/1.1 frames it: line, headers and body."""
878
+ if request is None:
879
+ return 0
880
+ try:
881
+ target = getattr(request.url, "raw_path", b"/")
882
+ total = len(request.method) + 1 + len(target) + _HTTP11_REQUEST_TAIL
883
+ total += _headers_bytes(request.headers)
884
+ length = request.headers.get("content-length")
885
+ if length is not None:
886
+ total += int(length)
887
+ else:
888
+ with contextlib.suppress(Exception):
889
+ total += len(request.content)
890
+ return total
891
+ except Exception: # noqa: BLE001 -- see the section comment
892
+ return 0
893
+
894
+
895
+ def response_head_bytes(response) -> int:
896
+ """The status line and headers. The body is counted as it is read."""
897
+ try:
898
+ reason = getattr(response, "reason_phrase", "") or ""
899
+ line = len("HTTP/1.1 000 \r\n") + len(reason)
900
+ return line + _headers_bytes(response.headers)
901
+ except Exception: # noqa: BLE001
902
+ return 0
903
+
904
+
905
+ def _request_of(response):
906
+ """The request a response answered, or None.
907
+
908
+ A PROPERTY THAT RAISES rather than an attribute that may be missing:
909
+ httpx's `Response.request` raises RuntimeError when no request was set,
910
+ so `getattr` with a default does not cover it.
911
+ """
912
+ try:
913
+ return response.request
914
+ except Exception: # noqa: BLE001 -- see the section comment
915
+ return None
916
+
917
+
918
+ def exchange_bytes(response) -> int:
919
+ """Every hop of a response the CALLER read: request, head and body.
920
+
921
+ `num_bytes_downloaded` is httpx's count of body bytes as received --
922
+ before decompression, after chunk framing -- which is the wire figure
923
+ (measured, above). `history` holds the redirect hops httpx followed.
924
+ """
925
+ total = 0
926
+ for hop in [*getattr(response, "history", ()), response]:
927
+ total += request_bytes(_request_of(hop))
928
+ total += response_head_bytes(hop)
929
+ total += int(getattr(hop, "num_bytes_downloaded", 0) or 0)
930
+ return total
931
+
932
+
933
+ # ---- The tools' calls (IO3) ------------------------------------------------
934
+ #
935
+ # `httpx` is imported INSIDE each call, for two reasons: scholar's lookup
936
+ # imports it lazily so a run with `scholar_lookup` off never loads it, and
937
+ # the suite installs a fake `httpx` module these must reach (the root
938
+ # conftest). The module function is looked up per call, so a test's fake is
939
+ # what runs.
940
+
941
+ @contextlib.contextmanager
942
+ def stream(method: str, url: str, **kwargs):
943
+ """`httpx.stream`, admitted and counted against the tool meter."""
944
+ import httpx
945
+
946
+ TOOLS.admit()
947
+ with httpx.stream(method, url, **kwargs) as response:
948
+ try:
949
+ yield response
950
+ finally:
951
+ if TOOLS.enabled():
952
+ TOOLS.record(exchange_bytes(response))
953
+
954
+
955
+ def get(url: str, **kwargs):
956
+ """`httpx.get`, admitted and counted against the tool meter."""
957
+ import httpx
958
+
959
+ TOOLS.admit()
960
+ response = httpx.get(url, **kwargs)
961
+ if TOOLS.enabled():
962
+ TOOLS.record(exchange_bytes(response))
963
+ return response
964
+
965
+
966
+ def record_web_search(results_requested: int) -> None:
967
+ """IO15's estimate for one search, which has no client to count."""
968
+ if TOOLS.enabled():
969
+ TOOLS.record(max(int(results_requested), 1)
970
+ * WEB_SEARCH_BYTES_PER_RESULT, estimated=True)
971
+
972
+
973
+ # ---- The provider SDKs and MCP (IO4) ---------------------------------------
974
+ #
975
+ # EVENT HOOKS ON AN httpx2 CLIENT, not a wrapped transport, and both halves
976
+ # of the reason were measured (batch 239, httpx2 2.16.0). With HTTPS_PROXY
977
+ # set and no transport passed, the client MOUNTS a proxy transport of its
978
+ # own and a proxied request goes through that, which a wrapper on the
979
+ # default transport never sees. Pass a transport instead, and the client
980
+ # drops the environment's proxy altogether -- a meter that silently breaks
981
+ # the user's proxy. Hooks run for every hop, whatever carries it. Measured
982
+ # the same day against all three SDKs: anthropic, openai and google-genai
983
+ # each call both hooks for an ordinary request, a stream and (google) an
984
+ # embedding.
985
+ #
986
+ # THE BODY IS COUNTED BY REPLACING `response.stream` in the response hook,
987
+ # which runs before anything reads the body, with a stream that counts each
988
+ # raw chunk as it passes.
989
+
990
+ @cache
991
+ def _stream_classes() -> tuple:
992
+ """The two counting streams, built on first use so `httpx2` is only
993
+ imported by a process that has a meter on."""
994
+ import httpx2
995
+
996
+ class CountingStream(httpx2.SyncByteStream):
997
+ def __init__(self, inner, meter: Meter) -> None:
998
+ self._inner = inner
999
+ self._meter = meter
1000
+
1001
+ def __iter__(self):
1002
+ for chunk in self._inner:
1003
+ self._meter.record(len(chunk))
1004
+ yield chunk
1005
+
1006
+ def close(self) -> None:
1007
+ self._inner.close()
1008
+
1009
+ class AsyncCountingStream(httpx2.AsyncByteStream):
1010
+ def __init__(self, inner, meter: Meter) -> None:
1011
+ self._inner = inner
1012
+ self._meter = meter
1013
+
1014
+ async def __aiter__(self):
1015
+ async for chunk in self._inner:
1016
+ self._meter.record(len(chunk))
1017
+ yield chunk
1018
+
1019
+ async def aclose(self) -> None:
1020
+ await self._inner.aclose()
1021
+
1022
+ return CountingStream, AsyncCountingStream
1023
+
1024
+
1025
+ def provider_event_hooks() -> dict:
1026
+ """Hooks for a provider SDK's client: admit, then count (IO4, IO11).
1027
+
1028
+ THE REQUEST IS SIZED EXACTLY BEFORE IT IS SENT, so an upload that would
1029
+ pass the ceiling is refused whole. The provider's upload is the big
1030
+ one -- every step re-sends the thread -- which is why this meter
1031
+ admits on the outgoing size and not only on what is already used.
1032
+ """
1033
+ counting, _ = _stream_classes()
1034
+ meter = PROVIDER
1035
+
1036
+ def on_request(request) -> None:
1037
+ outgoing = request_bytes(request)
1038
+ meter.admit(outgoing)
1039
+ meter.record(outgoing)
1040
+
1041
+ def on_response(response) -> None:
1042
+ meter.record(response_head_bytes(response))
1043
+ response.stream = counting(response.stream, meter)
1044
+
1045
+ return {"request": [on_request], "response": [on_response]}
1046
+
1047
+
1048
+ def mcp_event_hooks() -> dict:
1049
+ """Hooks for an MCP server's async client: count, never refuse.
1050
+
1051
+ MCP's admission is `mcp_client.call_tool`'s, not this hook's. A refusal
1052
+ raised here would surface inside the SDK's background writer task,
1053
+ where it reads as a timeout rather than as a reason; refused at the
1054
+ call, it is the one error shape `call_tool` already returns.
1055
+ """
1056
+ _, counting = _stream_classes()
1057
+ meter = MCP
1058
+
1059
+ async def on_request(request) -> None:
1060
+ meter.record(request_bytes(request))
1061
+
1062
+ async def on_response(response) -> None:
1063
+ meter.record(response_head_bytes(response))
1064
+ response.stream = counting(response.stream, meter)
1065
+
1066
+ return {"request": [on_request], "response": [on_response]}
1067
+
1068
+
1069
+ def mcp_call_bytes(params: Any) -> int:
1070
+ """What a tool call will send, near enough to admit on."""
1071
+ try:
1072
+ return len(json.dumps(params, default=str).encode("utf-8"))
1073
+ except (TypeError, ValueError):
1074
+ return 0
1075
+
1076
+
1077
+ # ---- What a person reads (IO13) --------------------------------------------
1078
+
1079
+ #: IO5's list, said once and shown by `/network` whatever the meters say.
1080
+ NOT_COUNTED = (
1081
+ "Not counted: shell commands (containers, WSL, SSH), stdio MCP servers, "
1082
+ "sign-in and token-refresh traffic, and TLS/TCP overhead. web_search is "
1083
+ "an estimate, about 21 kB per result asked for.")
1084
+
1085
+ #: The sidebar's names, which have 22 columns to share.
1086
+ SHORT_LABELS = {"tools": "tools", "provider": "api", "mcp": "mcp"}
1087
+
1088
+
1089
+ def network_report() -> list:
1090
+ """`/network`'s lines, the same in both shells (ROADMAP_v3 §89's rule:
1091
+ the shells bind handlers, and what they print is decided here)."""
1092
+ rows = states()
1093
+ window = describe_window(window_s())
1094
+ if not any(row.enabled for row in rows):
1095
+ return [
1096
+ "No network limit is set: every meter is off and nothing is "
1097
+ "counted.",
1098
+ "To set one, give network_limit_tools_mb, "
1099
+ "network_limit_provider_mb or network_limit_mcp_mb a value in "
1100
+ f"config.yaml (decimal MB per {window}, network_window_hours).",
1101
+ NOT_COUNTED,
1102
+ ]
1103
+ width = max(len(row.label) for row in rows)
1104
+ lines = [f"Network use over the last {window}:"]
1105
+ for row in rows:
1106
+ if not row.enabled:
1107
+ lines.append(f" {row.label:<{width}} off")
1108
+ continue
1109
+ line = (f" {row.label:<{width}} {format_bytes(row.used)} of "
1110
+ f"{format_bytes(row.ceiling)}")
1111
+ if row.estimated:
1112
+ line += f" ({format_bytes(row.estimated)} of it estimated)"
1113
+ if row.full:
1114
+ line += " -- full"
1115
+ if row.room_at is not None:
1116
+ line += f", room returns at about {_clock(row.room_at)}"
1117
+ lines.append(line)
1118
+ note = next((row.ledger_note for row in rows if row.ledger_note), None)
1119
+ if note:
1120
+ lines.append(f"Ledger: {note}.")
1121
+ else:
1122
+ lines.append(f"Ledger: {_ledger().path()}, shared by every harness "
1123
+ "process and written at most once a minute.")
1124
+ lines.append(NOT_COUNTED)
1125
+ return lines
1126
+
1127
+
1128
+ def sidebar_text(rows: list) -> Optional[str]:
1129
+ """The sidebar line for these states, or None while every meter is off.
1130
+
1131
+ `~` marks a figure that includes an estimate, IO3's rule that the
1132
+ weaker number says so wherever it is shown.
1133
+ """
1134
+ on = [row for row in rows if row.enabled]
1135
+ if not on:
1136
+ return None
1137
+ parts = []
1138
+ for row in on:
1139
+ mark = "~" if row.estimated else ""
1140
+ full = " full" if row.full else ""
1141
+ parts.append(f"{SHORT_LABELS[row.name]} {mark}"
1142
+ f"{format_bytes_short(row.used)}/"
1143
+ f"{format_bytes_short(row.ceiling)}{full}")
1144
+ return ("net · " + " · ".join(parts)
1145
+ + f" · per {describe_window(on[0].window_s)}")