@yahyahammad/venastine 0.3.16 → 0.3.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONFIG_ARCHITECTURE.md +14 -8
- package/README.md +5 -5
- package/cli_commands.py +82 -1
- package/core/client.py +130 -11
- package/core/commands.py +7 -3
- package/core/loop.py +21 -1
- package/core/memory.py +13 -0
- package/core/reasoning/json_retry.py +148 -31
- package/core/reasoning/orchestrator.py +49 -3
- package/core/shell_sessions.py +67 -6
- package/core/usage_view.py +119 -0
- package/docs/THIRD_PARTY_NOTICES.md +1 -1
- package/logging_setup.py +35 -2
- package/main.py +184 -21
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/safety/policy_enforcement.py +79 -9
- package/security/sandbox.py +11 -2
- package/tools/builtin/arxiv.py +8 -2
- package/tools/builtin/fetch_url.py +8 -1
- package/tools/builtin/file_ops.py +74 -8
- package/tools/builtin/project_docs.py +10 -4
- package/tools/builtin/shell.py +2 -1
- package/tools/builtin/shell_sessions.py +6 -3
- package/tools/builtin/web_search.py +5 -1
- package/tools/isolation.py +9 -6
- package/tui/app.py +240 -12
- package/tui/meters.py +124 -40
- package/tui/widgets.py +8 -7
package/main.py
CHANGED
|
@@ -467,11 +467,47 @@ def _deliver_held(thread_id) -> None:
|
|
|
467
467
|
print(f"— {ansi.for_display(text.splitlines()[0])} —")
|
|
468
468
|
|
|
469
469
|
|
|
470
|
+
class _WaitEnd:
|
|
471
|
+
"""How `_await_sessions` ended (ROADMAP_v3 §49, SS2 as amended in batch
|
|
472
|
+
265), which `run_chat` needs to know to carry on.
|
|
473
|
+
|
|
474
|
+
`line` is what the user sent while it waited, which `run_chat` runs
|
|
475
|
+
exactly as if it had been typed at the prompt; `resume` is the same
|
|
476
|
+
wait, to go back to once a COMMAND sent that way has run (a message
|
|
477
|
+
ends in a wait of its own). `prompt_open` says "You: " is already on
|
|
478
|
+
screen with nothing typed after it, so the next read prints none.
|
|
479
|
+
"""
|
|
480
|
+
|
|
481
|
+
__slots__ = ("line", "resume", "prompt_open")
|
|
482
|
+
|
|
483
|
+
def __init__(self, line=None, resume=None, prompt_open=False):
|
|
484
|
+
self.line = line
|
|
485
|
+
self.resume = resume
|
|
486
|
+
self.prompt_open = prompt_open
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
#: How often the CLI's session wait looks for a typed line, in seconds
|
|
490
|
+
#: (batch 265). A message takes at most this long to be picked up, and a
|
|
491
|
+
#: session's result at most this long longer to wake than it used to.
|
|
492
|
+
_WAIT_POLL_S = 0.2
|
|
493
|
+
|
|
494
|
+
|
|
470
495
|
def _await_sessions(thread_id, *, model, provider_name, effort,
|
|
471
|
-
authorization, turn=None, active_skills=()
|
|
496
|
+
authorization, turn=None, active_skills=(),
|
|
497
|
+
count_usage=None) -> _WaitEnd:
|
|
472
498
|
"""Between turns, wait on this conversation's live sessions and run a
|
|
473
499
|
wake turn for each batch (SS2, SS5).
|
|
474
500
|
|
|
501
|
+
THE PROMPT IS OPEN WHILE IT WAITS (batch 265, SS2 as amended, the
|
|
502
|
+
owner's choice). A line typed here ends the wait with the sessions
|
|
503
|
+
still running and goes back to `run_chat`, which runs it as it runs
|
|
504
|
+
one typed at the prompt -- a message gets a turn of its own, whose
|
|
505
|
+
own wait is where what landed meanwhile wakes. A line typed while a
|
|
506
|
+
wake turn runs is type-ahead, read once the turn has finished, as it
|
|
507
|
+
is during the user's own turn. A line and a result arriving together
|
|
508
|
+
go message first: the batch is put back. At the end of stdin -- a
|
|
509
|
+
piped run -- the wait carries on without reading, as it always did.
|
|
510
|
+
|
|
475
511
|
Ctrl+C here stops the SESSIONS and returns to the prompt: the interrupt
|
|
476
512
|
is aimed at what is running, not at the conversation. Their results are
|
|
477
513
|
then held for the next message (SS19), so nothing is printed about them
|
|
@@ -514,10 +550,45 @@ def _await_sessions(thread_id, *, model, provider_name, effort,
|
|
|
514
550
|
from core.session_wake import build_wake
|
|
515
551
|
|
|
516
552
|
if thread_id is None:
|
|
517
|
-
return
|
|
553
|
+
return _WaitEnd()
|
|
518
554
|
as_agent = ({} if turn is None else {
|
|
519
555
|
"context": turn.context, "system_prompt": turn.system_prompt,
|
|
520
556
|
"max_steps": turn.max_steps})
|
|
557
|
+
|
|
558
|
+
def _again():
|
|
559
|
+
# The same wait re-entered, so it counts into the same thread's
|
|
560
|
+
# usage (§97 UF4): a wake after the user's line is still this
|
|
561
|
+
# thread's call.
|
|
562
|
+
return _await_sessions(
|
|
563
|
+
thread_id, model=model, provider_name=provider_name,
|
|
564
|
+
effort=effort, authorization=authorization, turn=turn,
|
|
565
|
+
active_skills=active_skills, count_usage=count_usage)
|
|
566
|
+
|
|
567
|
+
reader = _stdin_reader()
|
|
568
|
+
# Batch 265. `reading` goes False for good at the end of stdin, and
|
|
569
|
+
# `prompted` is whether "You: " is on screen waiting for a line.
|
|
570
|
+
reading = [True]
|
|
571
|
+
prompted = [False]
|
|
572
|
+
|
|
573
|
+
def _typed(timeout):
|
|
574
|
+
"""The line sent, "" for Enter on nothing, or None."""
|
|
575
|
+
if not reading[0]:
|
|
576
|
+
return None
|
|
577
|
+
try:
|
|
578
|
+
line = reader.poll(timeout)
|
|
579
|
+
except EOFError:
|
|
580
|
+
reading[0] = False
|
|
581
|
+
if prompted[0]:
|
|
582
|
+
print()
|
|
583
|
+
prompted[0] = False
|
|
584
|
+
return None
|
|
585
|
+
return None if line is None else line.strip()
|
|
586
|
+
|
|
587
|
+
def _close_prompt():
|
|
588
|
+
if prompted[0]:
|
|
589
|
+
print()
|
|
590
|
+
prompted[0] = False
|
|
591
|
+
|
|
521
592
|
announced = False
|
|
522
593
|
# §64 A21. Set by the handler below instead of an unwind; read after the
|
|
523
594
|
# wake turn has finished. A list rather than a `nonlocal` because the
|
|
@@ -541,21 +612,48 @@ def _await_sessions(thread_id, *, model, provider_name, effort,
|
|
|
541
612
|
|
|
542
613
|
try:
|
|
543
614
|
while True:
|
|
544
|
-
|
|
615
|
+
# `waited_on`, not `live_for` (batch 265): an interactive shell
|
|
616
|
+
# is not waited on (SS25), and counting one spun this loop for
|
|
617
|
+
# as long as it stayed open, `wait_for_wake` answering None at
|
|
618
|
+
# once each time.
|
|
619
|
+
live = sessions.waited_on(thread_id)
|
|
545
620
|
if not live and not sessions.pending(thread_id):
|
|
546
|
-
return
|
|
621
|
+
return _WaitEnd(prompt_open=prompted[0])
|
|
547
622
|
if not announced:
|
|
548
|
-
print(f"[waiting on {live} background session(s) — "
|
|
549
|
-
f"Ctrl+C stops them]")
|
|
623
|
+
print(f"[waiting on {live} background session(s) — type "
|
|
624
|
+
f"a message to send it now; Ctrl+C stops them]")
|
|
550
625
|
announced = True
|
|
551
|
-
|
|
626
|
+
if reading[0] and not prompted[0]:
|
|
627
|
+
print("You: ", end="", flush=True)
|
|
628
|
+
prompted[0] = True
|
|
629
|
+
line = _typed(_WAIT_POLL_S)
|
|
630
|
+
if line:
|
|
631
|
+
prompted[0] = False
|
|
632
|
+
return _WaitEnd(line=line, resume=_again)
|
|
633
|
+
if line == "":
|
|
634
|
+
# Enter on nothing, which the prompt answers with itself.
|
|
635
|
+
prompted[0] = False
|
|
636
|
+
continue
|
|
637
|
+
batch = sessions.wait_for_wake(
|
|
638
|
+
thread_id, timeout=_WAIT_POLL_S if reading[0] else 1.0)
|
|
552
639
|
if batch is None:
|
|
553
640
|
if sessions.suspended(thread_id):
|
|
641
|
+
_close_prompt()
|
|
554
642
|
print("[no further automatic replies until you say "
|
|
555
643
|
"something — results will arrive with your next "
|
|
556
644
|
"message]")
|
|
557
|
-
return
|
|
645
|
+
return _WaitEnd()
|
|
558
646
|
continue
|
|
647
|
+
line = _typed(0)
|
|
648
|
+
if line:
|
|
649
|
+
# The owner's rule: a message and a result arriving
|
|
650
|
+
# together go message first.
|
|
651
|
+
sessions.put_back(thread_id, batch)
|
|
652
|
+
prompted[0] = False
|
|
653
|
+
return _WaitEnd(line=line, resume=_again)
|
|
654
|
+
if line == "":
|
|
655
|
+
prompted[0] = False
|
|
656
|
+
_close_prompt()
|
|
559
657
|
text, record = build_wake(batch)
|
|
560
658
|
previous_sigint = signal.signal(signal.SIGINT, _SecondPress(
|
|
561
659
|
"_stop_sessions_after_this_turn",
|
|
@@ -576,13 +674,17 @@ def _await_sessions(thread_id, *, model, provider_name, effort,
|
|
|
576
674
|
except Exception as e:
|
|
577
675
|
logger.exception("Wake turn failed")
|
|
578
676
|
print(f"\n[Error: {ansi.for_display(e)}]")
|
|
579
|
-
return
|
|
677
|
+
return _WaitEnd()
|
|
580
678
|
finally:
|
|
581
679
|
# Before anything else can be printed or read, and on the
|
|
582
680
|
# error path too: outside a wake turn the interrupt belongs
|
|
583
681
|
# to the sessions and takes the handler below.
|
|
584
682
|
signal.signal(signal.SIGINT, previous_sigint)
|
|
585
683
|
live.finish(response)
|
|
684
|
+
# ROADMAP_v3 §97 (UF4). A wake turn's calls are this
|
|
685
|
+
# thread's too, as they are on the TUI's memory.
|
|
686
|
+
if count_usage is not None:
|
|
687
|
+
count_usage(response)
|
|
586
688
|
if interrupted:
|
|
587
689
|
# The turn finished and wrote its rows; now the key does
|
|
588
690
|
# what it said. Returning to the prompt rather than waiting
|
|
@@ -592,10 +694,11 @@ def _await_sessions(thread_id, *, model, provider_name, effort,
|
|
|
592
694
|
killed = sessions.kill_owned(thread_id,
|
|
593
695
|
shell_sessions.KILL_USER)
|
|
594
696
|
print(f"\n[stopped {len(killed)} background session(s)]")
|
|
595
|
-
return
|
|
697
|
+
return _WaitEnd()
|
|
596
698
|
except KeyboardInterrupt:
|
|
597
699
|
killed = sessions.kill_owned(thread_id, shell_sessions.KILL_USER)
|
|
598
700
|
print(f"\n[stopped {len(killed)} background session(s)]")
|
|
701
|
+
return _WaitEnd()
|
|
599
702
|
|
|
600
703
|
|
|
601
704
|
def _close_sessions() -> None:
|
|
@@ -686,18 +789,30 @@ def run_chat(
|
|
|
686
789
|
print("Ctrl+C twice stops the current turn; Ctrl+C or Ctrl+D at the "
|
|
687
790
|
"prompt exits. /help lists commands.\n")
|
|
688
791
|
|
|
792
|
+
# Batch 265 (SS2 as amended). How the last session wait ended: a line
|
|
793
|
+
# the user sent while it waited runs below exactly as one typed at the
|
|
794
|
+
# prompt would, and a COMMAND sent that way goes back to the wait it
|
|
795
|
+
# interrupted (a message ends in a wait of its own).
|
|
796
|
+
waited = _WaitEnd()
|
|
689
797
|
while True:
|
|
798
|
+
from_wait = None
|
|
690
799
|
if first_message is not None:
|
|
691
800
|
user_input = first_message
|
|
692
801
|
first_message = None
|
|
802
|
+
elif waited.line is not None:
|
|
803
|
+
from_wait, user_input = waited, waited.line
|
|
693
804
|
else:
|
|
805
|
+
# "You: " is already on screen when the wait ended with it
|
|
806
|
+
# open, and printing a second would put two on one line.
|
|
807
|
+
prompt = "" if waited.prompt_open else "You: "
|
|
808
|
+
waited = _WaitEnd()
|
|
694
809
|
try:
|
|
695
810
|
# §29 (N1). Audit #100: this was input(), a SECOND reader
|
|
696
811
|
# on the stdin the approval pump already owns, so from the
|
|
697
812
|
# first gated call onwards the pump took this line and
|
|
698
813
|
# input() blocked forever. EOFError still arrives here --
|
|
699
814
|
# readline() raises it on Ctrl+D exactly as input() did.
|
|
700
|
-
user_input = _stdin_reader().readline(
|
|
815
|
+
user_input = _stdin_reader().readline(prompt).strip()
|
|
701
816
|
except (KeyboardInterrupt, EOFError):
|
|
702
817
|
# ROADMAP_v3 §49 (SS6). Quitting with sessions still
|
|
703
818
|
# running confirms first -- they are killed on the way out
|
|
@@ -708,6 +823,7 @@ def run_chat(
|
|
|
708
823
|
continue
|
|
709
824
|
print("\nExiting.")
|
|
710
825
|
break
|
|
826
|
+
waited = _WaitEnd()
|
|
711
827
|
|
|
712
828
|
if not user_input:
|
|
713
829
|
continue
|
|
@@ -715,15 +831,22 @@ def run_chat(
|
|
|
715
831
|
agent = session.active_agent
|
|
716
832
|
one_shot = False
|
|
717
833
|
if user_input.startswith("/"):
|
|
718
|
-
|
|
834
|
+
# `waiting`: the commands the TUI refuses while a turn's
|
|
835
|
+
# sessions run are refused here too, when this one was sent
|
|
836
|
+
# during the wait (SS2 as amended, the owner's choice).
|
|
837
|
+
run_command(session, user_input, waiting=from_wait is not None)
|
|
719
838
|
if session.quit_requested:
|
|
720
839
|
# /quit takes Ctrl+D's route, confirmation and all.
|
|
721
840
|
session.quit_requested = False
|
|
722
841
|
if _quit_confirmed():
|
|
723
842
|
print("Exiting.")
|
|
724
843
|
break
|
|
844
|
+
if from_wait is not None:
|
|
845
|
+
waited = from_wait.resume()
|
|
725
846
|
continue
|
|
726
847
|
if session.one_shot is None:
|
|
848
|
+
if from_wait is not None:
|
|
849
|
+
waited = from_wait.resume()
|
|
727
850
|
continue
|
|
728
851
|
# /grill-me: one turn in this thread under that agent's prompt,
|
|
729
852
|
# run like a typed message so it prints live and stops the same
|
|
@@ -731,13 +854,16 @@ def run_chat(
|
|
|
731
854
|
agent, session.one_shot = session.one_shot, None
|
|
732
855
|
user_input, one_shot = GRILL_ME_TASK, True
|
|
733
856
|
|
|
734
|
-
_chat_turn(session, user_input, agent, one_shot=one_shot)
|
|
857
|
+
waited = _chat_turn(session, user_input, agent, one_shot=one_shot)
|
|
735
858
|
|
|
736
859
|
|
|
737
860
|
def _chat_turn(session: ChatSession, user_input: str, agent,
|
|
738
|
-
one_shot: bool = False) ->
|
|
861
|
+
one_shot: bool = False) -> _WaitEnd:
|
|
739
862
|
"""One turn of `run_chat`'s conversation, and the wake turns after it.
|
|
740
863
|
|
|
864
|
+
Returns how the wait after it ended (`_WaitEnd`), since batch 265 a
|
|
865
|
+
line the user sent during it among other things.
|
|
866
|
+
|
|
741
867
|
`agent` is the session's active agent, or /grill-me's for its one turn
|
|
742
868
|
(`one_shot`). What the agent changes about the turn is
|
|
743
869
|
`core.loop.agent_turn`'s to say, the function the TUI's turn calls, so
|
|
@@ -820,11 +946,13 @@ def _chat_turn(session: ChatSession, user_input: str, agent,
|
|
|
820
946
|
except Exception as e:
|
|
821
947
|
logger.exception("Chat turn failed")
|
|
822
948
|
print(f"\n[Error: {ansi.for_display(e)}]")
|
|
823
|
-
return
|
|
949
|
+
return _WaitEnd()
|
|
824
950
|
finally:
|
|
825
951
|
signal.signal(signal.SIGINT, previous_sigint)
|
|
826
952
|
|
|
827
953
|
session.thread_id = response.thread_id
|
|
954
|
+
# ROADMAP_v3 §97 (UF4): this turn's calls, for /usage.
|
|
955
|
+
session.count_usage(response)
|
|
828
956
|
|
|
829
957
|
# ROADMAP_v2 §21's "no silent compaction, ever": a notice printed
|
|
830
958
|
# live when it happened, and one that never arrived live is printed
|
|
@@ -871,12 +999,14 @@ def _chat_turn(session: ChatSession, user_input: str, agent,
|
|
|
871
999
|
# SS2/SS5. The turn is over, but this conversation may not be: a
|
|
872
1000
|
# session it started reports later, and that report starts a turn
|
|
873
1001
|
# of its own -- under the same agent, prompt and skills, because it
|
|
874
|
-
# IS this turn continuing.
|
|
875
|
-
#
|
|
876
|
-
_await_sessions(
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
1002
|
+
# IS this turn continuing. Since batch 265 the user can type while it
|
|
1003
|
+
# waits (SS2 as amended): a line sent then comes back from here.
|
|
1004
|
+
return _await_sessions(
|
|
1005
|
+
session.thread_id, model=turn.model,
|
|
1006
|
+
provider_name=turn.provider_name, effort=session.effort,
|
|
1007
|
+
authorization=session.authorization, turn=turn,
|
|
1008
|
+
active_skills=session.active_skills,
|
|
1009
|
+
count_usage=session.count_usage)
|
|
880
1010
|
|
|
881
1011
|
|
|
882
1012
|
def build_research_authorization(grant_spec, attended: bool = False):
|
|
@@ -1301,6 +1431,39 @@ class _StdinReader:
|
|
|
1301
1431
|
raise EOFError
|
|
1302
1432
|
return line.rstrip("\r\n")
|
|
1303
1433
|
|
|
1434
|
+
def poll(self, timeout: float):
|
|
1435
|
+
"""A line typed while the chat waits on background sessions, or
|
|
1436
|
+
None if none arrived within *timeout* (ROADMAP_v3 §49, SS2 as
|
|
1437
|
+
amended in batch 265). Raises EOFError once stdin has ended.
|
|
1438
|
+
|
|
1439
|
+
readline()'s discipline, with ask()'s deadline: it KEEPS
|
|
1440
|
+
type-ahead, because a line typed during the turn that started the
|
|
1441
|
+
sessions is a message meant for the prompt this wait now shows,
|
|
1442
|
+
and it drains nothing for the same reason. It takes no prompt, as
|
|
1443
|
+
it is called again every fraction of a second; the wait prints the
|
|
1444
|
+
prompt once.
|
|
1445
|
+
|
|
1446
|
+
IT STARTS THE PUMP, which readline() takes care not to do: a read
|
|
1447
|
+
with a deadline cannot be made on a thread that might never come
|
|
1448
|
+
back, which is the reason the pump exists at all. Chat mode is the
|
|
1449
|
+
only caller and never hands the terminal to Textual afterwards, so
|
|
1450
|
+
batch 121's parked-reader hazard does not reach it.
|
|
1451
|
+
"""
|
|
1452
|
+
import queue as _queue
|
|
1453
|
+
|
|
1454
|
+
if self._released:
|
|
1455
|
+
raise EOFError
|
|
1456
|
+
self._start()
|
|
1457
|
+
if self._ended():
|
|
1458
|
+
raise EOFError
|
|
1459
|
+
try:
|
|
1460
|
+
line = self._lines.get(timeout=timeout)
|
|
1461
|
+
except _queue.Empty:
|
|
1462
|
+
return None
|
|
1463
|
+
if line is self._EOF:
|
|
1464
|
+
raise EOFError
|
|
1465
|
+
return line.rstrip("\r\n")
|
|
1466
|
+
|
|
1304
1467
|
def _read_inline(self) -> str:
|
|
1305
1468
|
"""One line, on this thread, leaving no reader behind.
|
|
1306
1469
|
|
package/package.json
CHANGED
package/pyproject.toml
CHANGED
|
@@ -18,7 +18,7 @@ build-backend = "setuptools.build_meta"
|
|
|
18
18
|
|
|
19
19
|
[project]
|
|
20
20
|
name = "venastine-research-harness"
|
|
21
|
-
version = "0.3.
|
|
21
|
+
version = "0.3.18"
|
|
22
22
|
description = "A from-scratch agentic harness: tool-using chat plus a ten-pass self-critiquing research pipeline."
|
|
23
23
|
readme = "README.md"
|
|
24
24
|
# The same words as package.json's `keywords`, in the same order: what a
|
|
@@ -234,6 +234,17 @@ _PEM_SEP = _PEM_BREAK + r"+[ \t]*"
|
|
|
234
234
|
# key carries before its base64.
|
|
235
235
|
_PEM_UNIT = r"(?:[A-Za-z0-9+/=]{16,}|(?:Proc-Type|DEK-Info):[^\r\n\\]*)"
|
|
236
236
|
|
|
237
|
+
# A token character that CLOSES AN ESCAPE, so a pattern that refuses to start
|
|
238
|
+
# after a token character still starts here: the second hex digit of `%3D`
|
|
239
|
+
# in a URL-encoded value, and the letter of `\n`, `\r` or `\t` in a
|
|
240
|
+
# JSON-escaped one. Batch 260 wrote it for GitHub's stateless form. Batch 268
|
|
241
|
+
# measured the same miss in every other pattern anchored that way: `hf_`,
|
|
242
|
+
# `ms-` and the JWT each passed whole after `%3D`, `\n` and `\t`, and the
|
|
243
|
+
# `Authorization` header after `\n`. They share this one copy. Each such
|
|
244
|
+
# start sits two characters after a `%` or a `\`, which none of their runs
|
|
245
|
+
# contains, so a pattern that was linear stays linear.
|
|
246
|
+
_AFTER_ESCAPE = r"(?<=%[0-9A-Fa-f]{2})|(?<=\\[nrt])"
|
|
247
|
+
|
|
237
248
|
_SECRET_PATTERNS = [
|
|
238
249
|
# OpenAI-style keys
|
|
239
250
|
re.compile(r"sk-[a-zA-Z0-9]{20,}"),
|
|
@@ -273,6 +284,35 @@ _SECRET_PATTERNS = [
|
|
|
273
284
|
# --- `sk-` always have; one vendor list, one vocabulary.
|
|
274
285
|
# GitHub fine-grained PATs, the format `ghp_` is being retired for
|
|
275
286
|
re.compile(r"github_pat_[A-Za-z0-9_]{22,}"),
|
|
287
|
+
# GitHub's STATELESS tokens (batch 260): `ghs_APPID_JWT`, a JWT after
|
|
288
|
+
# the prefix, about 520 characters with two dots where the classic
|
|
289
|
+
# shape below is 40 with none. GitHub's docs and changelog say every
|
|
290
|
+
# installation token minted since its rollout finished on 2026-10-02
|
|
291
|
+
# has this form, Actions' GITHUB_TOKEN included, and the classic
|
|
292
|
+
# pattern stops at the first underscore. Measured before this: a fake
|
|
293
|
+
# passed `redact_output_text` whole and refused no input.
|
|
294
|
+
# Three base64url runs of at least 20 characters, joined by the two
|
|
295
|
+
# dots GitHub says tell the forms apart. Nothing about the JWT's
|
|
296
|
+
# contents, which GitHub says clients must not rely on, and no dot in
|
|
297
|
+
# the runs, so a sentence's final period stays (GitHub's own
|
|
298
|
+
# suggestion, `ghs_` and 36 of `[A-Za-z0-9._-]`, redacted 3 of 8
|
|
299
|
+
# lines of code and prose that merely name something `ghs_`). All
|
|
300
|
+
# four prefixes, the owner's choice: GitHub has changed `ghs_` and
|
|
301
|
+
# says user-to-server tokens are next.
|
|
302
|
+
# The lookbehind keeps it linear. Without it, every `ghs_` inside one
|
|
303
|
+
# long run is a start that scans to the run's end: measured quadratic,
|
|
304
|
+
# 99 s on a hostile megabyte against 0.03 s with it. A token character
|
|
305
|
+
# is allowed before the prefix only where it closes an escape, `%3D`
|
|
306
|
+
# or `\n` (`_AFTER_ESCAPE`), since a URL-encoded or JSON-escaped token
|
|
307
|
+
# is still a token, and the classic pattern catches those. Each of those
|
|
308
|
+
# starts sits two characters after a `%` or a `\`, which no run
|
|
309
|
+
# contains, so the scan stays linear.
|
|
310
|
+
# This pattern runs BEFORE the classic one, because a 36-character
|
|
311
|
+
# alphanumeric run inside a segment would otherwise take a piece of
|
|
312
|
+
# the token and leave the rest standing.
|
|
313
|
+
re.compile(r"(?:(?<![A-Za-z0-9_-])|" + _AFTER_ESCAPE + r")"
|
|
314
|
+
r"gh[ousr]_[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}"
|
|
315
|
+
r"\.[A-Za-z0-9_-]{20,}"),
|
|
276
316
|
# GitHub OAuth, user-to-server, server-to-server and refresh tokens
|
|
277
317
|
re.compile(r"gh[ousr]_[A-Za-z0-9]{36}"),
|
|
278
318
|
# OpenAI project, service-account and admin keys: `sk-proj-...`
|
|
@@ -287,14 +327,21 @@ _SECRET_PATTERNS = [
|
|
|
287
327
|
# --- the rest of the list.
|
|
288
328
|
# Hugging Face user access tokens and OAuth access tokens. An
|
|
289
329
|
# identifier like `hf_hub_download` breaks the run of word characters
|
|
290
|
-
# at its second underscore and is not matched.
|
|
291
|
-
|
|
330
|
+
# at its second underscore and is not matched. An escape's closing
|
|
331
|
+
# character may come before the prefix (`_AFTER_ESCAPE`, batch 268):
|
|
332
|
+
# before that, `%3Dhf_...` and `\nhf_...` were neither redacted nor
|
|
333
|
+
# refused.
|
|
334
|
+
re.compile(r"(?:(?<![A-Za-z0-9])|" + _AFTER_ESCAPE + r")"
|
|
335
|
+
r"hf_(?:oauth_)?[A-Za-z0-9]{30,}"),
|
|
292
336
|
# OpenRouter keys, which `sk-` above misses at the second hyphen
|
|
293
337
|
re.compile(r"sk-or-v1-[A-Za-z0-9]{32,}"),
|
|
294
338
|
# ModelScope access tokens in their prefixed form: `ms-` and a UUID. A
|
|
295
339
|
# bare UUID is NOT matched -- thread ids are UUIDs -- and neither is one
|
|
296
|
-
# that merely ENDS a word like `items-<uuid>`.
|
|
297
|
-
|
|
340
|
+
# that merely ENDS a word like `items-<uuid>`. An escape's closing
|
|
341
|
+
# character is not a word (`_AFTER_ESCAPE`, batch 268), so `%3Dms-...`
|
|
342
|
+
# and `\nms-...` are matched, as `hf_` is.
|
|
343
|
+
re.compile(r"(?:(?<![A-Za-z0-9])|" + _AFTER_ESCAPE + r")"
|
|
344
|
+
r"ms-[0-9a-f]{8}-[0-9a-f]{4}"
|
|
298
345
|
r"-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}"),
|
|
299
346
|
]
|
|
300
347
|
|
|
@@ -397,19 +444,26 @@ _ASSIGNMENT_RE = re.compile(
|
|
|
397
444
|
# `Digest username=...`, whose secret is not one value, is left alone. A
|
|
398
445
|
# value with NO scheme word must also be 16 characters and carry a digit,
|
|
399
446
|
# because API documentation writes "Authorization: required" and a bare
|
|
400
|
-
# 8-character floor redacted the word.
|
|
447
|
+
# 8-character floor redacted the word. The header may follow an escape's
|
|
448
|
+
# closing character (`_AFTER_ESCAPE`, batch 268), case-sensitively, since
|
|
449
|
+
# `\b` fails between the `n` of a JSON-escaped `\n` and the header name: a
|
|
450
|
+
# captured request's `\nAuthorization: Bearer ...` passed whole before it.
|
|
401
451
|
# Groups: 1 header name, 2 separator, 3 scheme (or None).
|
|
402
452
|
_AUTH_HEADER_RE = re.compile(
|
|
403
|
-
r"(?i)
|
|
453
|
+
r"(?i)(?:\b|(?-i:" + _AFTER_ESCAPE + r"))"
|
|
454
|
+
r"((?:proxy-)?authorization)([\"']?\s*[:=]\s*[\"']?)"
|
|
404
455
|
r"(?:((?:bearer|basic|token)\s+)[A-Za-z0-9._~+/=\-]{8,}"
|
|
405
456
|
r"|(?=[A-Za-z._~+/=\-]*[0-9])[A-Za-z0-9._~+/=\-]{16,})")
|
|
406
457
|
|
|
407
458
|
# A JSON Web Token, whole: three base64url segments, the first two of
|
|
408
459
|
# which open with `eyJ` because both are JSON objects (`{"`). A bearer
|
|
409
460
|
# credential wherever it appears, and the shape `_AUTH_HEADER_RE` cannot
|
|
410
|
-
# reach when the token is pasted without its header.
|
|
461
|
+
# reach when the token is pasted without its header. It may follow an
|
|
462
|
+
# escape's closing character (`_AFTER_ESCAPE`, batch 268), where `\b` fails:
|
|
463
|
+
# `%3DeyJ...` and `\neyJ...` passed whole before that.
|
|
411
464
|
_JWT_RE = re.compile(
|
|
412
|
-
r"
|
|
465
|
+
r"(?:\b|" + _AFTER_ESCAPE + r")"
|
|
466
|
+
r"eyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}")
|
|
413
467
|
|
|
414
468
|
# The END of a PEM key block whose BEGIN is not in this string: the footer
|
|
415
469
|
# and the run of base64 lines above it. The vendor pattern takes a block
|
|
@@ -1036,7 +1090,23 @@ def check_output_policy(tool_name: str, result: dict) -> dict:
|
|
|
1036
1090
|
# claims and reports constantly, and refusing every mention would make the
|
|
1037
1091
|
# blocklist unusable while blocking nothing an attacker couldn't rephrase.
|
|
1038
1092
|
# fetch_url.py keeps its own bare-domain check for the value it is given.
|
|
1039
|
-
|
|
1093
|
+
#
|
|
1094
|
+
# The scan starts once per RUN of scheme characters (TECHNICAL_DEBT 117,
|
|
1095
|
+
# batch 268). It used to be `[a-zA-Z][a-zA-Z0-9+.\-]*://...`, tried at every
|
|
1096
|
+
# letter, and on one long run of letters and digits each try scanned to the
|
|
1097
|
+
# run's end before failing to find `://`. That was quadratic: 4.8 s at
|
|
1098
|
+
# 100 KB and 441 s at a megabyte, before the handler ran. Now the lookbehind
|
|
1099
|
+
# refuses every start inside a run, the run's leading digits and `+.-` are
|
|
1100
|
+
# skipped, and the group captures from the run's first letter, which is
|
|
1101
|
+
# where the old pattern's leftmost match began. So `findall`, which returns
|
|
1102
|
+
# that one group, gives exactly the list it gave before (`0https://h` still
|
|
1103
|
+
# gives `https://h`), and the blocklist reads the same strings. Measured:
|
|
1104
|
+
# 0.05 s or less on each of ten hostile megabytes, and the old pattern's
|
|
1105
|
+
# list on 303,000 random strings. The shape the register recorded, a scheme
|
|
1106
|
+
# bounded to 32 characters, was linear too, but it found nothing in `a` and
|
|
1107
|
+
# 40 digits before `://evil.com`, where today's scan finds the host.
|
|
1108
|
+
_URL_RE = re.compile(
|
|
1109
|
+
r"(?<![a-zA-Z0-9+.\-])[0-9+.\-]*([a-zA-Z][a-zA-Z0-9+.\-]*://[^\s\"'<>]+)")
|
|
1040
1110
|
|
|
1041
1111
|
|
|
1042
1112
|
def _input_leaf(text: str) -> str:
|
package/security/sandbox.py
CHANGED
|
@@ -2599,6 +2599,15 @@ def _interactive_command(prompt_token: str) -> str:
|
|
|
2599
2599
|
prompt=prompt_token)
|
|
2600
2600
|
|
|
2601
2601
|
|
|
2602
|
+
#: Where the container mounts the workspace, and its working directory.
|
|
2603
|
+
#: Named once (ROADMAP_v3 §96, CW1) because the file tools read it too:
|
|
2604
|
+
#: `file_ops.resolve_path` maps a path spelled this way onto the workspace
|
|
2605
|
+
#: it names, and a second literal there would let the mount move without
|
|
2606
|
+
#: the map. Always a POSIX path, whatever the host is -- it is a path
|
|
2607
|
+
#: inside a Linux container.
|
|
2608
|
+
CONTAINER_WORKSPACE = "/workspace"
|
|
2609
|
+
|
|
2610
|
+
|
|
2602
2611
|
def _docker_argv(
|
|
2603
2612
|
command: str,
|
|
2604
2613
|
workspace_dir: str,
|
|
@@ -2674,8 +2683,8 @@ def _docker_argv(
|
|
|
2674
2683
|
# may run at once, so cleaning up "our" containers by the label
|
|
2675
2684
|
# above would kill the other's sessions; this one scopes it.
|
|
2676
2685
|
"--label", f"venastine.process={PROCESS_TOKEN}",
|
|
2677
|
-
"-v", f"{workspace_real}
|
|
2678
|
-
"-w",
|
|
2686
|
+
"-v", f"{workspace_real}:{CONTAINER_WORKSPACE}",
|
|
2687
|
+
"-w", CONTAINER_WORKSPACE,
|
|
2679
2688
|
"--memory", f"{config.SANDBOX_MEMORY_MB}m",
|
|
2680
2689
|
"--cpus", "1",
|
|
2681
2690
|
"--pids-limit", str(config.SANDBOX_MAX_PIDS),
|
package/tools/builtin/arxiv.py
CHANGED
|
@@ -577,7 +577,9 @@ def run(params: dict) -> dict:
|
|
|
577
577
|
# Returned rather than raised, like every other dead end in
|
|
578
578
|
# this function: a search tool that cannot answer is a
|
|
579
579
|
# result the model can work around.
|
|
580
|
-
|
|
580
|
+
# INFO (batch 266): the result below says this, and both
|
|
581
|
+
# shells print it; the file keeps the query and the page.
|
|
582
|
+
logger.info(
|
|
581
583
|
"arxiv_search dropped a feed over %d bytes for %r "
|
|
582
584
|
"(max_results=%d, start=%d)",
|
|
583
585
|
MAX_FEED_BYTES, parsed.keywords, parsed.max_results,
|
|
@@ -636,7 +638,11 @@ def run(params: dict) -> dict:
|
|
|
636
638
|
}
|
|
637
639
|
except (httpx.HTTPError, ET.ParseError) as e:
|
|
638
640
|
last_exc = e
|
|
639
|
-
|
|
641
|
+
# INFO (batch 266), as web_search's retries: the last
|
|
642
|
+
# attempt's error is the result, and a recovered attempt is
|
|
643
|
+
# nothing to show. NOT the hostile-feed refusal above, whose
|
|
644
|
+
# line keeps what the result deliberately leaves out.
|
|
645
|
+
logger.info(
|
|
640
646
|
"arxiv_search attempt %s/%s failed for %r: %s",
|
|
641
647
|
attempt + 1, MAX_RETRIES + 1, parsed.keywords, e)
|
|
642
648
|
else:
|
|
@@ -616,7 +616,14 @@ def run(params: dict) -> dict:
|
|
|
616
616
|
# dropped -- fifteen consecutive "fetch_url failed" lines with
|
|
617
617
|
# no URL and no reason, which is what made a run of ordinary
|
|
618
618
|
# 404s indistinguishable from a broken tool.
|
|
619
|
-
|
|
619
|
+
#
|
|
620
|
+
# INFO, not WARNING (batch 266, the owner's choice). The error
|
|
621
|
+
# below IS the result, and both shells print a failed result as
|
|
622
|
+
# its own line, so a WARNING drew the same failure twice: the
|
|
623
|
+
# owner's screenshot had both, line for line. The file keeps
|
|
624
|
+
# this one with the hop's URL, which the result does not name
|
|
625
|
+
# after a redirect.
|
|
626
|
+
logger.info("fetch_url failed for %s: %s", url, e)
|
|
620
627
|
return {"error": f"Could not fetch URL: {e}"}
|
|
621
628
|
|
|
622
629
|
held = _held(url, kind, raw, more, response, want)
|