@yahyahammad/venastine 0.3.16 → 0.3.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CONFIG_ARCHITECTURE.md +14 -8
- package/README.md +5 -5
- package/cli_commands.py +82 -1
- package/core/client.py +130 -11
- package/core/commands.py +7 -3
- package/core/loop.py +21 -1
- package/core/memory.py +13 -0
- package/core/reasoning/json_retry.py +148 -31
- package/core/reasoning/orchestrator.py +49 -3
- package/core/shell_sessions.py +67 -6
- package/core/usage_view.py +119 -0
- package/docs/THIRD_PARTY_NOTICES.md +1 -1
- package/logging_setup.py +35 -2
- package/main.py +184 -21
- package/package.json +1 -1
- package/pyproject.toml +1 -1
- package/safety/policy_enforcement.py +29 -0
- package/security/sandbox.py +11 -2
- package/tools/builtin/arxiv.py +8 -2
- package/tools/builtin/fetch_url.py +8 -1
- package/tools/builtin/file_ops.py +74 -8
- package/tools/builtin/project_docs.py +10 -4
- package/tools/builtin/shell.py +2 -1
- package/tools/builtin/shell_sessions.py +6 -3
- package/tools/builtin/web_search.py +5 -1
- package/tools/isolation.py +9 -6
- package/tui/app.py +240 -12
- package/tui/meters.py +124 -40
- package/tui/widgets.py +8 -7
package/main.py
CHANGED
|
@@ -467,11 +467,47 @@ def _deliver_held(thread_id) -> None:
|
|
|
467
467
|
print(f"— {ansi.for_display(text.splitlines()[0])} —")
|
|
468
468
|
|
|
469
469
|
|
|
470
|
+
class _WaitEnd:
|
|
471
|
+
"""How `_await_sessions` ended (ROADMAP_v3 §49, SS2 as amended in batch
|
|
472
|
+
265), which `run_chat` needs to know to carry on.
|
|
473
|
+
|
|
474
|
+
`line` is what the user sent while it waited, which `run_chat` runs
|
|
475
|
+
exactly as if it had been typed at the prompt; `resume` is the same
|
|
476
|
+
wait, to go back to once a COMMAND sent that way has run (a message
|
|
477
|
+
ends in a wait of its own). `prompt_open` says "You: " is already on
|
|
478
|
+
screen with nothing typed after it, so the next read prints none.
|
|
479
|
+
"""
|
|
480
|
+
|
|
481
|
+
__slots__ = ("line", "resume", "prompt_open")
|
|
482
|
+
|
|
483
|
+
def __init__(self, line=None, resume=None, prompt_open=False):
|
|
484
|
+
self.line = line
|
|
485
|
+
self.resume = resume
|
|
486
|
+
self.prompt_open = prompt_open
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
#: How often the CLI's session wait looks for a typed line, in seconds
|
|
490
|
+
#: (batch 265). A message takes at most this long to be picked up, and a
|
|
491
|
+
#: session's result at most this long longer to wake than it used to.
|
|
492
|
+
_WAIT_POLL_S = 0.2
|
|
493
|
+
|
|
494
|
+
|
|
470
495
|
def _await_sessions(thread_id, *, model, provider_name, effort,
|
|
471
|
-
authorization, turn=None, active_skills=()
|
|
496
|
+
authorization, turn=None, active_skills=(),
|
|
497
|
+
count_usage=None) -> _WaitEnd:
|
|
472
498
|
"""Between turns, wait on this conversation's live sessions and run a
|
|
473
499
|
wake turn for each batch (SS2, SS5).
|
|
474
500
|
|
|
501
|
+
THE PROMPT IS OPEN WHILE IT WAITS (batch 265, SS2 as amended, the
|
|
502
|
+
owner's choice). A line typed here ends the wait with the sessions
|
|
503
|
+
still running and goes back to `run_chat`, which runs it as it runs
|
|
504
|
+
one typed at the prompt -- a message gets a turn of its own, whose
|
|
505
|
+
own wait is where what landed meanwhile wakes. A line typed while a
|
|
506
|
+
wake turn runs is type-ahead, read once the turn has finished, as it
|
|
507
|
+
is during the user's own turn. A line and a result arriving together
|
|
508
|
+
go message first: the batch is put back. At the end of stdin -- a
|
|
509
|
+
piped run -- the wait carries on without reading, as it always did.
|
|
510
|
+
|
|
475
511
|
Ctrl+C here stops the SESSIONS and returns to the prompt: the interrupt
|
|
476
512
|
is aimed at what is running, not at the conversation. Their results are
|
|
477
513
|
then held for the next message (SS19), so nothing is printed about them
|
|
@@ -514,10 +550,45 @@ def _await_sessions(thread_id, *, model, provider_name, effort,
|
|
|
514
550
|
from core.session_wake import build_wake
|
|
515
551
|
|
|
516
552
|
if thread_id is None:
|
|
517
|
-
return
|
|
553
|
+
return _WaitEnd()
|
|
518
554
|
as_agent = ({} if turn is None else {
|
|
519
555
|
"context": turn.context, "system_prompt": turn.system_prompt,
|
|
520
556
|
"max_steps": turn.max_steps})
|
|
557
|
+
|
|
558
|
+
def _again():
|
|
559
|
+
# The same wait re-entered, so it counts into the same thread's
|
|
560
|
+
# usage (§97 UF4): a wake after the user's line is still this
|
|
561
|
+
# thread's call.
|
|
562
|
+
return _await_sessions(
|
|
563
|
+
thread_id, model=model, provider_name=provider_name,
|
|
564
|
+
effort=effort, authorization=authorization, turn=turn,
|
|
565
|
+
active_skills=active_skills, count_usage=count_usage)
|
|
566
|
+
|
|
567
|
+
reader = _stdin_reader()
|
|
568
|
+
# Batch 265. `reading` goes False for good at the end of stdin, and
|
|
569
|
+
# `prompted` is whether "You: " is on screen waiting for a line.
|
|
570
|
+
reading = [True]
|
|
571
|
+
prompted = [False]
|
|
572
|
+
|
|
573
|
+
def _typed(timeout):
|
|
574
|
+
"""The line sent, "" for Enter on nothing, or None."""
|
|
575
|
+
if not reading[0]:
|
|
576
|
+
return None
|
|
577
|
+
try:
|
|
578
|
+
line = reader.poll(timeout)
|
|
579
|
+
except EOFError:
|
|
580
|
+
reading[0] = False
|
|
581
|
+
if prompted[0]:
|
|
582
|
+
print()
|
|
583
|
+
prompted[0] = False
|
|
584
|
+
return None
|
|
585
|
+
return None if line is None else line.strip()
|
|
586
|
+
|
|
587
|
+
def _close_prompt():
|
|
588
|
+
if prompted[0]:
|
|
589
|
+
print()
|
|
590
|
+
prompted[0] = False
|
|
591
|
+
|
|
521
592
|
announced = False
|
|
522
593
|
# §64 A21. Set by the handler below instead of an unwind; read after the
|
|
523
594
|
# wake turn has finished. A list rather than a `nonlocal` because the
|
|
@@ -541,21 +612,48 @@ def _await_sessions(thread_id, *, model, provider_name, effort,
|
|
|
541
612
|
|
|
542
613
|
try:
|
|
543
614
|
while True:
|
|
544
|
-
|
|
615
|
+
# `waited_on`, not `live_for` (batch 265): an interactive shell
|
|
616
|
+
# is not waited on (SS25), and counting one spun this loop for
|
|
617
|
+
# as long as it stayed open, `wait_for_wake` answering None at
|
|
618
|
+
# once each time.
|
|
619
|
+
live = sessions.waited_on(thread_id)
|
|
545
620
|
if not live and not sessions.pending(thread_id):
|
|
546
|
-
return
|
|
621
|
+
return _WaitEnd(prompt_open=prompted[0])
|
|
547
622
|
if not announced:
|
|
548
|
-
print(f"[waiting on {live} background session(s) — "
|
|
549
|
-
f"Ctrl+C stops them]")
|
|
623
|
+
print(f"[waiting on {live} background session(s) — type "
|
|
624
|
+
f"a message to send it now; Ctrl+C stops them]")
|
|
550
625
|
announced = True
|
|
551
|
-
|
|
626
|
+
if reading[0] and not prompted[0]:
|
|
627
|
+
print("You: ", end="", flush=True)
|
|
628
|
+
prompted[0] = True
|
|
629
|
+
line = _typed(_WAIT_POLL_S)
|
|
630
|
+
if line:
|
|
631
|
+
prompted[0] = False
|
|
632
|
+
return _WaitEnd(line=line, resume=_again)
|
|
633
|
+
if line == "":
|
|
634
|
+
# Enter on nothing, which the prompt answers with itself.
|
|
635
|
+
prompted[0] = False
|
|
636
|
+
continue
|
|
637
|
+
batch = sessions.wait_for_wake(
|
|
638
|
+
thread_id, timeout=_WAIT_POLL_S if reading[0] else 1.0)
|
|
552
639
|
if batch is None:
|
|
553
640
|
if sessions.suspended(thread_id):
|
|
641
|
+
_close_prompt()
|
|
554
642
|
print("[no further automatic replies until you say "
|
|
555
643
|
"something — results will arrive with your next "
|
|
556
644
|
"message]")
|
|
557
|
-
return
|
|
645
|
+
return _WaitEnd()
|
|
558
646
|
continue
|
|
647
|
+
line = _typed(0)
|
|
648
|
+
if line:
|
|
649
|
+
# The owner's rule: a message and a result arriving
|
|
650
|
+
# together go message first.
|
|
651
|
+
sessions.put_back(thread_id, batch)
|
|
652
|
+
prompted[0] = False
|
|
653
|
+
return _WaitEnd(line=line, resume=_again)
|
|
654
|
+
if line == "":
|
|
655
|
+
prompted[0] = False
|
|
656
|
+
_close_prompt()
|
|
559
657
|
text, record = build_wake(batch)
|
|
560
658
|
previous_sigint = signal.signal(signal.SIGINT, _SecondPress(
|
|
561
659
|
"_stop_sessions_after_this_turn",
|
|
@@ -576,13 +674,17 @@ def _await_sessions(thread_id, *, model, provider_name, effort,
|
|
|
576
674
|
except Exception as e:
|
|
577
675
|
logger.exception("Wake turn failed")
|
|
578
676
|
print(f"\n[Error: {ansi.for_display(e)}]")
|
|
579
|
-
return
|
|
677
|
+
return _WaitEnd()
|
|
580
678
|
finally:
|
|
581
679
|
# Before anything else can be printed or read, and on the
|
|
582
680
|
# error path too: outside a wake turn the interrupt belongs
|
|
583
681
|
# to the sessions and takes the handler below.
|
|
584
682
|
signal.signal(signal.SIGINT, previous_sigint)
|
|
585
683
|
live.finish(response)
|
|
684
|
+
# ROADMAP_v3 §97 (UF4). A wake turn's calls are this
|
|
685
|
+
# thread's too, as they are on the TUI's memory.
|
|
686
|
+
if count_usage is not None:
|
|
687
|
+
count_usage(response)
|
|
586
688
|
if interrupted:
|
|
587
689
|
# The turn finished and wrote its rows; now the key does
|
|
588
690
|
# what it said. Returning to the prompt rather than waiting
|
|
@@ -592,10 +694,11 @@ def _await_sessions(thread_id, *, model, provider_name, effort,
|
|
|
592
694
|
killed = sessions.kill_owned(thread_id,
|
|
593
695
|
shell_sessions.KILL_USER)
|
|
594
696
|
print(f"\n[stopped {len(killed)} background session(s)]")
|
|
595
|
-
return
|
|
697
|
+
return _WaitEnd()
|
|
596
698
|
except KeyboardInterrupt:
|
|
597
699
|
killed = sessions.kill_owned(thread_id, shell_sessions.KILL_USER)
|
|
598
700
|
print(f"\n[stopped {len(killed)} background session(s)]")
|
|
701
|
+
return _WaitEnd()
|
|
599
702
|
|
|
600
703
|
|
|
601
704
|
def _close_sessions() -> None:
|
|
@@ -686,18 +789,30 @@ def run_chat(
|
|
|
686
789
|
print("Ctrl+C twice stops the current turn; Ctrl+C or Ctrl+D at the "
|
|
687
790
|
"prompt exits. /help lists commands.\n")
|
|
688
791
|
|
|
792
|
+
# Batch 265 (SS2 as amended). How the last session wait ended: a line
|
|
793
|
+
# the user sent while it waited runs below exactly as one typed at the
|
|
794
|
+
# prompt would, and a COMMAND sent that way goes back to the wait it
|
|
795
|
+
# interrupted (a message ends in a wait of its own).
|
|
796
|
+
waited = _WaitEnd()
|
|
689
797
|
while True:
|
|
798
|
+
from_wait = None
|
|
690
799
|
if first_message is not None:
|
|
691
800
|
user_input = first_message
|
|
692
801
|
first_message = None
|
|
802
|
+
elif waited.line is not None:
|
|
803
|
+
from_wait, user_input = waited, waited.line
|
|
693
804
|
else:
|
|
805
|
+
# "You: " is already on screen when the wait ended with it
|
|
806
|
+
# open, and printing a second would put two on one line.
|
|
807
|
+
prompt = "" if waited.prompt_open else "You: "
|
|
808
|
+
waited = _WaitEnd()
|
|
694
809
|
try:
|
|
695
810
|
# §29 (N1). Audit #100: this was input(), a SECOND reader
|
|
696
811
|
# on the stdin the approval pump already owns, so from the
|
|
697
812
|
# first gated call onwards the pump took this line and
|
|
698
813
|
# input() blocked forever. EOFError still arrives here --
|
|
699
814
|
# readline() raises it on Ctrl+D exactly as input() did.
|
|
700
|
-
user_input = _stdin_reader().readline(
|
|
815
|
+
user_input = _stdin_reader().readline(prompt).strip()
|
|
701
816
|
except (KeyboardInterrupt, EOFError):
|
|
702
817
|
# ROADMAP_v3 §49 (SS6). Quitting with sessions still
|
|
703
818
|
# running confirms first -- they are killed on the way out
|
|
@@ -708,6 +823,7 @@ def run_chat(
|
|
|
708
823
|
continue
|
|
709
824
|
print("\nExiting.")
|
|
710
825
|
break
|
|
826
|
+
waited = _WaitEnd()
|
|
711
827
|
|
|
712
828
|
if not user_input:
|
|
713
829
|
continue
|
|
@@ -715,15 +831,22 @@ def run_chat(
|
|
|
715
831
|
agent = session.active_agent
|
|
716
832
|
one_shot = False
|
|
717
833
|
if user_input.startswith("/"):
|
|
718
|
-
|
|
834
|
+
# `waiting`: the commands the TUI refuses while a turn's
|
|
835
|
+
# sessions run are refused here too, when this one was sent
|
|
836
|
+
# during the wait (SS2 as amended, the owner's choice).
|
|
837
|
+
run_command(session, user_input, waiting=from_wait is not None)
|
|
719
838
|
if session.quit_requested:
|
|
720
839
|
# /quit takes Ctrl+D's route, confirmation and all.
|
|
721
840
|
session.quit_requested = False
|
|
722
841
|
if _quit_confirmed():
|
|
723
842
|
print("Exiting.")
|
|
724
843
|
break
|
|
844
|
+
if from_wait is not None:
|
|
845
|
+
waited = from_wait.resume()
|
|
725
846
|
continue
|
|
726
847
|
if session.one_shot is None:
|
|
848
|
+
if from_wait is not None:
|
|
849
|
+
waited = from_wait.resume()
|
|
727
850
|
continue
|
|
728
851
|
# /grill-me: one turn in this thread under that agent's prompt,
|
|
729
852
|
# run like a typed message so it prints live and stops the same
|
|
@@ -731,13 +854,16 @@ def run_chat(
|
|
|
731
854
|
agent, session.one_shot = session.one_shot, None
|
|
732
855
|
user_input, one_shot = GRILL_ME_TASK, True
|
|
733
856
|
|
|
734
|
-
_chat_turn(session, user_input, agent, one_shot=one_shot)
|
|
857
|
+
waited = _chat_turn(session, user_input, agent, one_shot=one_shot)
|
|
735
858
|
|
|
736
859
|
|
|
737
860
|
def _chat_turn(session: ChatSession, user_input: str, agent,
|
|
738
|
-
one_shot: bool = False) ->
|
|
861
|
+
one_shot: bool = False) -> _WaitEnd:
|
|
739
862
|
"""One turn of `run_chat`'s conversation, and the wake turns after it.
|
|
740
863
|
|
|
864
|
+
Returns how the wait after it ended (`_WaitEnd`), since batch 265 a
|
|
865
|
+
line the user sent during it among other things.
|
|
866
|
+
|
|
741
867
|
`agent` is the session's active agent, or /grill-me's for its one turn
|
|
742
868
|
(`one_shot`). What the agent changes about the turn is
|
|
743
869
|
`core.loop.agent_turn`'s to say, the function the TUI's turn calls, so
|
|
@@ -820,11 +946,13 @@ def _chat_turn(session: ChatSession, user_input: str, agent,
|
|
|
820
946
|
except Exception as e:
|
|
821
947
|
logger.exception("Chat turn failed")
|
|
822
948
|
print(f"\n[Error: {ansi.for_display(e)}]")
|
|
823
|
-
return
|
|
949
|
+
return _WaitEnd()
|
|
824
950
|
finally:
|
|
825
951
|
signal.signal(signal.SIGINT, previous_sigint)
|
|
826
952
|
|
|
827
953
|
session.thread_id = response.thread_id
|
|
954
|
+
# ROADMAP_v3 §97 (UF4): this turn's calls, for /usage.
|
|
955
|
+
session.count_usage(response)
|
|
828
956
|
|
|
829
957
|
# ROADMAP_v2 §21's "no silent compaction, ever": a notice printed
|
|
830
958
|
# live when it happened, and one that never arrived live is printed
|
|
@@ -871,12 +999,14 @@ def _chat_turn(session: ChatSession, user_input: str, agent,
|
|
|
871
999
|
# SS2/SS5. The turn is over, but this conversation may not be: a
|
|
872
1000
|
# session it started reports later, and that report starts a turn
|
|
873
1001
|
# of its own -- under the same agent, prompt and skills, because it
|
|
874
|
-
# IS this turn continuing.
|
|
875
|
-
#
|
|
876
|
-
_await_sessions(
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
1002
|
+
# IS this turn continuing. Since batch 265 the user can type while it
|
|
1003
|
+
# waits (SS2 as amended): a line sent then comes back from here.
|
|
1004
|
+
return _await_sessions(
|
|
1005
|
+
session.thread_id, model=turn.model,
|
|
1006
|
+
provider_name=turn.provider_name, effort=session.effort,
|
|
1007
|
+
authorization=session.authorization, turn=turn,
|
|
1008
|
+
active_skills=session.active_skills,
|
|
1009
|
+
count_usage=session.count_usage)
|
|
880
1010
|
|
|
881
1011
|
|
|
882
1012
|
def build_research_authorization(grant_spec, attended: bool = False):
|
|
@@ -1301,6 +1431,39 @@ class _StdinReader:
|
|
|
1301
1431
|
raise EOFError
|
|
1302
1432
|
return line.rstrip("\r\n")
|
|
1303
1433
|
|
|
1434
|
+
def poll(self, timeout: float):
|
|
1435
|
+
"""A line typed while the chat waits on background sessions, or
|
|
1436
|
+
None if none arrived within *timeout* (ROADMAP_v3 §49, SS2 as
|
|
1437
|
+
amended in batch 265). Raises EOFError once stdin has ended.
|
|
1438
|
+
|
|
1439
|
+
readline()'s discipline, with ask()'s deadline: it KEEPS
|
|
1440
|
+
type-ahead, because a line typed during the turn that started the
|
|
1441
|
+
sessions is a message meant for the prompt this wait now shows,
|
|
1442
|
+
and it drains nothing for the same reason. It takes no prompt, as
|
|
1443
|
+
it is called again every fraction of a second; the wait prints the
|
|
1444
|
+
prompt once.
|
|
1445
|
+
|
|
1446
|
+
IT STARTS THE PUMP, which readline() takes care not to do: a read
|
|
1447
|
+
with a deadline cannot be made on a thread that might never come
|
|
1448
|
+
back, which is the reason the pump exists at all. Chat mode is the
|
|
1449
|
+
only caller and never hands the terminal to Textual afterwards, so
|
|
1450
|
+
batch 121's parked-reader hazard does not reach it.
|
|
1451
|
+
"""
|
|
1452
|
+
import queue as _queue
|
|
1453
|
+
|
|
1454
|
+
if self._released:
|
|
1455
|
+
raise EOFError
|
|
1456
|
+
self._start()
|
|
1457
|
+
if self._ended():
|
|
1458
|
+
raise EOFError
|
|
1459
|
+
try:
|
|
1460
|
+
line = self._lines.get(timeout=timeout)
|
|
1461
|
+
except _queue.Empty:
|
|
1462
|
+
return None
|
|
1463
|
+
if line is self._EOF:
|
|
1464
|
+
raise EOFError
|
|
1465
|
+
return line.rstrip("\r\n")
|
|
1466
|
+
|
|
1304
1467
|
def _read_inline(self) -> str:
|
|
1305
1468
|
"""One line, on this thread, leaving no reader behind.
|
|
1306
1469
|
|
package/package.json
CHANGED
package/pyproject.toml
CHANGED
|
@@ -18,7 +18,7 @@ build-backend = "setuptools.build_meta"
|
|
|
18
18
|
|
|
19
19
|
[project]
|
|
20
20
|
name = "venastine-research-harness"
|
|
21
|
-
version = "0.3.
|
|
21
|
+
version = "0.3.17"
|
|
22
22
|
description = "A from-scratch agentic harness: tool-using chat plus a ten-pass self-critiquing research pipeline."
|
|
23
23
|
readme = "README.md"
|
|
24
24
|
# The same words as package.json's `keywords`, in the same order: what a
|
|
@@ -273,6 +273,35 @@ _SECRET_PATTERNS = [
|
|
|
273
273
|
# --- `sk-` always have; one vendor list, one vocabulary.
|
|
274
274
|
# GitHub fine-grained PATs, the format `ghp_` is being retired for
|
|
275
275
|
re.compile(r"github_pat_[A-Za-z0-9_]{22,}"),
|
|
276
|
+
# GitHub's STATELESS tokens (batch 260): `ghs_APPID_JWT`, a JWT after
|
|
277
|
+
# the prefix, about 520 characters with two dots where the classic
|
|
278
|
+
# shape below is 40 with none. GitHub's docs and changelog say every
|
|
279
|
+
# installation token minted since its rollout finished on 2026-10-02
|
|
280
|
+
# has this form, Actions' GITHUB_TOKEN included, and the classic
|
|
281
|
+
# pattern stops at the first underscore. Measured before this: a fake
|
|
282
|
+
# passed `redact_output_text` whole and refused no input.
|
|
283
|
+
# Three base64url runs of at least 20 characters, joined by the two
|
|
284
|
+
# dots GitHub says tell the forms apart. Nothing about the JWT's
|
|
285
|
+
# contents, which GitHub says clients must not rely on, and no dot in
|
|
286
|
+
# the runs, so a sentence's final period stays (GitHub's own
|
|
287
|
+
# suggestion, `ghs_` and 36 of `[A-Za-z0-9._-]`, redacted 3 of 8
|
|
288
|
+
# lines of code and prose that merely name something `ghs_`). All
|
|
289
|
+
# four prefixes, the owner's choice: GitHub has changed `ghs_` and
|
|
290
|
+
# says user-to-server tokens are next.
|
|
291
|
+
# The lookbehind keeps it linear. Without it, every `ghs_` inside one
|
|
292
|
+
# long run is a start that scans to the run's end: measured quadratic,
|
|
293
|
+
# 99 s on a hostile megabyte against 0.03 s with it. A token character
|
|
294
|
+
# is allowed before the prefix only where it closes an escape, `%3D`
|
|
295
|
+
# or `\n`, since a URL-encoded or JSON-escaped token is still a token,
|
|
296
|
+
# and the classic pattern catches those. Each of those starts sits two
|
|
297
|
+
# characters after a `%` or a `\`, which no run contains, so the scan
|
|
298
|
+
# stays linear.
|
|
299
|
+
# This pattern runs BEFORE the classic one, because a 36-character
|
|
300
|
+
# alphanumeric run inside a segment would otherwise take a piece of
|
|
301
|
+
# the token and leave the rest standing.
|
|
302
|
+
re.compile(r"(?:(?<![A-Za-z0-9_-])|(?<=%[0-9A-Fa-f]{2})|(?<=\\[nrt]))"
|
|
303
|
+
r"gh[ousr]_[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}"
|
|
304
|
+
r"\.[A-Za-z0-9_-]{20,}"),
|
|
276
305
|
# GitHub OAuth, user-to-server, server-to-server and refresh tokens
|
|
277
306
|
re.compile(r"gh[ousr]_[A-Za-z0-9]{36}"),
|
|
278
307
|
# OpenAI project, service-account and admin keys: `sk-proj-...`
|
package/security/sandbox.py
CHANGED
|
@@ -2599,6 +2599,15 @@ def _interactive_command(prompt_token: str) -> str:
|
|
|
2599
2599
|
prompt=prompt_token)
|
|
2600
2600
|
|
|
2601
2601
|
|
|
2602
|
+
#: Where the container mounts the workspace, and its working directory.
|
|
2603
|
+
#: Named once (ROADMAP_v3 §96, CW1) because the file tools read it too:
|
|
2604
|
+
#: `file_ops.resolve_path` maps a path spelled this way onto the workspace
|
|
2605
|
+
#: it names, and a second literal there would let the mount move without
|
|
2606
|
+
#: the map. Always a POSIX path, whatever the host is -- it is a path
|
|
2607
|
+
#: inside a Linux container.
|
|
2608
|
+
CONTAINER_WORKSPACE = "/workspace"
|
|
2609
|
+
|
|
2610
|
+
|
|
2602
2611
|
def _docker_argv(
|
|
2603
2612
|
command: str,
|
|
2604
2613
|
workspace_dir: str,
|
|
@@ -2674,8 +2683,8 @@ def _docker_argv(
|
|
|
2674
2683
|
# may run at once, so cleaning up "our" containers by the label
|
|
2675
2684
|
# above would kill the other's sessions; this one scopes it.
|
|
2676
2685
|
"--label", f"venastine.process={PROCESS_TOKEN}",
|
|
2677
|
-
"-v", f"{workspace_real}
|
|
2678
|
-
"-w",
|
|
2686
|
+
"-v", f"{workspace_real}:{CONTAINER_WORKSPACE}",
|
|
2687
|
+
"-w", CONTAINER_WORKSPACE,
|
|
2679
2688
|
"--memory", f"{config.SANDBOX_MEMORY_MB}m",
|
|
2680
2689
|
"--cpus", "1",
|
|
2681
2690
|
"--pids-limit", str(config.SANDBOX_MAX_PIDS),
|
package/tools/builtin/arxiv.py
CHANGED
|
@@ -577,7 +577,9 @@ def run(params: dict) -> dict:
|
|
|
577
577
|
# Returned rather than raised, like every other dead end in
|
|
578
578
|
# this function: a search tool that cannot answer is a
|
|
579
579
|
# result the model can work around.
|
|
580
|
-
|
|
580
|
+
# INFO (batch 266): the result below says this, and both
|
|
581
|
+
# shells print it; the file keeps the query and the page.
|
|
582
|
+
logger.info(
|
|
581
583
|
"arxiv_search dropped a feed over %d bytes for %r "
|
|
582
584
|
"(max_results=%d, start=%d)",
|
|
583
585
|
MAX_FEED_BYTES, parsed.keywords, parsed.max_results,
|
|
@@ -636,7 +638,11 @@ def run(params: dict) -> dict:
|
|
|
636
638
|
}
|
|
637
639
|
except (httpx.HTTPError, ET.ParseError) as e:
|
|
638
640
|
last_exc = e
|
|
639
|
-
|
|
641
|
+
# INFO (batch 266), as web_search's retries: the last
|
|
642
|
+
# attempt's error is the result, and a recovered attempt is
|
|
643
|
+
# nothing to show. NOT the hostile-feed refusal above, whose
|
|
644
|
+
# line keeps what the result deliberately leaves out.
|
|
645
|
+
logger.info(
|
|
640
646
|
"arxiv_search attempt %s/%s failed for %r: %s",
|
|
641
647
|
attempt + 1, MAX_RETRIES + 1, parsed.keywords, e)
|
|
642
648
|
else:
|
|
@@ -616,7 +616,14 @@ def run(params: dict) -> dict:
|
|
|
616
616
|
# dropped -- fifteen consecutive "fetch_url failed" lines with
|
|
617
617
|
# no URL and no reason, which is what made a run of ordinary
|
|
618
618
|
# 404s indistinguishable from a broken tool.
|
|
619
|
-
|
|
619
|
+
#
|
|
620
|
+
# INFO, not WARNING (batch 266, the owner's choice). The error
|
|
621
|
+
# below IS the result, and both shells print a failed result as
|
|
622
|
+
# its own line, so a WARNING drew the same failure twice: the
|
|
623
|
+
# owner's screenshot had both, line for line. The file keeps
|
|
624
|
+
# this one with the hop's URL, which the result does not name
|
|
625
|
+
# after a redirect.
|
|
626
|
+
logger.info("fetch_url failed for %s: %s", url, e)
|
|
620
627
|
return {"error": f"Could not fetch URL: {e}"}
|
|
621
628
|
|
|
622
629
|
held = _held(url, kind, raw, more, response, want)
|
|
@@ -21,6 +21,12 @@ SECURITY MODEL (ROADMAP §6, deviated per user decision):
|
|
|
21
21
|
resolved via os.path.realpath before any check, so a path that looks
|
|
22
22
|
inside the workspace but resolves outside it is treated as outside.
|
|
23
23
|
|
|
24
|
+
`/workspace/x` IS INSIDE (ROADMAP_v3 §96, batch 263): it is the
|
|
25
|
+
container's name for `<workspace>/x`, and resolve_path maps it there
|
|
26
|
+
while a tool that starts shell commands is enabled. Every other path
|
|
27
|
+
outside the workspace still asks, the owner's choice: a prompt is how
|
|
28
|
+
this layer avoids saying whether a file exists before anyone said yes.
|
|
29
|
+
|
|
24
30
|
PROTECTED SEGMENTS (security/protected_paths.py) are the one hard
|
|
25
31
|
deny, and the exception to both rules above: any path -- inside or
|
|
26
32
|
outside the workspace -- that resolves into `.venastine/` is refused
|
|
@@ -84,6 +90,7 @@ from pydantic import BaseModel, Field
|
|
|
84
90
|
import config
|
|
85
91
|
from json_store import write_text_atomic
|
|
86
92
|
from security.protected_paths import hardlinked_segment, protected_segment
|
|
93
|
+
from security.sandbox import CONTAINER_WORKSPACE
|
|
87
94
|
from tools.base import invalid_params
|
|
88
95
|
|
|
89
96
|
logger = logging.getLogger(__name__)
|
|
@@ -113,15 +120,67 @@ def resolve_path(user_path: str) -> str:
|
|
|
113
120
|
``os.path.join(WORKSPACE_ROOT, ...)`` at the call site is this
|
|
114
121
|
project's canonical producer/consumer bug, and here it would mean a
|
|
115
122
|
diff of one file beside an edit to another.
|
|
123
|
+
|
|
124
|
+
THE CONTAINER'S SPELLING (ROADMAP_v3 §96, CW1-CW3). The shell runs
|
|
125
|
+
in a container that mounts the workspace at `/workspace`, so the
|
|
126
|
+
paths the model sees there -- `pwd`, a build's error lines -- name
|
|
127
|
+
workspace files as `/workspace/x`. These tools run on the host, where
|
|
128
|
+
that path is somewhere else or nowhere: until batch 263 such a call
|
|
129
|
+
asked to leave the workspace and then failed on a file that was
|
|
130
|
+
there all along. So `/workspace/x` resolves to `<workspace>/x`, here
|
|
131
|
+
and nowhere else, which is what makes the approval check, the four
|
|
132
|
+
handlers, the protected deny and the TUI's diff agree on it.
|
|
133
|
+
|
|
134
|
+
Only while a tool that starts commands is enabled (CW1), because that
|
|
135
|
+
is where the model learns the spelling. Only when the host's own
|
|
136
|
+
reading of the path lands OUTSIDE the workspace, so a host whose
|
|
137
|
+
workspace really lives under `/workspace` keeps its meaning, and only
|
|
138
|
+
when the mapped path lands INSIDE it, so `..` or a link out of it
|
|
139
|
+
keeps the prompt it had (CW2).
|
|
140
|
+
"""
|
|
141
|
+
host = os.path.realpath(os.path.join(WORKSPACE_ROOT, user_path))
|
|
142
|
+
rest = _container_remainder(user_path)
|
|
143
|
+
if rest is None or _inside(host) or not _shell_starts_commands():
|
|
144
|
+
return host
|
|
145
|
+
mapped = os.path.realpath(os.path.join(WORKSPACE_ROOT, rest))
|
|
146
|
+
return mapped if _inside(mapped) else host
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _container_remainder(user_path: str) -> Optional[str]:
|
|
150
|
+
"""What follows the container's mount point in *user_path*, or None
|
|
151
|
+
when the path is not spelled from it.
|
|
152
|
+
|
|
153
|
+
The mount point is a whole SEGMENT: `/workspace` and `/workspace/x`
|
|
154
|
+
are the container's spelling and `/workspacefoo` is not. Leading
|
|
155
|
+
slashes are dropped from the remainder because POSIX folds a run of
|
|
156
|
+
them into one -- `/workspace//x` is `/workspace/x` in the container --
|
|
157
|
+
while `os.path.join` would read the second as a new root.
|
|
116
158
|
"""
|
|
117
|
-
|
|
159
|
+
if user_path == CONTAINER_WORKSPACE:
|
|
160
|
+
return ""
|
|
161
|
+
if user_path.startswith(CONTAINER_WORKSPACE + "/"):
|
|
162
|
+
return user_path[len(CONTAINER_WORKSPACE):].lstrip("/")
|
|
163
|
+
return None
|
|
118
164
|
|
|
119
165
|
|
|
120
|
-
def
|
|
121
|
-
|
|
166
|
+
def _shell_starts_commands() -> bool:
|
|
167
|
+
"""Whether any tool that runs a command in the container may be
|
|
168
|
+
called (§96 CW1): the same four `shell_approval_mode` governs, asked
|
|
169
|
+
through `is_tool_allowed`, so a hosted deployment that refuses them
|
|
170
|
+
maps nothing either."""
|
|
171
|
+
from security.permissions import APPROVAL_BY_SHELL_MODE, is_tool_allowed
|
|
172
|
+
|
|
173
|
+
return any(is_tool_allowed(name) for name in APPROVAL_BY_SHELL_MODE)
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _inside(resolved: str) -> bool:
|
|
122
177
|
return resolved == WORKSPACE_ROOT or resolved.startswith(WORKSPACE_ROOT + os.sep)
|
|
123
178
|
|
|
124
179
|
|
|
180
|
+
def _is_within_workspace(user_path: str) -> bool:
|
|
181
|
+
return _inside(resolve_path(user_path))
|
|
182
|
+
|
|
183
|
+
|
|
125
184
|
def _file_approval_check(tool_name: str, params: dict) -> bool:
|
|
126
185
|
"""Path-dependent approval policy for read / write / edit.
|
|
127
186
|
|
|
@@ -414,7 +473,8 @@ def read_run(params: dict) -> dict:
|
|
|
414
473
|
with open(resolved, encoding="utf-8", errors="replace") as f:
|
|
415
474
|
all_lines = f.read().splitlines()
|
|
416
475
|
except Exception as e:
|
|
417
|
-
|
|
476
|
+
# INFO (batch 266): the result below says this; the file keeps the trace.
|
|
477
|
+
logger.info("read tool failed on %s", parsed.path, exc_info=True)
|
|
418
478
|
return {"error": f"Could not read file: {e}"}
|
|
419
479
|
|
|
420
480
|
content, lines_returned, truncated, message = _truncate_lines(
|
|
@@ -482,7 +542,8 @@ def write_run(params: dict) -> dict:
|
|
|
482
542
|
created = not os.path.exists(resolved)
|
|
483
543
|
write_text_atomic(resolved, parsed.content)
|
|
484
544
|
except Exception as e:
|
|
485
|
-
|
|
545
|
+
# INFO (batch 266): the result below says this; the file keeps the trace.
|
|
546
|
+
logger.info("write tool failed on %s", parsed.path, exc_info=True)
|
|
486
547
|
return {"error": f"Could not write file: {e}"}
|
|
487
548
|
|
|
488
549
|
if created:
|
|
@@ -551,7 +612,9 @@ def edit_run(params: dict) -> dict:
|
|
|
551
612
|
with open(resolved, encoding="utf-8", newline="") as f:
|
|
552
613
|
content = f.read()
|
|
553
614
|
except Exception as e:
|
|
554
|
-
|
|
615
|
+
# INFO (batch 266): the result below says this; the file keeps the trace.
|
|
616
|
+
logger.info("edit tool read failed on %s", parsed.path,
|
|
617
|
+
exc_info=True)
|
|
555
618
|
return {"error": f"Could not read file: {e}"}
|
|
556
619
|
|
|
557
620
|
old_text, new_text = parsed.old_text, parsed.new_text
|
|
@@ -586,7 +649,9 @@ def edit_run(params: dict) -> dict:
|
|
|
586
649
|
try:
|
|
587
650
|
write_text_atomic(resolved, content)
|
|
588
651
|
except Exception as e:
|
|
589
|
-
|
|
652
|
+
# INFO (batch 266): the result below says this; the file keeps the trace.
|
|
653
|
+
logger.info("edit tool write failed on %s", parsed.path,
|
|
654
|
+
exc_info=True)
|
|
590
655
|
return {"error": f"Could not write file: {e}"}
|
|
591
656
|
|
|
592
657
|
return {"result": f"Replaced 1 occurrence in {parsed.path}"}
|
|
@@ -685,7 +750,8 @@ def view_image_run(params: dict, memory=None, call_id=None,
|
|
|
685
750
|
with open(resolved, "rb") as f:
|
|
686
751
|
raw = f.read(config.MAX_IMAGE_BYTES + 1)
|
|
687
752
|
except OSError as e:
|
|
688
|
-
|
|
753
|
+
# INFO (batch 266): the result below says this; the file keeps the trace.
|
|
754
|
+
logger.info("view_image failed on %s", parsed.path, exc_info=True)
|
|
689
755
|
return {"error": f"Could not read file: {e}"}
|
|
690
756
|
if len(raw) > config.MAX_IMAGE_BYTES:
|
|
691
757
|
return {"error": (
|
|
@@ -311,7 +311,10 @@ def read_run(params: dict) -> dict:
|
|
|
311
311
|
chunk = f.read(config.INIT_READ_CHARS)
|
|
312
312
|
more = f.read(1) != ""
|
|
313
313
|
except OSError as e:
|
|
314
|
-
|
|
314
|
+
# INFO, not WARNING (batch 266, the owner's choice): the error below
|
|
315
|
+
# is the result, and both shells print a failed result as a line of
|
|
316
|
+
# its own, so a WARNING here drew the one failure twice.
|
|
317
|
+
logger.info("read_project_doc failed for %s: %s", user_path, e)
|
|
315
318
|
return {"error": f"Could not read {user_path}: {e}"}
|
|
316
319
|
|
|
317
320
|
result = {"path": user_path, "content": chunk, "truncated": more}
|
|
@@ -442,8 +445,10 @@ def write_run(params: dict) -> dict:
|
|
|
442
445
|
destination = doc_path(root, name)
|
|
443
446
|
resolved = _resolve_in_project(root, name)
|
|
444
447
|
if resolved is None:
|
|
445
|
-
|
|
446
|
-
|
|
448
|
+
# INFO (batch 266): the error below is the result, which both shells
|
|
449
|
+
# print as a line of its own. The file keeps the path it resolved to.
|
|
450
|
+
logger.info("write_project_doc refused %s: resolves outside the "
|
|
451
|
+
"project", destination)
|
|
447
452
|
return {"error": f"write_project_doc: {name!r} resolves outside the "
|
|
448
453
|
f"project directory, so it is not one of this "
|
|
449
454
|
f"project's documents."}
|
|
@@ -458,7 +463,8 @@ def write_run(params: dict) -> dict:
|
|
|
458
463
|
# approval that was shown a diff of the lines that changed.
|
|
459
464
|
write_text_atomic(resolved, content)
|
|
460
465
|
except OSError as e:
|
|
461
|
-
|
|
466
|
+
# INFO (batch 266), as `read_project_doc`'s failure: the result says it.
|
|
467
|
+
logger.info("write_project_doc failed for %s: %s", resolved, e)
|
|
462
468
|
return {"error": f"Could not write {resolved}: {e}"}
|
|
463
469
|
|
|
464
470
|
return {"result": f"Wrote {len(content)} characters to {resolved}"}
|
package/tools/builtin/shell.py
CHANGED
|
@@ -1115,5 +1115,6 @@ def run(params: dict) -> dict:
|
|
|
1115
1115
|
except SandboxUnavailable as e:
|
|
1116
1116
|
return {"error": str(e)}
|
|
1117
1117
|
except Exception as e:
|
|
1118
|
-
|
|
1118
|
+
# INFO (batch 266): the result below says this; the file keeps the trace.
|
|
1119
|
+
logger.info("shell tool failed", exc_info=True)
|
|
1119
1120
|
return {"error": f"Shell execution failed: {e}"}
|