@yahyahammad/venastine 0.2.0 → 0.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +9 -4
- package/CONFIG_ARCHITECTURE.md +2030 -0
- package/LICENSE +201 -201
- package/NOTICE +1 -1
- package/README.md +1034 -895
- package/agents/builtin/build.md +73 -0
- package/agents/builtin/compactor.md +75 -75
- package/agents/builtin/explore.md +77 -77
- package/agents/builtin/general.md +80 -0
- package/agents/builtin/pipeline-reviewer.md +100 -100
- package/agents/builtin/plan.md +106 -88
- package/agents/builtin/review.md +74 -74
- package/agents/builtin/test.md +65 -0
- package/agents/builtin/writer.md +67 -0
- package/agents/manager.py +68 -2
- package/agents/subagent_tool.py +472 -238
- package/agents/tui_commands.py +238 -151
- package/bin/venastine.mjs +150 -4
- package/bin/venastine.py +71 -0
- package/cli_commands.py +838 -0
- package/config.py +95 -1198
- package/config.yaml +1103 -0
- package/config_edit.py +1301 -0
- package/config_schema.py +971 -0
- package/config_update.py +749 -0
- package/core/agent_activity.py +256 -118
- package/core/ansi.py +452 -0
- package/core/approval.py +137 -115
- package/core/call_view.py +364 -0
- package/core/cancel.py +121 -0
- package/core/chatgpt_auth.py +747 -0
- package/core/claims_view.py +107 -0
- package/core/client.py +2715 -1559
- package/core/commands.py +460 -0
- package/core/compaction.py +1735 -971
- package/core/config_loader.py +1982 -1563
- package/core/connect.py +686 -0
- package/core/events.py +123 -61
- package/core/file_mentions.py +280 -0
- package/core/interaction.py +411 -328
- package/core/line_pattern.py +128 -0
- package/core/loop.py +2566 -1490
- package/core/memory.py +482 -370
- package/core/pipeline_models.py +190 -176
- package/core/provider_errors.py +234 -0
- package/core/reasoning/authorization.py +130 -130
- package/core/reasoning/base.py +219 -219
- package/core/reasoning/confidence_scoring.py +484 -468
- package/core/reasoning/events.py +149 -149
- package/core/reasoning/json_retry.py +19 -1
- package/core/reasoning/orchestrator.py +1561 -1382
- package/core/reasoning/output_writer.py +345 -286
- package/core/reasoning/payload_validation.py +524 -398
- package/core/reasoning/pipeline_storage.py +88 -22
- package/core/reasoning/review.py +659 -615
- package/core/reasoning/scholar.py +458 -434
- package/core/reasoning/source_corpus.py +433 -327
- package/core/reasoning/source_scoring.py +855 -855
- package/core/replay.py +315 -183
- package/core/session.py +36 -12
- package/core/session_view.py +259 -0
- package/core/session_wake.py +259 -0
- package/core/shell_sessions.py +1553 -0
- package/core/workspace_trust.py +1 -1
- package/credentials.py +148 -20
- package/database.py +333 -252
- package/{PRIVACY.md → docs/PRIVACY.md} +78 -66
- package/docs/SECURITY.md +108 -0
- package/docs/THIRD_PARTY_NOTICES.md +148 -0
- package/env_secrets.py +50 -9
- package/json_store.py +343 -123
- package/logging_setup.py +295 -185
- package/main.py +3433 -1961
- package/mcp_client/client.py +733 -499
- package/mcp_client/config.py +126 -21
- package/mcp_client/registration.py +343 -182
- package/memories/manager.py +147 -132
- package/memories/tui_commands.py +319 -244
- package/package.json +303 -73
- package/project_init/config_files.py +282 -278
- package/project_init/generator.py +728 -564
- package/project_init/manifest.py +18 -6
- package/project_init/tui_commands.py +147 -144
- package/prompts/critic_pass.md +24 -24
- package/prompts/revalidate.md +25 -25
- package/prompts/revise.md +2 -0
- package/prompts/source_grounding.md +66 -66
- package/prompts/system_prompts.py +316 -262
- package/providers.json.example +4 -2
- package/pyproject.toml +149 -100
- package/requirements.txt +295 -72
- package/safety/policy_enforcement.py +1082 -746
- package/security/askpass.py +419 -0
- package/security/capability.py +207 -177
- package/security/confine.py +484 -0
- package/security/hosted.py +182 -0
- package/security/permissions.py +237 -190
- package/security/posture.py +124 -12
- package/security/protected_paths.py +501 -309
- package/security/sandbox.py +4108 -985
- package/security/secrets.py +855 -0
- package/skills/builtin/math/formal-specification.md +67 -67
- package/skills/builtin/math/numerical-methods.md +66 -66
- package/skills/builtin/research/experiment-design.md +62 -62
- package/skills/builtin/research/reproducibility.md +55 -55
- package/skills/builtin/research/source-evaluation.md +70 -70
- package/skills/builtin/research/statistical-inference.md +63 -63
- package/skills/builtin/research/technical-writing.md +59 -59
- package/skills/builtin/software/code-review.md +61 -61
- package/skills/builtin/software/debugging.md +61 -61
- package/skills/builtin/software/test-design.md +67 -67
- package/skills/tui_commands.py +110 -111
- package/storage.py +1338 -1005
- package/tools/base.py +161 -1
- package/tools/builtin/_math_common.py +667 -377
- package/tools/builtin/_net_common.py +225 -19
- package/tools/builtin/arxiv.py +497 -39
- package/tools/builtin/ask_user.py +231 -213
- package/tools/builtin/discrete_math.py +35 -4
- package/tools/builtin/fetch_url.py +371 -153
- package/tools/builtin/file_ops.py +720 -412
- package/tools/builtin/geometry.py +48 -12
- package/tools/builtin/get_time.py +2 -2
- package/tools/builtin/linear_algebra.py +66 -23
- package/tools/builtin/load_skill.py +76 -66
- package/tools/builtin/logic.py +171 -12
- package/tools/builtin/pin.py +227 -193
- package/tools/builtin/probability_stats.py +12 -16
- package/tools/builtin/project_docs.py +464 -405
- package/tools/builtin/remember.py +175 -158
- package/tools/builtin/shell.py +980 -461
- package/tools/builtin/shell_sessions.py +613 -0
- package/tools/builtin/symbolic_math.py +80 -10
- package/tools/builtin/todo.py +149 -147
- package/tools/builtin/web_search.py +191 -13
- package/tools/context.py +159 -136
- package/tools/isolation.py +107 -34
- package/tools/registry.py +1242 -925
- package/tui/app.py +8318 -4100
- package/tui/app.tcss +747 -556
- package/tui/commands.py +26 -170
- package/tui/diffs.py +34 -13
- package/tui/history.py +128 -128
- package/tui/hosted_driver.py +102 -0
- package/tui/markdown.py +1329 -1279
- package/tui/meters.py +364 -268
- package/tui/ravens.py +146 -146
- package/tui/screens.py +1255 -943
- package/tui/themes.py +599 -559
- package/tui/widgets.py +3813 -2332
- package/SECURITY.md +0 -88
- package/THIRD_PARTY_NOTICES.md +0 -100
package/core/events.py
CHANGED
|
@@ -1,61 +1,123 @@
|
|
|
1
|
-
"""
|
|
2
|
-
core/events.py
|
|
3
|
-
|
|
4
|
-
LoopEvent is the single event type yielded by RunAgentLoop._run() as it
|
|
5
|
-
progresses through the call-dispatch-repeat cycle. Exactly one field is
|
|
6
|
-
meaningfully populated per event, matching what happened at that point
|
|
7
|
-
in the loop.
|
|
8
|
-
|
|
9
|
-
Consumers (CLI, TUI, research pipeline) iterate the generator and react
|
|
10
|
-
to whichever field is set. run_to_completion() in core/loop.py drains
|
|
11
|
-
the generator for callers that only need the final ModelResponse.
|
|
12
|
-
|
|
13
|
-
Error handling: there is deliberately no error variant. Exceptions
|
|
14
|
-
raised mid-loop propagate naturally out of the generator, unchanged
|
|
15
|
-
from the pre-streaming behavior. orchestrator.py's failure-path
|
|
16
|
-
persistence (except Exception: update_pipeline_run(..., status="failed");
|
|
17
|
-
raise) depends on a real exception propagating -- converting failures
|
|
18
|
-
into data-shaped events would require every consumer to know to unpack
|
|
19
|
-
and re-raise. Consumers that want graceful error display (the TUI) wrap
|
|
20
|
-
their own consumption loop in try/except.
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
#
|
|
41
|
-
#
|
|
42
|
-
#
|
|
43
|
-
#
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
#
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
#
|
|
52
|
-
#
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
# §
|
|
58
|
-
# is
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
1
|
+
"""
|
|
2
|
+
core/events.py
|
|
3
|
+
|
|
4
|
+
LoopEvent is the single event type yielded by RunAgentLoop._run() as it
|
|
5
|
+
progresses through the call-dispatch-repeat cycle. Exactly one field is
|
|
6
|
+
meaningfully populated per event, matching what happened at that point
|
|
7
|
+
in the loop.
|
|
8
|
+
|
|
9
|
+
Consumers (CLI, TUI, research pipeline) iterate the generator and react
|
|
10
|
+
to whichever field is set. run_to_completion() in core/loop.py drains
|
|
11
|
+
the generator for callers that only need the final ModelResponse.
|
|
12
|
+
|
|
13
|
+
Error handling: there is deliberately no error variant. Exceptions
|
|
14
|
+
raised mid-loop propagate naturally out of the generator, unchanged
|
|
15
|
+
from the pre-streaming behavior. orchestrator.py's failure-path
|
|
16
|
+
persistence (except Exception: update_pipeline_run(..., status="failed");
|
|
17
|
+
raise) depends on a real exception propagating -- converting failures
|
|
18
|
+
into data-shaped events would require every consumer to know to unpack
|
|
19
|
+
and re-raise. Consumers that want graceful error display (the TUI) wrap
|
|
20
|
+
their own consumption loop in try/except.
|
|
21
|
+
|
|
22
|
+
`retract` is NOT an error variant, and batch 113 added it without
|
|
23
|
+
reopening that decision. The rule above is about where a failure is
|
|
24
|
+
HANDLED: a failure must stay an exception so that the persistence which
|
|
25
|
+
depends on one still fires. A retraction says nothing about whether the
|
|
26
|
+
call failed for good -- it says the rows drawn so far are being replaced,
|
|
27
|
+
and the exception still propagates, unchanged, when the retries run out.
|
|
28
|
+
The two coexist because they answer different questions, and a consumer
|
|
29
|
+
that ignores `retract` entirely is still correct about failures.
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from dataclasses import dataclass
|
|
33
|
+
from typing import Any, Optional
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass
|
|
37
|
+
class LoopEvent:
|
|
38
|
+
"""One event yielded by RunAgentLoop._run()."""
|
|
39
|
+
token_delta: Optional[str] = None
|
|
40
|
+
# ROADMAP_v2 §38 (O4) -- the EIGHTH field, and P1 re-made rather than
|
|
41
|
+
# widened by accretion. P1 rejected putting §22's and §23's kinds here
|
|
42
|
+
# because those describe a ten-pass RUN and live for the run; a
|
|
43
|
+
# thinking delta describes one model call's progress and lives for a
|
|
44
|
+
# turn, which is token_delta's family exactly. So this is the case P1's
|
|
45
|
+
# rule was never about, and the ninth field still has to argue for
|
|
46
|
+
# itself in tests/test_pipeline_events.py.
|
|
47
|
+
#
|
|
48
|
+
# Reasoning text, streamed as it arrives, for the shells that want to
|
|
49
|
+
# show it. Display-only: it is not persisted, not part of
|
|
50
|
+
# ModelResponse.text, and never sent back on the wire. Empty on
|
|
51
|
+
# providers that return no reasoning (OPENAI, GOOGLE) -- see
|
|
52
|
+
# StreamToken's note.
|
|
53
|
+
thinking_delta: Optional[str] = None
|
|
54
|
+
tool_call_start: Optional[dict] = None # {"id", "name", "input"}
|
|
55
|
+
tool_result: Optional[dict] = None # {"id", "result"}
|
|
56
|
+
permission_request: Optional[dict] = None # {"tool_name", "params"}
|
|
57
|
+
# ROADMAP_v2 §21: {"kind", "text"} -- a compaction that happened, an
|
|
58
|
+
# early warning that one is coming, or a compaction that could not
|
|
59
|
+
# help. §21's "no silent compaction, ever": both automatic and manual
|
|
60
|
+
# compaction show a marker inline.
|
|
61
|
+
#
|
|
62
|
+
# ALSO carried on the ModelResponse, because run_to_completion()
|
|
63
|
+
# discards every non-final event -- so a notice delivered only here
|
|
64
|
+
# would be invisible to the pipeline, which drains the generator
|
|
65
|
+
# rather than watching it. That is the same defect §20 and §25 each hit
|
|
66
|
+
# once; the event is for live display, the response field is for
|
|
67
|
+
# everyone else.
|
|
68
|
+
#
|
|
69
|
+
# The CLI drained too until ROADMAP_v3 §86 (A81). It now hands
|
|
70
|
+
# run_to_completion() an observer, prints this event when it arrives,
|
|
71
|
+
# and prints from the response only a notice that never arrived. That
|
|
72
|
+
# is also how a TOOL's notice (§23 J10) reaches it: those ride only
|
|
73
|
+
# this event, never the response.
|
|
74
|
+
notice: Optional[dict] = None
|
|
75
|
+
# Batch 113 (TECHNICAL_DEBT 25). THE NINTH FIELD, and it argues for
|
|
76
|
+
# itself where test_pipeline_events.py says it must. This is not a
|
|
77
|
+
# notice ABOUT the run -- it is an instruction to a drawing surface:
|
|
78
|
+
# take back everything you have drawn since the last flush, because
|
|
79
|
+
# the model call that produced it failed and is being tried again.
|
|
80
|
+
# thinking_delta's family exactly, and P1's rule was never about this
|
|
81
|
+
# case either: it describes one model call's progress and lives for a
|
|
82
|
+
# turn.
|
|
83
|
+
#
|
|
84
|
+
# {"text", "attempt", "attempts"} -- the sentence to show, and the
|
|
85
|
+
# numbers behind it for a consumer that would rather phrase its own.
|
|
86
|
+
#
|
|
87
|
+
# DISPLAY-ONLY, and unlike `notice` it is deliberately NOT mirrored
|
|
88
|
+
# onto ModelResponse. A consumer that drains has drawn nothing and so
|
|
89
|
+
# has nothing to retract; handing it a retraction in the response
|
|
90
|
+
# would be asking it to undo something that never happened. A
|
|
91
|
+
# consumer with no screen correctly ignores this field, which is why
|
|
92
|
+
# the research pipeline translates it into nothing at all.
|
|
93
|
+
#
|
|
94
|
+
# The CLI (ROADMAP_v3 §86, A81) has a screen and still draws nothing
|
|
95
|
+
# a retraction could reach: it prints a step only once the step has
|
|
96
|
+
# finished, which is after the last moment one can arrive. It honours
|
|
97
|
+
# this field by dropping what it was holding.
|
|
98
|
+
retract: Optional[dict] = None
|
|
99
|
+
final_response: Optional[Any] = None # ModelResponse on terminal event
|
|
100
|
+
# "complete" | "max_steps_reached" | "token_budget_exceeded" |
|
|
101
|
+
# "cancelled"
|
|
102
|
+
#
|
|
103
|
+
# ROADMAP_v3 §61 (CN3) added the fourth, and added NO field: this
|
|
104
|
+
# comment is the only place the set is written down, so a value
|
|
105
|
+
# missing from here is a reader's silent gap -- which is why
|
|
106
|
+
# tests/test_cancel.py asserts all four appear in this file.
|
|
107
|
+
#
|
|
108
|
+
# "cancelled" is NEVER "failed". A failed run is recorded on the
|
|
109
|
+
# thread by mark_turn_failed and reads as something broken; a
|
|
110
|
+
# cancelled one is a person changing their mind. The loop RETURNS on
|
|
111
|
+
# it rather than raising, for the reason the module docstring above
|
|
112
|
+
# gives about exceptions -- an exception here would reach _run's
|
|
113
|
+
# frame and stamp a failure on a turn nobody saw fail, which is the
|
|
114
|
+
# same argument that frame already makes for GeneratorExit.
|
|
115
|
+
#
|
|
116
|
+
# A CONSUMER THAT TREATS EVERY NON-"complete" VALUE AS TRUNCATION IS
|
|
117
|
+
# NOW WRONG, and there is one: core/reasoning/orchestrator.py's
|
|
118
|
+
# _check_truncation continues the pipeline on a truncated pass with
|
|
119
|
+
# text. No research pass can return "cancelled" today (§61 CN12
|
|
120
|
+
# defers the pipeline deliberately), so that reader is correct as it
|
|
121
|
+
# stands -- and it has to be taught the difference before the
|
|
122
|
+
# deferral is lifted.
|
|
123
|
+
stop_reason: Optional[str] = None
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
"""Project path completion and mention grammar (ROADMAP_v3 §54).
|
|
2
|
+
|
|
3
|
+
Only names are indexed. No file contents or access decisions belong here.
|
|
4
|
+
Git is optional; its ignore semantics are used when available. Snapshots are
|
|
5
|
+
bounded and replaced by a worker, never walked by the keystroke handler.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import shutil
|
|
14
|
+
|
|
15
|
+
# Optional git listing uses fixed argv and no shell.
|
|
16
|
+
import subprocess # nosec B404
|
|
17
|
+
import threading
|
|
18
|
+
import time
|
|
19
|
+
import weakref
|
|
20
|
+
from collections import deque
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
|
|
23
|
+
INDEX_LIMIT = 50_000
|
|
24
|
+
MATCH_LIMIT = 200
|
|
25
|
+
SCAN_SECONDS = 5
|
|
26
|
+
GIT_BYTE_LIMIT = 16 * 1024 * 1024
|
|
27
|
+
SKIP_DIRS = frozenset({
|
|
28
|
+
"__pycache__", ".venv", "venv", "node_modules", ".pytest_cache",
|
|
29
|
+
".mypy_cache", ".ruff_cache", "dist", "build",
|
|
30
|
+
})
|
|
31
|
+
_START = re.compile(r'(?<![\w@/\\])@(?=\S|$)')
|
|
32
|
+
_BARE = re.compile(r'[\w./\\:-]+', re.UNICODE)
|
|
33
|
+
_indexes: weakref.WeakSet = weakref.WeakSet()
|
|
34
|
+
_indexes_lock = threading.Lock()
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
@dataclass(frozen=True)
|
|
38
|
+
class Mention:
|
|
39
|
+
start: int
|
|
40
|
+
end: int
|
|
41
|
+
path: str
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def mentions(text: str, *, incomplete: bool = False) -> list[Mention]:
|
|
45
|
+
"""Parse standalone @paths and JSON-quoted @\"paths with spaces\".
|
|
46
|
+
|
|
47
|
+
Bare email addresses and package scopes embedded in a path do not
|
|
48
|
+
trigger. Quoting also permits punctuation in a filename, unambiguously.
|
|
49
|
+
The incomplete form is used only by completion, never by the prompt.
|
|
50
|
+
"""
|
|
51
|
+
found = []
|
|
52
|
+
consumed = 0
|
|
53
|
+
for match in _START.finditer(text):
|
|
54
|
+
if match.start() < consumed:
|
|
55
|
+
continue
|
|
56
|
+
begin = match.end()
|
|
57
|
+
if text[begin:begin + 1] == '"':
|
|
58
|
+
try:
|
|
59
|
+
path, length = json.JSONDecoder().raw_decode(text[begin:])
|
|
60
|
+
end = begin + length
|
|
61
|
+
except ValueError:
|
|
62
|
+
if not incomplete:
|
|
63
|
+
continue
|
|
64
|
+
end = text.find("\n", begin)
|
|
65
|
+
end = len(text) if end < 0 else end
|
|
66
|
+
path = text[begin + 1:end]
|
|
67
|
+
else:
|
|
68
|
+
token = _BARE.match(text, begin)
|
|
69
|
+
end = token.end() if token else begin
|
|
70
|
+
path = text[begin:end]
|
|
71
|
+
if isinstance(path, str) and (path or incomplete):
|
|
72
|
+
found.append(Mention(match.start(), end, path))
|
|
73
|
+
consumed = end
|
|
74
|
+
return found
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def active_mention(text: str, cursor: int) -> Mention | None:
|
|
78
|
+
for item in mentions(text, incomplete=True):
|
|
79
|
+
if item.start < cursor <= item.end:
|
|
80
|
+
return item
|
|
81
|
+
return None
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def quote_path(path: str) -> str:
|
|
85
|
+
return path if _BARE.fullmatch(path) else json.dumps(path, ensure_ascii=False)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass(frozen=True)
|
|
89
|
+
class FileSuggestion:
|
|
90
|
+
path: str
|
|
91
|
+
prefix: str = "@"
|
|
92
|
+
usage: str | None = None
|
|
93
|
+
|
|
94
|
+
@property
|
|
95
|
+
def name(self) -> str:
|
|
96
|
+
return quote_path(self.path)
|
|
97
|
+
|
|
98
|
+
@property
|
|
99
|
+
def summary(self) -> str:
|
|
100
|
+
return "folder" if self.path.endswith("/") else "file"
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@dataclass(frozen=True)
|
|
104
|
+
class Snapshot:
|
|
105
|
+
visible: tuple[str, ...] = ()
|
|
106
|
+
ignored: tuple[str, ...] = ()
|
|
107
|
+
source: str = "indexing"
|
|
108
|
+
limited: bool = False
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _git_paths(root: str, ignored: bool):
|
|
112
|
+
"""Stream bounded NUL records; fixed argv, no shell or fsmonitor hook."""
|
|
113
|
+
git = shutil.which("git")
|
|
114
|
+
if not git:
|
|
115
|
+
raise OSError("git unavailable")
|
|
116
|
+
args = [git, "-c", "core.fsmonitor=false", "ls-files", "-z",
|
|
117
|
+
"--others", "--exclude-standard"]
|
|
118
|
+
args += ["--ignored"] if ignored else ["--cached"]
|
|
119
|
+
# B603: executable from PATH, fixed options; root is cwd, never argv.
|
|
120
|
+
process = subprocess.Popen( # nosec B603
|
|
121
|
+
args, cwd=root, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
|
|
122
|
+
expired = threading.Event()
|
|
123
|
+
|
|
124
|
+
def stop():
|
|
125
|
+
expired.set()
|
|
126
|
+
try:
|
|
127
|
+
process.kill()
|
|
128
|
+
except OSError: # process exited as the timer fired
|
|
129
|
+
pass
|
|
130
|
+
|
|
131
|
+
timer = threading.Timer(SCAN_SECONDS, stop)
|
|
132
|
+
timer.daemon = True
|
|
133
|
+
timer.start()
|
|
134
|
+
try:
|
|
135
|
+
pending = b""
|
|
136
|
+
received = 0
|
|
137
|
+
while chunk := process.stdout.read1(65536):
|
|
138
|
+
received += len(chunk)
|
|
139
|
+
if received > GIT_BYTE_LIMIT:
|
|
140
|
+
raise TimeoutError("git listing byte limit")
|
|
141
|
+
records = (pending + chunk).split(b"\0")
|
|
142
|
+
pending = records.pop()
|
|
143
|
+
for raw in records:
|
|
144
|
+
yield os.fsdecode(raw)
|
|
145
|
+
if process.wait() or expired.is_set():
|
|
146
|
+
raise OSError("git listing failed or timed out")
|
|
147
|
+
finally:
|
|
148
|
+
timer.cancel()
|
|
149
|
+
if process.poll() is None:
|
|
150
|
+
process.kill()
|
|
151
|
+
process.wait()
|
|
152
|
+
process.stdout.close()
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _safe_name(path: str) -> bool:
|
|
156
|
+
return (bool(path) and not os.path.isabs(path)
|
|
157
|
+
and not any(part in ("..", ".git") for part in path.split("/"))
|
|
158
|
+
and not any(ord(char) < 32 or 127 <= ord(char) < 160
|
|
159
|
+
or 0xD800 <= ord(char) < 0xE000 for char in path))
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def scan(root: str) -> Snapshot:
|
|
163
|
+
visible, ignored = set(), set()
|
|
164
|
+
limited = False
|
|
165
|
+
|
|
166
|
+
def add(path, hidden):
|
|
167
|
+
nonlocal limited
|
|
168
|
+
path = path.replace(os.sep, "/")
|
|
169
|
+
if not _safe_name(path):
|
|
170
|
+
return
|
|
171
|
+
target = ignored if hidden else visible
|
|
172
|
+
parts = path.rstrip("/").split("/")
|
|
173
|
+
entries = ["/".join(parts[:n]) + "/" for n in range(1, len(parts))]
|
|
174
|
+
entries.append(path)
|
|
175
|
+
for entry in entries:
|
|
176
|
+
if entry in visible or entry in ignored:
|
|
177
|
+
continue
|
|
178
|
+
if len(visible) + len(ignored) >= INDEX_LIMIT:
|
|
179
|
+
limited = True
|
|
180
|
+
return
|
|
181
|
+
target.add(entry)
|
|
182
|
+
|
|
183
|
+
try:
|
|
184
|
+
for hidden in (False, True):
|
|
185
|
+
listing = _git_paths(root, hidden)
|
|
186
|
+
try:
|
|
187
|
+
for path in listing:
|
|
188
|
+
add(path, hidden)
|
|
189
|
+
if limited:
|
|
190
|
+
break
|
|
191
|
+
finally:
|
|
192
|
+
listing.close()
|
|
193
|
+
if limited:
|
|
194
|
+
break
|
|
195
|
+
source = "git ignores"
|
|
196
|
+
except (OSError, TimeoutError):
|
|
197
|
+
visible.clear()
|
|
198
|
+
ignored.clear()
|
|
199
|
+
limited = False
|
|
200
|
+
source = "fallback skips"
|
|
201
|
+
pending = deque([(root, False)])
|
|
202
|
+
deadline = time.monotonic() + SCAN_SECONDS
|
|
203
|
+
visited = 0
|
|
204
|
+
while pending and not limited:
|
|
205
|
+
directory, hidden = pending.popleft()
|
|
206
|
+
try:
|
|
207
|
+
with os.scandir(directory) as entries:
|
|
208
|
+
for entry in entries:
|
|
209
|
+
visited += 1
|
|
210
|
+
if visited > INDEX_LIMIT or time.monotonic() >= deadline:
|
|
211
|
+
limited = True
|
|
212
|
+
break
|
|
213
|
+
if entry.name.casefold() == ".git":
|
|
214
|
+
continue
|
|
215
|
+
is_dir = entry.is_dir(follow_symlinks=False)
|
|
216
|
+
child_hidden = hidden or (is_dir and entry.name in SKIP_DIRS)
|
|
217
|
+
rel = os.path.relpath(entry.path, root).replace(os.sep, "/")
|
|
218
|
+
add(rel + ("/" if is_dir else ""), child_hidden)
|
|
219
|
+
if is_dir:
|
|
220
|
+
pending.append((entry.path, child_hidden))
|
|
221
|
+
if limited:
|
|
222
|
+
break
|
|
223
|
+
except OSError:
|
|
224
|
+
# An unreadable subtree makes the snapshot partial, visibly.
|
|
225
|
+
limited = True
|
|
226
|
+
return Snapshot(tuple(sorted(visible)), tuple(sorted(ignored)), source, limited)
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
class FileIndex:
|
|
230
|
+
def __init__(self, root: str | None):
|
|
231
|
+
self.root = os.path.realpath(root) if root else None
|
|
232
|
+
self.snapshot = Snapshot(source="indexing" if root else "no project")
|
|
233
|
+
self.show_ignored = False
|
|
234
|
+
self.dirty = threading.Event()
|
|
235
|
+
if root:
|
|
236
|
+
self.dirty.set()
|
|
237
|
+
with _indexes_lock:
|
|
238
|
+
_indexes.add(self)
|
|
239
|
+
|
|
240
|
+
def refresh(self):
|
|
241
|
+
self.dirty.clear()
|
|
242
|
+
if self.root:
|
|
243
|
+
self.snapshot = scan(self.root)
|
|
244
|
+
|
|
245
|
+
def matching(self, query: str) -> tuple[list[FileSuggestion], str]:
|
|
246
|
+
snapshot = self.snapshot
|
|
247
|
+
query = query.replace("\\", "/").casefold()
|
|
248
|
+
paths = snapshot.visible + (snapshot.ignored if self.show_ignored else ())
|
|
249
|
+
# Keep bounded prefix/substring buckets instead of sorting 50,000
|
|
250
|
+
# rows at every keystroke. Snapshot order breaks ties predictably.
|
|
251
|
+
prefixes, others = [], []
|
|
252
|
+
total = 0
|
|
253
|
+
for path in paths:
|
|
254
|
+
folded = path.casefold()
|
|
255
|
+
if query not in folded:
|
|
256
|
+
continue
|
|
257
|
+
total += 1
|
|
258
|
+
starts = folded.startswith(query) or folded.rstrip("/").rsplit("/", 1)[-1].startswith(query)
|
|
259
|
+
bucket = prefixes if starts else others
|
|
260
|
+
if len(bucket) < MATCH_LIMIT:
|
|
261
|
+
bucket.append(FileSuggestion(path))
|
|
262
|
+
notices = []
|
|
263
|
+
if total > MATCH_LIMIT:
|
|
264
|
+
notices.append(f"first {MATCH_LIMIT} of {total} matches")
|
|
265
|
+
if snapshot.limited:
|
|
266
|
+
notices.append(f"partial index (limit {INDEX_LIMIT:,})")
|
|
267
|
+
notices += [snapshot.source, f"ignored {'shown' if self.show_ignored else 'hidden'}"]
|
|
268
|
+
return (prefixes + others)[:MATCH_LIMIT], " · ".join(notices)
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def note_file_created(path: str) -> None:
|
|
272
|
+
"""Called only after write succeeds; subagents use the same producer."""
|
|
273
|
+
with _indexes_lock:
|
|
274
|
+
indexes = list(_indexes)
|
|
275
|
+
for index in indexes:
|
|
276
|
+
try:
|
|
277
|
+
if os.path.commonpath((index.root, path)) == index.root:
|
|
278
|
+
index.dirty.set()
|
|
279
|
+
except ValueError: # different drives
|
|
280
|
+
continue
|