@softspark/ai-toolkit 4.16.0 → 4.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +41 -0
- package/README.md +11 -16
- package/app/.claude-plugin/plugin.json +1 -1
- package/app/hooks/session-end.sh +1 -13
- package/app/hooks.json +0 -10
- package/benchmarks/ecosystem-doctor-snapshot.json +14 -15
- package/bin/ai-toolkit.js +0 -2
- package/kb/history/completed/output-filter-retirement-20260726.md +128 -0
- package/kb/reference/architecture-overview.md +2 -3
- package/kb/reference/cli-reference.md +3 -13
- package/kb/reference/enterprise-config-guide.md +1 -21
- package/kb/reference/hooks-catalog.md +3 -60
- package/kb/reference/supported-tools-registry.md +0 -4
- package/llms-full.txt +143 -395
- package/llms.txt +1 -1
- package/manifest.json +147 -36
- package/package.json +1 -2
- package/scripts/claude_app.py +2 -21
- package/scripts/config_cli.py +4 -0
- package/scripts/config_merger.py +0 -17
- package/scripts/config_validator.py +11 -138
- package/scripts/doctor.py +3 -20
- package/scripts/generate_copilot.py +35 -4
- package/scripts/install.py +7 -2
- package/scripts/install_steps/ai_tools.py +28 -99
- package/scripts/install_steps/hooks.py +26 -24
- package/scripts/merge-hooks.py +33 -2
- package/scripts/output_filter_retirement.py +395 -0
- package/scripts/schemas/ai-toolkit-config.schema.json +0 -60
- package/scripts/uninstall.py +13 -27
- package/app/hooks/filter-tool-output.sh +0 -76
- package/app/output-filter-policy.json +0 -15
- package/benchmarks/output-filter/README.md +0 -11
- package/benchmarks/output-filter/scenarios.json +0 -25
- package/kb/reference/tool-output-filter.md +0 -288
- package/scripts/benchmark_output_filter.py +0 -343
- package/scripts/output_filter_cli.py +0 -347
- package/scripts/output_filter_hook.py +0 -23
- package/scripts/tool_output_filter/__init__.py +0 -33
- package/scripts/tool_output_filter/contracts.py +0 -173
- package/scripts/tool_output_filter/engine.py +0 -260
- package/scripts/tool_output_filter/hook_runtime.py +0 -369
- package/scripts/tool_output_filter/input.py +0 -56
- package/scripts/tool_output_filter/invariants.py +0 -40
- package/scripts/tool_output_filter/policy.py +0 -153
- package/scripts/tool_output_filter/profiles/__init__.py +0 -68
- package/scripts/tool_output_filter/profiles/repeat_lines.py +0 -71
- package/scripts/tool_output_filter/profiles/tap_success.py +0 -154
- package/scripts/tool_output_filter/recovery.py +0 -846
- package/scripts/tool_output_filter/telemetry.py +0 -13
|
@@ -1,846 +0,0 @@
|
|
|
1
|
-
"""Recovery-store boundary for lossless safe-mode replacements."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import hashlib
|
|
6
|
-
import json
|
|
7
|
-
import os
|
|
8
|
-
import stat
|
|
9
|
-
import time
|
|
10
|
-
from collections.abc import Callable
|
|
11
|
-
|
|
12
|
-
from .contracts import FilterTelemetry
|
|
13
|
-
from .invariants import DEFAULT_FAILURE_LIMIT
|
|
14
|
-
from .telemetry import TelemetrySink
|
|
15
|
-
|
|
16
|
-
_DIRECTORY_FLAGS = (
|
|
17
|
-
os.O_RDONLY
|
|
18
|
-
| getattr(os, "O_CLOEXEC", 0)
|
|
19
|
-
| getattr(os, "O_DIRECTORY", 0)
|
|
20
|
-
| getattr(os, "O_NOFOLLOW", 0)
|
|
21
|
-
)
|
|
22
|
-
_READ_FLAGS = (
|
|
23
|
-
os.O_RDONLY
|
|
24
|
-
| getattr(os, "O_CLOEXEC", 0)
|
|
25
|
-
| getattr(os, "O_NOFOLLOW", 0)
|
|
26
|
-
)
|
|
27
|
-
_WRITE_FLAGS = (
|
|
28
|
-
os.O_WRONLY
|
|
29
|
-
| os.O_CREAT
|
|
30
|
-
| os.O_EXCL
|
|
31
|
-
| getattr(os, "O_CLOEXEC", 0)
|
|
32
|
-
| getattr(os, "O_NOFOLLOW", 0)
|
|
33
|
-
)
|
|
34
|
-
_APPEND_FLAGS = (
|
|
35
|
-
os.O_WRONLY
|
|
36
|
-
| os.O_CREAT
|
|
37
|
-
| os.O_APPEND
|
|
38
|
-
| getattr(os, "O_CLOEXEC", 0)
|
|
39
|
-
| getattr(os, "O_NOFOLLOW", 0)
|
|
40
|
-
)
|
|
41
|
-
_HANDLE_LENGTH = 32
|
|
42
|
-
_CIRCUIT_STATE_NAME = ".circuit-state.json"
|
|
43
|
-
_TELEMETRY_NAME = ".telemetry.jsonl"
|
|
44
|
-
_MAX_TELEMETRY_BYTES = 1024 * 1024
|
|
45
|
-
DEFAULT_MAX_SESSION_BYTES = 32 * 1024 * 1024
|
|
46
|
-
DEFAULT_TTL_MINUTES = 60
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
def _secure_hex(byte_count: int) -> str:
|
|
50
|
-
return os.urandom(byte_count).hex()
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
class RecoveryUnavailableError(RuntimeError):
|
|
54
|
-
"""Secure recovery cannot operate on this platform or path."""
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
class RecoveryStore:
|
|
58
|
-
"""Storage contract implemented by the runtime integration."""
|
|
59
|
-
|
|
60
|
-
def save(self, response: object) -> str:
|
|
61
|
-
"""Persist the exact native response object and return an opaque handle."""
|
|
62
|
-
raise NotImplementedError
|
|
63
|
-
|
|
64
|
-
def load(self, handle: str) -> object | None:
|
|
65
|
-
"""Load the exact response object for verification or recovery."""
|
|
66
|
-
raise NotImplementedError
|
|
67
|
-
|
|
68
|
-
def delete(self, handle: str) -> None:
|
|
69
|
-
"""Delete an unused or expired recovery artifact."""
|
|
70
|
-
raise NotImplementedError
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
def _supports_secure_recovery() -> bool:
|
|
74
|
-
# os.rename is the documented supports_dir_fd member for renameat-backed
|
|
75
|
-
# calls; os.replace shares that implementation.
|
|
76
|
-
required = (
|
|
77
|
-
os.open,
|
|
78
|
-
os.mkdir,
|
|
79
|
-
os.stat,
|
|
80
|
-
os.unlink,
|
|
81
|
-
os.link,
|
|
82
|
-
os.rename,
|
|
83
|
-
os.rmdir,
|
|
84
|
-
)
|
|
85
|
-
return (
|
|
86
|
-
hasattr(os, "O_DIRECTORY")
|
|
87
|
-
and hasattr(os, "O_NOFOLLOW")
|
|
88
|
-
and hasattr(os, "fchmod")
|
|
89
|
-
and hasattr(os, "getuid")
|
|
90
|
-
and os.listdir in os.supports_fd
|
|
91
|
-
and all(function in os.supports_dir_fd for function in required)
|
|
92
|
-
)
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
def _verify_private_directory(directory_fd: int, label: str) -> None:
|
|
96
|
-
metadata = os.fstat(directory_fd)
|
|
97
|
-
if not stat.S_ISDIR(metadata.st_mode):
|
|
98
|
-
raise RecoveryUnavailableError(f"{label} is not a directory")
|
|
99
|
-
if stat.S_IMODE(metadata.st_mode) & 0o077:
|
|
100
|
-
raise RecoveryUnavailableError(f"{label} is not private")
|
|
101
|
-
if metadata.st_uid != os.getuid():
|
|
102
|
-
raise RecoveryUnavailableError(f"{label} is not owned by this user")
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
def _open_base_directory(path: str | os.PathLike[str]) -> int:
|
|
106
|
-
try:
|
|
107
|
-
directory_fd = os.open(path, _DIRECTORY_FLAGS)
|
|
108
|
-
except OSError as error:
|
|
109
|
-
raise RecoveryUnavailableError(
|
|
110
|
-
f"recovery base is unavailable: {error}"
|
|
111
|
-
) from error
|
|
112
|
-
metadata = os.fstat(directory_fd)
|
|
113
|
-
if not stat.S_ISDIR(metadata.st_mode):
|
|
114
|
-
os.close(directory_fd)
|
|
115
|
-
raise RecoveryUnavailableError("recovery base is not a directory")
|
|
116
|
-
return directory_fd
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
def _open_private_child(parent_fd: int, name: str) -> int:
|
|
120
|
-
try:
|
|
121
|
-
os.mkdir(name, mode=0o700, dir_fd=parent_fd)
|
|
122
|
-
except FileExistsError:
|
|
123
|
-
pass
|
|
124
|
-
try:
|
|
125
|
-
child_fd = os.open(name, _DIRECTORY_FLAGS, dir_fd=parent_fd)
|
|
126
|
-
except OSError as error:
|
|
127
|
-
raise RecoveryUnavailableError(
|
|
128
|
-
f"recovery directory is unsafe: {error}"
|
|
129
|
-
) from error
|
|
130
|
-
try:
|
|
131
|
-
_verify_private_directory(child_fd, "recovery directory")
|
|
132
|
-
except BaseException:
|
|
133
|
-
os.close(child_fd)
|
|
134
|
-
raise
|
|
135
|
-
return child_fd
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
def _validate_handle(handle: str) -> str:
|
|
139
|
-
if len(handle) != _HANDLE_LENGTH:
|
|
140
|
-
raise ValueError("invalid recovery handle")
|
|
141
|
-
if any(character not in "0123456789abcdef" for character in handle):
|
|
142
|
-
raise ValueError("invalid recovery handle")
|
|
143
|
-
return f"{handle}.json"
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
def _owned_artifact_name(name: str) -> bool:
|
|
147
|
-
if not name.endswith(".json"):
|
|
148
|
-
return False
|
|
149
|
-
handle = name[:-5]
|
|
150
|
-
try:
|
|
151
|
-
_validate_handle(handle)
|
|
152
|
-
except ValueError:
|
|
153
|
-
return False
|
|
154
|
-
return True
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
def _owned_pending_artifact_name(name: str) -> bool:
|
|
158
|
-
prefix = ".pending-"
|
|
159
|
-
token = name[len(prefix):] if name.startswith(prefix) else ""
|
|
160
|
-
return (
|
|
161
|
-
len(token) == _HANDLE_LENGTH
|
|
162
|
-
and all(character in "0123456789abcdef" for character in token)
|
|
163
|
-
)
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
def _owned_recovery_storage_name(name: str) -> bool:
|
|
167
|
-
return _owned_artifact_name(name) or _owned_pending_artifact_name(name)
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
def _owned_runtime_artifact_name(name: str) -> bool:
|
|
171
|
-
return (
|
|
172
|
-
_owned_recovery_storage_name(name)
|
|
173
|
-
or name in {_CIRCUIT_STATE_NAME, _TELEMETRY_NAME}
|
|
174
|
-
)
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
def _session_key(session_identifier: str) -> str:
|
|
178
|
-
return hashlib.sha256(
|
|
179
|
-
session_identifier.encode("utf-8")
|
|
180
|
-
).hexdigest()[:_HANDLE_LENGTH]
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
class EphemeralRecoveryStore(RecoveryStore, TelemetrySink):
|
|
184
|
-
"""Private, bounded session recovery backed by pinned directory fds."""
|
|
185
|
-
|
|
186
|
-
def __init__(
|
|
187
|
-
self,
|
|
188
|
-
base_directory: str | os.PathLike[str],
|
|
189
|
-
*,
|
|
190
|
-
session_identifier: str,
|
|
191
|
-
max_session_bytes: int = DEFAULT_MAX_SESSION_BYTES,
|
|
192
|
-
ttl_minutes: int = DEFAULT_TTL_MINUTES,
|
|
193
|
-
clock: Callable[[], float] = time.time,
|
|
194
|
-
random_handle: Callable[[], str] | None = None,
|
|
195
|
-
) -> None:
|
|
196
|
-
if not _supports_secure_recovery():
|
|
197
|
-
raise RecoveryUnavailableError(
|
|
198
|
-
"secure recovery requires dir_fd and O_NOFOLLOW"
|
|
199
|
-
)
|
|
200
|
-
self._random_handle = random_handle or (
|
|
201
|
-
lambda: _secure_hex(_HANDLE_LENGTH // 2)
|
|
202
|
-
)
|
|
203
|
-
self._max_session_bytes = max_session_bytes
|
|
204
|
-
self._ttl_seconds = ttl_minutes * 60
|
|
205
|
-
self._clock = clock
|
|
206
|
-
base_fd = _open_base_directory(base_directory)
|
|
207
|
-
output_fd = -1
|
|
208
|
-
try:
|
|
209
|
-
output_fd = _open_private_child(base_fd, "output-filter")
|
|
210
|
-
session_key = _session_key(session_identifier)
|
|
211
|
-
self._directory_fd = _open_private_child(output_fd, session_key)
|
|
212
|
-
finally:
|
|
213
|
-
if output_fd >= 0:
|
|
214
|
-
os.close(output_fd)
|
|
215
|
-
os.close(base_fd)
|
|
216
|
-
|
|
217
|
-
def _stored_bytes(self) -> int:
|
|
218
|
-
total = 0
|
|
219
|
-
for name in os.listdir(self._directory_fd):
|
|
220
|
-
if not _owned_recovery_storage_name(name):
|
|
221
|
-
continue
|
|
222
|
-
try:
|
|
223
|
-
metadata = os.stat(
|
|
224
|
-
name,
|
|
225
|
-
dir_fd=self._directory_fd,
|
|
226
|
-
follow_symlinks=False,
|
|
227
|
-
)
|
|
228
|
-
except FileNotFoundError:
|
|
229
|
-
# A concurrent session process removed the artifact between
|
|
230
|
-
# listdir and stat; that is not a safety failure.
|
|
231
|
-
continue
|
|
232
|
-
if not stat.S_ISREG(metadata.st_mode):
|
|
233
|
-
raise RecoveryUnavailableError(
|
|
234
|
-
"recovery artifact is not regular"
|
|
235
|
-
)
|
|
236
|
-
total += metadata.st_size
|
|
237
|
-
return total
|
|
238
|
-
|
|
239
|
-
def _write_atomic(self, filename: str, content: bytes) -> None:
|
|
240
|
-
temporary_name = f".pending-{_secure_hex(16)}"
|
|
241
|
-
file_fd = os.open(
|
|
242
|
-
temporary_name,
|
|
243
|
-
_WRITE_FLAGS,
|
|
244
|
-
0o600,
|
|
245
|
-
dir_fd=self._directory_fd,
|
|
246
|
-
)
|
|
247
|
-
try:
|
|
248
|
-
with os.fdopen(file_fd, "wb") as file_handle:
|
|
249
|
-
file_fd = -1
|
|
250
|
-
os.fchmod(file_handle.fileno(), 0o600)
|
|
251
|
-
file_handle.write(content)
|
|
252
|
-
file_handle.flush()
|
|
253
|
-
os.fsync(file_handle.fileno())
|
|
254
|
-
os.link(
|
|
255
|
-
temporary_name,
|
|
256
|
-
filename,
|
|
257
|
-
src_dir_fd=self._directory_fd,
|
|
258
|
-
dst_dir_fd=self._directory_fd,
|
|
259
|
-
follow_symlinks=False,
|
|
260
|
-
)
|
|
261
|
-
try:
|
|
262
|
-
os.fsync(self._directory_fd)
|
|
263
|
-
except OSError:
|
|
264
|
-
# The artifact is already published; retract it so a failed
|
|
265
|
-
# save never leaves an orphan counting against the quota.
|
|
266
|
-
try:
|
|
267
|
-
os.unlink(filename, dir_fd=self._directory_fd)
|
|
268
|
-
except FileNotFoundError:
|
|
269
|
-
pass
|
|
270
|
-
raise
|
|
271
|
-
finally:
|
|
272
|
-
if file_fd >= 0:
|
|
273
|
-
os.close(file_fd)
|
|
274
|
-
try:
|
|
275
|
-
os.unlink(temporary_name, dir_fd=self._directory_fd)
|
|
276
|
-
except FileNotFoundError:
|
|
277
|
-
pass
|
|
278
|
-
|
|
279
|
-
def _replace_atomic(self, filename: str, content: bytes) -> None:
|
|
280
|
-
temporary_name = f".pending-{_secure_hex(16)}"
|
|
281
|
-
file_fd = os.open(
|
|
282
|
-
temporary_name,
|
|
283
|
-
_WRITE_FLAGS,
|
|
284
|
-
0o600,
|
|
285
|
-
dir_fd=self._directory_fd,
|
|
286
|
-
)
|
|
287
|
-
try:
|
|
288
|
-
with os.fdopen(file_fd, "wb") as file_handle:
|
|
289
|
-
file_fd = -1
|
|
290
|
-
os.fchmod(file_handle.fileno(), 0o600)
|
|
291
|
-
file_handle.write(content)
|
|
292
|
-
file_handle.flush()
|
|
293
|
-
os.fsync(file_handle.fileno())
|
|
294
|
-
os.replace(
|
|
295
|
-
temporary_name,
|
|
296
|
-
filename,
|
|
297
|
-
src_dir_fd=self._directory_fd,
|
|
298
|
-
dst_dir_fd=self._directory_fd,
|
|
299
|
-
)
|
|
300
|
-
os.fsync(self._directory_fd)
|
|
301
|
-
finally:
|
|
302
|
-
if file_fd >= 0:
|
|
303
|
-
os.close(file_fd)
|
|
304
|
-
try:
|
|
305
|
-
os.unlink(temporary_name, dir_fd=self._directory_fd)
|
|
306
|
-
except FileNotFoundError:
|
|
307
|
-
pass
|
|
308
|
-
|
|
309
|
-
def save(self, response: object) -> str:
|
|
310
|
-
content = json.dumps(
|
|
311
|
-
response,
|
|
312
|
-
ensure_ascii=False,
|
|
313
|
-
separators=(",", ":"),
|
|
314
|
-
sort_keys=True,
|
|
315
|
-
).encode("utf-8")
|
|
316
|
-
self.clean_expired()
|
|
317
|
-
if self._stored_bytes() + len(content) > self._max_session_bytes:
|
|
318
|
-
raise RecoveryUnavailableError("recovery session quota exceeded")
|
|
319
|
-
for _ in range(5):
|
|
320
|
-
handle = self._random_handle()
|
|
321
|
-
filename = _validate_handle(handle)
|
|
322
|
-
try:
|
|
323
|
-
self._write_atomic(filename, content)
|
|
324
|
-
except FileExistsError:
|
|
325
|
-
continue
|
|
326
|
-
return handle
|
|
327
|
-
raise RecoveryUnavailableError("could not allocate recovery handle")
|
|
328
|
-
|
|
329
|
-
def clean_expired(self) -> int:
|
|
330
|
-
cutoff = self._clock() - self._ttl_seconds
|
|
331
|
-
removed = 0
|
|
332
|
-
for name in os.listdir(self._directory_fd):
|
|
333
|
-
if not _owned_recovery_storage_name(name):
|
|
334
|
-
continue
|
|
335
|
-
try:
|
|
336
|
-
metadata = os.stat(
|
|
337
|
-
name,
|
|
338
|
-
dir_fd=self._directory_fd,
|
|
339
|
-
follow_symlinks=False,
|
|
340
|
-
)
|
|
341
|
-
except FileNotFoundError:
|
|
342
|
-
continue
|
|
343
|
-
if not stat.S_ISREG(metadata.st_mode):
|
|
344
|
-
raise RecoveryUnavailableError(
|
|
345
|
-
"recovery artifact is not regular"
|
|
346
|
-
)
|
|
347
|
-
if metadata.st_mtime > cutoff:
|
|
348
|
-
continue
|
|
349
|
-
try:
|
|
350
|
-
os.unlink(name, dir_fd=self._directory_fd)
|
|
351
|
-
except FileNotFoundError:
|
|
352
|
-
continue
|
|
353
|
-
removed += 1
|
|
354
|
-
if removed:
|
|
355
|
-
os.fsync(self._directory_fd)
|
|
356
|
-
return removed
|
|
357
|
-
|
|
358
|
-
def clean_all(self) -> int:
|
|
359
|
-
return self.clean_recovery()
|
|
360
|
-
|
|
361
|
-
def clean_recovery(self) -> int:
|
|
362
|
-
names = []
|
|
363
|
-
for name in os.listdir(self._directory_fd):
|
|
364
|
-
if not _owned_runtime_artifact_name(name):
|
|
365
|
-
continue
|
|
366
|
-
try:
|
|
367
|
-
metadata = os.stat(
|
|
368
|
-
name,
|
|
369
|
-
dir_fd=self._directory_fd,
|
|
370
|
-
follow_symlinks=False,
|
|
371
|
-
)
|
|
372
|
-
except FileNotFoundError:
|
|
373
|
-
continue
|
|
374
|
-
if not stat.S_ISREG(metadata.st_mode):
|
|
375
|
-
raise RecoveryUnavailableError(
|
|
376
|
-
"recovery artifact is not regular"
|
|
377
|
-
)
|
|
378
|
-
names.append(name)
|
|
379
|
-
removed = 0
|
|
380
|
-
for name in names:
|
|
381
|
-
try:
|
|
382
|
-
os.unlink(name, dir_fd=self._directory_fd)
|
|
383
|
-
except FileNotFoundError:
|
|
384
|
-
continue
|
|
385
|
-
removed += 1
|
|
386
|
-
if removed:
|
|
387
|
-
os.fsync(self._directory_fd)
|
|
388
|
-
return removed
|
|
389
|
-
|
|
390
|
-
def save_failure_count(self, count: int) -> None:
|
|
391
|
-
if isinstance(count, bool) or not 0 <= count <= DEFAULT_FAILURE_LIMIT:
|
|
392
|
-
raise ValueError(
|
|
393
|
-
"failure count must be between zero and "
|
|
394
|
-
f"{DEFAULT_FAILURE_LIMIT}"
|
|
395
|
-
)
|
|
396
|
-
current_count, current_warning = self._load_failure_state()
|
|
397
|
-
warning_emitted = (
|
|
398
|
-
current_warning if count == DEFAULT_FAILURE_LIMIT else False
|
|
399
|
-
)
|
|
400
|
-
if (count, warning_emitted) == (current_count, current_warning):
|
|
401
|
-
return
|
|
402
|
-
content = json.dumps(
|
|
403
|
-
{
|
|
404
|
-
"consecutiveFailures": count,
|
|
405
|
-
"warningEmitted": warning_emitted,
|
|
406
|
-
},
|
|
407
|
-
separators=(",", ":"),
|
|
408
|
-
).encode("utf-8")
|
|
409
|
-
self._replace_atomic(_CIRCUIT_STATE_NAME, content)
|
|
410
|
-
|
|
411
|
-
def _load_circuit_state(self) -> dict[str, object]:
|
|
412
|
-
try:
|
|
413
|
-
metadata = os.stat(
|
|
414
|
-
_CIRCUIT_STATE_NAME,
|
|
415
|
-
dir_fd=self._directory_fd,
|
|
416
|
-
follow_symlinks=False,
|
|
417
|
-
)
|
|
418
|
-
except FileNotFoundError:
|
|
419
|
-
return {"consecutiveFailures": 0, "warningEmitted": False}
|
|
420
|
-
if not stat.S_ISREG(metadata.st_mode):
|
|
421
|
-
raise RecoveryUnavailableError("circuit state is not regular")
|
|
422
|
-
if stat.S_IMODE(metadata.st_mode) != 0o600:
|
|
423
|
-
raise RecoveryUnavailableError("circuit state is not private")
|
|
424
|
-
file_fd = os.open(
|
|
425
|
-
_CIRCUIT_STATE_NAME,
|
|
426
|
-
_READ_FLAGS,
|
|
427
|
-
dir_fd=self._directory_fd,
|
|
428
|
-
)
|
|
429
|
-
with os.fdopen(file_fd, "rb") as file_handle:
|
|
430
|
-
data = json.load(file_handle)
|
|
431
|
-
if not isinstance(data, dict):
|
|
432
|
-
raise RecoveryUnavailableError("circuit state is invalid")
|
|
433
|
-
return data
|
|
434
|
-
|
|
435
|
-
def _load_failure_state(self) -> tuple[int, bool]:
|
|
436
|
-
data = self._load_circuit_state()
|
|
437
|
-
count = data.get("consecutiveFailures")
|
|
438
|
-
if isinstance(count, bool) or not isinstance(count, int):
|
|
439
|
-
raise RecoveryUnavailableError("circuit state is invalid")
|
|
440
|
-
if not 0 <= count <= DEFAULT_FAILURE_LIMIT:
|
|
441
|
-
raise RecoveryUnavailableError("circuit state is invalid")
|
|
442
|
-
value = data.get("warningEmitted", False)
|
|
443
|
-
if not isinstance(value, bool):
|
|
444
|
-
raise RecoveryUnavailableError("circuit state is invalid")
|
|
445
|
-
return count, value
|
|
446
|
-
|
|
447
|
-
def load_failure_count(self) -> int:
|
|
448
|
-
return self._load_failure_state()[0]
|
|
449
|
-
|
|
450
|
-
def load_warning_emitted(self) -> bool:
|
|
451
|
-
return self._load_failure_state()[1]
|
|
452
|
-
|
|
453
|
-
def mark_warning_emitted(self) -> None:
|
|
454
|
-
count, warning_emitted = self._load_failure_state()
|
|
455
|
-
if warning_emitted:
|
|
456
|
-
return
|
|
457
|
-
content = json.dumps(
|
|
458
|
-
{
|
|
459
|
-
"consecutiveFailures": count,
|
|
460
|
-
"warningEmitted": True,
|
|
461
|
-
},
|
|
462
|
-
separators=(",", ":"),
|
|
463
|
-
).encode("utf-8")
|
|
464
|
-
self._replace_atomic(_CIRCUIT_STATE_NAME, content)
|
|
465
|
-
|
|
466
|
-
def record(self, event: FilterTelemetry) -> None:
|
|
467
|
-
payload = {
|
|
468
|
-
"profileId": event.profile_id,
|
|
469
|
-
"profileVersion": event.profile_version,
|
|
470
|
-
"inputBytes": event.input_bytes,
|
|
471
|
-
"outputBytes": event.output_bytes,
|
|
472
|
-
"inputLines": event.input_lines,
|
|
473
|
-
"outputLines": event.output_lines,
|
|
474
|
-
"durationMs": round(event.duration_ms, 3),
|
|
475
|
-
"outcome": event.outcome,
|
|
476
|
-
"fallbackReason": event.fallback_reason,
|
|
477
|
-
}
|
|
478
|
-
line = (
|
|
479
|
-
json.dumps(payload, ensure_ascii=True, separators=(",", ":"))
|
|
480
|
-
+ "\n"
|
|
481
|
-
).encode("ascii")
|
|
482
|
-
try:
|
|
483
|
-
metadata = os.stat(
|
|
484
|
-
_TELEMETRY_NAME,
|
|
485
|
-
dir_fd=self._directory_fd,
|
|
486
|
-
follow_symlinks=False,
|
|
487
|
-
)
|
|
488
|
-
except FileNotFoundError:
|
|
489
|
-
metadata = None
|
|
490
|
-
if metadata is not None:
|
|
491
|
-
if not stat.S_ISREG(metadata.st_mode):
|
|
492
|
-
raise RecoveryUnavailableError("telemetry is not regular")
|
|
493
|
-
if stat.S_IMODE(metadata.st_mode) != 0o600:
|
|
494
|
-
raise RecoveryUnavailableError("telemetry is not private")
|
|
495
|
-
if metadata.st_size + len(line) > _MAX_TELEMETRY_BYTES:
|
|
496
|
-
return
|
|
497
|
-
file_fd = os.open(
|
|
498
|
-
_TELEMETRY_NAME,
|
|
499
|
-
_APPEND_FLAGS,
|
|
500
|
-
0o600,
|
|
501
|
-
dir_fd=self._directory_fd,
|
|
502
|
-
)
|
|
503
|
-
try:
|
|
504
|
-
os.fchmod(file_fd, 0o600)
|
|
505
|
-
if os.write(file_fd, line) != len(line):
|
|
506
|
-
raise OSError("partial telemetry write")
|
|
507
|
-
finally:
|
|
508
|
-
os.close(file_fd)
|
|
509
|
-
|
|
510
|
-
def load(self, handle: str) -> object | None:
|
|
511
|
-
filename = _validate_handle(handle)
|
|
512
|
-
try:
|
|
513
|
-
metadata = os.stat(
|
|
514
|
-
filename,
|
|
515
|
-
dir_fd=self._directory_fd,
|
|
516
|
-
follow_symlinks=False,
|
|
517
|
-
)
|
|
518
|
-
except FileNotFoundError:
|
|
519
|
-
return None
|
|
520
|
-
if not stat.S_ISREG(metadata.st_mode):
|
|
521
|
-
raise RecoveryUnavailableError("recovery artifact is not regular")
|
|
522
|
-
if stat.S_IMODE(metadata.st_mode) != 0o600:
|
|
523
|
-
raise RecoveryUnavailableError("recovery artifact is not private")
|
|
524
|
-
file_fd = os.open(filename, _READ_FLAGS, dir_fd=self._directory_fd)
|
|
525
|
-
with os.fdopen(file_fd, "rb") as file_handle:
|
|
526
|
-
content = file_handle.read()
|
|
527
|
-
return json.loads(content.decode("utf-8"))
|
|
528
|
-
|
|
529
|
-
def delete(self, handle: str) -> None:
|
|
530
|
-
filename = _validate_handle(handle)
|
|
531
|
-
try:
|
|
532
|
-
os.unlink(filename, dir_fd=self._directory_fd)
|
|
533
|
-
os.fsync(self._directory_fd)
|
|
534
|
-
except FileNotFoundError:
|
|
535
|
-
pass
|
|
536
|
-
|
|
537
|
-
def close(self) -> None:
|
|
538
|
-
if self._directory_fd >= 0:
|
|
539
|
-
os.close(self._directory_fd)
|
|
540
|
-
self._directory_fd = -1
|
|
541
|
-
|
|
542
|
-
def __enter__(self) -> "EphemeralRecoveryStore":
|
|
543
|
-
return self
|
|
544
|
-
|
|
545
|
-
def __exit__(self, *_: object) -> None:
|
|
546
|
-
self.close()
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
def clean_session(
|
|
550
|
-
base_directory: str | os.PathLike[str],
|
|
551
|
-
session_identifier: str,
|
|
552
|
-
) -> int:
|
|
553
|
-
"""Delete owned runtime artifacts for one session, preserving all else."""
|
|
554
|
-
|
|
555
|
-
with EphemeralRecoveryStore(
|
|
556
|
-
base_directory,
|
|
557
|
-
session_identifier=session_identifier,
|
|
558
|
-
) as recovery:
|
|
559
|
-
removed = recovery.clean_recovery()
|
|
560
|
-
|
|
561
|
-
base_fd = _open_base_directory(base_directory)
|
|
562
|
-
output_fd = -1
|
|
563
|
-
try:
|
|
564
|
-
output_fd = os.open(
|
|
565
|
-
"output-filter",
|
|
566
|
-
_DIRECTORY_FLAGS,
|
|
567
|
-
dir_fd=base_fd,
|
|
568
|
-
)
|
|
569
|
-
try:
|
|
570
|
-
os.rmdir(_session_key(session_identifier), dir_fd=output_fd)
|
|
571
|
-
except OSError:
|
|
572
|
-
pass
|
|
573
|
-
try:
|
|
574
|
-
os.rmdir("output-filter", dir_fd=base_fd)
|
|
575
|
-
except OSError:
|
|
576
|
-
pass
|
|
577
|
-
finally:
|
|
578
|
-
if output_fd >= 0:
|
|
579
|
-
os.close(output_fd)
|
|
580
|
-
os.close(base_fd)
|
|
581
|
-
return removed
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
class _TreeSession:
|
|
585
|
-
__slots__ = ("artifact_names", "output_fd", "session_fd", "session_name")
|
|
586
|
-
|
|
587
|
-
def __init__(
|
|
588
|
-
self,
|
|
589
|
-
output_fd: int,
|
|
590
|
-
session_fd: int,
|
|
591
|
-
session_name: str,
|
|
592
|
-
artifact_names: tuple[str, ...],
|
|
593
|
-
) -> None:
|
|
594
|
-
self.output_fd = output_fd
|
|
595
|
-
self.session_fd = session_fd
|
|
596
|
-
self.session_name = session_name
|
|
597
|
-
self.artifact_names = artifact_names
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
class _TreeOutput:
|
|
601
|
-
__slots__ = ("output_fd", "repo_fd", "sessions")
|
|
602
|
-
|
|
603
|
-
def __init__(
|
|
604
|
-
self,
|
|
605
|
-
repo_fd: int,
|
|
606
|
-
output_fd: int,
|
|
607
|
-
sessions: list[_TreeSession],
|
|
608
|
-
) -> None:
|
|
609
|
-
self.repo_fd = repo_fd
|
|
610
|
-
self.output_fd = output_fd
|
|
611
|
-
self.sessions = sessions
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
def _is_session_hash(name: str) -> bool:
|
|
615
|
-
return (
|
|
616
|
-
len(name) == _HANDLE_LENGTH
|
|
617
|
-
and all(character in "0123456789abcdef" for character in name)
|
|
618
|
-
)
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
def _open_directory_at(parent_fd: int, name: str, label: str) -> int:
|
|
622
|
-
metadata = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
|
|
623
|
-
if not stat.S_ISDIR(metadata.st_mode):
|
|
624
|
-
raise RecoveryUnavailableError(f"{label} is not a directory")
|
|
625
|
-
return os.open(name, _DIRECTORY_FLAGS, dir_fd=parent_fd)
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
def _scan_tree_session(output_fd: int, session_name: str) -> _TreeSession:
|
|
629
|
-
session_fd = _open_directory_at(
|
|
630
|
-
output_fd,
|
|
631
|
-
session_name,
|
|
632
|
-
"owned recovery session",
|
|
633
|
-
)
|
|
634
|
-
try:
|
|
635
|
-
_verify_private_directory(session_fd, "owned recovery session")
|
|
636
|
-
artifacts = tuple(
|
|
637
|
-
name
|
|
638
|
-
for name in os.listdir(session_fd)
|
|
639
|
-
if _owned_runtime_artifact_name(name)
|
|
640
|
-
)
|
|
641
|
-
for artifact_name in artifacts:
|
|
642
|
-
metadata = os.stat(
|
|
643
|
-
artifact_name,
|
|
644
|
-
dir_fd=session_fd,
|
|
645
|
-
follow_symlinks=False,
|
|
646
|
-
)
|
|
647
|
-
if not stat.S_ISREG(metadata.st_mode):
|
|
648
|
-
raise RecoveryUnavailableError(
|
|
649
|
-
"owned recovery artifact is not regular"
|
|
650
|
-
)
|
|
651
|
-
if stat.S_IMODE(metadata.st_mode) != 0o600:
|
|
652
|
-
raise RecoveryUnavailableError(
|
|
653
|
-
"owned recovery artifact is not private"
|
|
654
|
-
)
|
|
655
|
-
return _TreeSession(output_fd, session_fd, session_name, artifacts)
|
|
656
|
-
except BaseException:
|
|
657
|
-
os.close(session_fd)
|
|
658
|
-
raise
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
def _scan_tree_output(repo_fd: int) -> _TreeOutput | None:
|
|
662
|
-
try:
|
|
663
|
-
output_fd = _open_directory_at(
|
|
664
|
-
repo_fd,
|
|
665
|
-
"output-filter",
|
|
666
|
-
"owned output-filter directory",
|
|
667
|
-
)
|
|
668
|
-
except FileNotFoundError:
|
|
669
|
-
return None
|
|
670
|
-
sessions: list[_TreeSession] = []
|
|
671
|
-
try:
|
|
672
|
-
_verify_private_directory(output_fd, "owned output-filter directory")
|
|
673
|
-
for name in os.listdir(output_fd):
|
|
674
|
-
if _is_session_hash(name):
|
|
675
|
-
sessions.append(_scan_tree_session(output_fd, name))
|
|
676
|
-
return _TreeOutput(repo_fd, output_fd, sessions)
|
|
677
|
-
except BaseException:
|
|
678
|
-
for session in sessions:
|
|
679
|
-
os.close(session.session_fd)
|
|
680
|
-
os.close(output_fd)
|
|
681
|
-
raise
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
def _delete_tree_output(output: _TreeOutput) -> int:
|
|
685
|
-
removed = 0
|
|
686
|
-
for session in output.sessions:
|
|
687
|
-
for artifact_name in session.artifact_names:
|
|
688
|
-
os.unlink(artifact_name, dir_fd=session.session_fd)
|
|
689
|
-
removed += 1
|
|
690
|
-
if session.artifact_names:
|
|
691
|
-
os.fsync(session.session_fd)
|
|
692
|
-
try:
|
|
693
|
-
os.rmdir(session.session_name, dir_fd=output.output_fd)
|
|
694
|
-
except OSError:
|
|
695
|
-
pass
|
|
696
|
-
try:
|
|
697
|
-
os.rmdir("output-filter", dir_fd=output.repo_fd)
|
|
698
|
-
except OSError:
|
|
699
|
-
pass
|
|
700
|
-
return removed
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
def clean_owned_repo_recovery(
|
|
704
|
-
base_directory: str | os.PathLike[str],
|
|
705
|
-
) -> int:
|
|
706
|
-
"""Delete validated runtime artifacts for the current repo."""
|
|
707
|
-
|
|
708
|
-
repo_fd = _open_base_directory(base_directory)
|
|
709
|
-
output: _TreeOutput | None = None
|
|
710
|
-
try:
|
|
711
|
-
output = _scan_tree_output(repo_fd)
|
|
712
|
-
return _delete_tree_output(output) if output is not None else 0
|
|
713
|
-
finally:
|
|
714
|
-
if output is not None:
|
|
715
|
-
for session in output.sessions:
|
|
716
|
-
os.close(session.session_fd)
|
|
717
|
-
os.close(output.output_fd)
|
|
718
|
-
os.close(repo_fd)
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
def _for_each_repo_output(
|
|
722
|
-
root_fd: int,
|
|
723
|
-
action: Callable[[_TreeOutput], int],
|
|
724
|
-
) -> int:
|
|
725
|
-
"""Scan and process one repo tree at a time to bound open descriptors."""
|
|
726
|
-
total = 0
|
|
727
|
-
for repo_name in os.listdir(root_fd):
|
|
728
|
-
try:
|
|
729
|
-
metadata = os.stat(
|
|
730
|
-
repo_name,
|
|
731
|
-
dir_fd=root_fd,
|
|
732
|
-
follow_symlinks=False,
|
|
733
|
-
)
|
|
734
|
-
except FileNotFoundError:
|
|
735
|
-
continue
|
|
736
|
-
if not stat.S_ISDIR(metadata.st_mode):
|
|
737
|
-
continue
|
|
738
|
-
repo_fd = os.open(repo_name, _DIRECTORY_FLAGS, dir_fd=root_fd)
|
|
739
|
-
try:
|
|
740
|
-
output = _scan_tree_output(repo_fd)
|
|
741
|
-
if output is None:
|
|
742
|
-
continue
|
|
743
|
-
try:
|
|
744
|
-
total += action(output)
|
|
745
|
-
finally:
|
|
746
|
-
for session in output.sessions:
|
|
747
|
-
os.close(session.session_fd)
|
|
748
|
-
os.close(output.output_fd)
|
|
749
|
-
finally:
|
|
750
|
-
os.close(repo_fd)
|
|
751
|
-
return total
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
def count_owned_recovery_artifacts(
|
|
755
|
-
sessions_root: str | os.PathLike[str],
|
|
756
|
-
) -> int:
|
|
757
|
-
"""Securely count owned runtime artifacts without mutating the tree."""
|
|
758
|
-
|
|
759
|
-
root_fd = _open_base_directory(sessions_root)
|
|
760
|
-
try:
|
|
761
|
-
return _for_each_repo_output(
|
|
762
|
-
root_fd,
|
|
763
|
-
lambda output: sum(
|
|
764
|
-
len(session.artifact_names)
|
|
765
|
-
for session in output.sessions
|
|
766
|
-
),
|
|
767
|
-
)
|
|
768
|
-
finally:
|
|
769
|
-
os.close(root_fd)
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
def clean_owned_recovery_tree(
|
|
773
|
-
sessions_root: str | os.PathLike[str],
|
|
774
|
-
) -> int:
|
|
775
|
-
"""Delete validated runtime artifacts below all repo sessions."""
|
|
776
|
-
|
|
777
|
-
root_fd = _open_base_directory(sessions_root)
|
|
778
|
-
try:
|
|
779
|
-
return _for_each_repo_output(root_fd, _delete_tree_output)
|
|
780
|
-
finally:
|
|
781
|
-
os.close(root_fd)
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
def _load_response_at(session_fd: int, filename: str) -> object | None:
|
|
785
|
-
try:
|
|
786
|
-
metadata = os.stat(
|
|
787
|
-
filename,
|
|
788
|
-
dir_fd=session_fd,
|
|
789
|
-
follow_symlinks=False,
|
|
790
|
-
)
|
|
791
|
-
except FileNotFoundError:
|
|
792
|
-
return None
|
|
793
|
-
if not stat.S_ISREG(metadata.st_mode):
|
|
794
|
-
raise RecoveryUnavailableError("recovery artifact is not regular")
|
|
795
|
-
if stat.S_IMODE(metadata.st_mode) != 0o600:
|
|
796
|
-
raise RecoveryUnavailableError("recovery artifact is not private")
|
|
797
|
-
file_fd = os.open(filename, _READ_FLAGS, dir_fd=session_fd)
|
|
798
|
-
with os.fdopen(file_fd, "rb") as file_handle:
|
|
799
|
-
return json.load(file_handle)
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
def recover_by_handle(
|
|
803
|
-
base_directory: str | os.PathLike[str],
|
|
804
|
-
handle: str,
|
|
805
|
-
) -> object | None:
|
|
806
|
-
"""Find one exact response by opaque handle below the current repo."""
|
|
807
|
-
|
|
808
|
-
filename = _validate_handle(handle)
|
|
809
|
-
base_fd = _open_base_directory(base_directory)
|
|
810
|
-
output_fd = -1
|
|
811
|
-
session_fds: list[int] = []
|
|
812
|
-
matches: list[object] = []
|
|
813
|
-
try:
|
|
814
|
-
try:
|
|
815
|
-
output_fd = _open_directory_at(
|
|
816
|
-
base_fd,
|
|
817
|
-
"output-filter",
|
|
818
|
-
"owned output-filter directory",
|
|
819
|
-
)
|
|
820
|
-
except FileNotFoundError:
|
|
821
|
-
return None
|
|
822
|
-
_verify_private_directory(output_fd, "owned output-filter directory")
|
|
823
|
-
for session_name in os.listdir(output_fd):
|
|
824
|
-
if not _is_session_hash(session_name):
|
|
825
|
-
continue
|
|
826
|
-
session_fd = _open_directory_at(
|
|
827
|
-
output_fd,
|
|
828
|
-
session_name,
|
|
829
|
-
"owned recovery session",
|
|
830
|
-
)
|
|
831
|
-
session_fds.append(session_fd)
|
|
832
|
-
_verify_private_directory(session_fd, "owned recovery session")
|
|
833
|
-
response = _load_response_at(session_fd, filename)
|
|
834
|
-
if response is not None:
|
|
835
|
-
matches.append(response)
|
|
836
|
-
if len(matches) > 1:
|
|
837
|
-
raise RecoveryUnavailableError(
|
|
838
|
-
"recovery handle is ambiguous across sessions"
|
|
839
|
-
)
|
|
840
|
-
return matches[0] if matches else None
|
|
841
|
-
finally:
|
|
842
|
-
for session_fd in session_fds:
|
|
843
|
-
os.close(session_fd)
|
|
844
|
-
if output_fd >= 0:
|
|
845
|
-
os.close(output_fd)
|
|
846
|
-
os.close(base_fd)
|