@ictechgy/context-guard 0.4.8 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.ko.md +92 -37
- package/README.md +111 -37
- package/docs/benchmark-fixtures/token-savings-12task-baseline.prompt.example.md +7 -0
- package/docs/benchmark-fixtures/token-savings-12task-contextguard.prompt.example.md +7 -0
- package/docs/benchmark-fixtures/token-savings-12task.tasks.example.json +182 -0
- package/docs/benchmark-fixtures/token-savings-12task.variants.example.json +10 -0
- package/docs/distribution.md +10 -7
- package/docs/experimental-benchmark-fixtures.md +8 -1
- package/package.json +3 -6
- package/packaging/homebrew/context-guard.rb.template +1 -1
- package/plugins/context-guard/.claude-plugin/plugin.json +1 -1
- package/plugins/context-guard/README.ko.md +9 -6
- package/plugins/context-guard/README.md +27 -12
- package/plugins/context-guard/bin/context-guard +113 -26
- package/plugins/context-guard/bin/context-guard-artifact +542 -46
- package/plugins/context-guard/bin/context-guard-cache-score +380 -0
- package/plugins/context-guard/bin/context-guard-compress +146 -1
- package/plugins/context-guard/bin/context-guard-cost +783 -4
- package/plugins/context-guard/bin/context-guard-experiments +2211 -121
- package/plugins/context-guard/bin/context-guard-failed-nudge +3 -0
- package/plugins/context-guard/bin/context-guard-filter +163 -7
- package/plugins/context-guard/bin/context-guard-guard-read +3 -0
- package/plugins/context-guard/bin/context-guard-pack +602 -43
- package/plugins/context-guard/bin/context-guard-rewrite-bash +3 -0
- package/plugins/context-guard/bin/context-guard-setup +165 -31
- package/plugins/context-guard/bin/context-guard-statusline +490 -283
- package/plugins/context-guard/bin/context-guard-statusline-merged +5 -0
- package/plugins/context-guard/bin/context-guard-tool-prune +241 -1
- package/plugins/context-guard/lib/context_guard_commands.py +206 -0
- package/plugins/context-guard/skills/setup/SKILL.md +1 -0
- package/context-guard-kit/README.md +0 -91
- package/context-guard-kit/benchmark_runner.py +0 -2401
- package/context-guard-kit/claude_transcript_cost_audit.py +0 -2346
- package/context-guard-kit/context_compress.py +0 -695
- package/context-guard-kit/context_escrow.py +0 -935
- package/context-guard-kit/context_filter.py +0 -637
- package/context-guard-kit/context_guard_cli.py +0 -325
- package/context-guard-kit/context_guard_diet.py +0 -1711
- package/context-guard-kit/context_pack.py +0 -2713
- package/context-guard-kit/cost_guard.py +0 -2349
- package/context-guard-kit/experimental_registry.py +0 -2339
- package/context-guard-kit/failed_attempt_nudge.py +0 -567
- package/context-guard-kit/guard_large_read.py +0 -690
- package/context-guard-kit/hook_secret_patterns.py +0 -43
- package/context-guard-kit/read_symbol.py +0 -483
- package/context-guard-kit/rewrite_bash_for_token_budget.py +0 -501
- package/context-guard-kit/sanitize_output.py +0 -725
- package/context-guard-kit/settings.example.json +0 -67
- package/context-guard-kit/setup_wizard.py +0 -2515
- package/context-guard-kit/statusline.sh +0 -362
- package/context-guard-kit/statusline_merged.sh +0 -157
- package/context-guard-kit/tool_schema_pruner.py +0 -837
- package/context-guard-kit/trim_command_output.py +0 -1449
|
@@ -0,0 +1,380 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Static prompt cacheability lint for ContextGuard.
|
|
3
|
+
|
|
4
|
+
``context-guard-cache-score`` is advisory-only: it does not call provider APIs,
|
|
5
|
+
does not estimate price, does not observe cache hits, and does not write raw
|
|
6
|
+
prompts to disk. It only inspects a prompt/request fixture for stable-prefix
|
|
7
|
+
shape, common dynamic markers, deterministic ordering hints, and provider cache
|
|
8
|
+
eligibility using a tokenizer-free char/4 proxy.
|
|
9
|
+
"""
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import argparse
|
|
13
|
+
import json
|
|
14
|
+
import math
|
|
15
|
+
import os
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
import re
|
|
18
|
+
import stat
|
|
19
|
+
import sys
|
|
20
|
+
from typing import Any, NoReturn
|
|
21
|
+
|
|
22
|
+
TOOL_NAME = "context-guard-cache-score"
|
|
23
|
+
SCHEMA_VERSION = "contextguard.cache-score.v1"
|
|
24
|
+
DEFAULT_MAX_INPUT_BYTES = 1_000_000
|
|
25
|
+
TOKEN_PROXY_CHARS_PER_TOKEN = 4
|
|
26
|
+
PROVIDER_MINIMUM_CACHEABLE_TOKENS = {
|
|
27
|
+
# Provider and model minimums move over time. These defaults are advisory
|
|
28
|
+
# and can be overridden with --minimum-cacheable-tokens.
|
|
29
|
+
"openai": 1024,
|
|
30
|
+
"anthropic": 1024,
|
|
31
|
+
"gemini": 2048,
|
|
32
|
+
"generic": 1024,
|
|
33
|
+
}
|
|
34
|
+
PROVIDER_CAVEATS = {
|
|
35
|
+
"openai": (
|
|
36
|
+
"OpenAI prompt caching is automatic for eligible prompts; verify real "
|
|
37
|
+
"hits with provider usage.prompt_tokens_details.cached_tokens."
|
|
38
|
+
),
|
|
39
|
+
"anthropic": (
|
|
40
|
+
"Anthropic prompt caching is model/platform-specific and usually needs "
|
|
41
|
+
"cache_control around the reusable prefix; verify cache_creation/read "
|
|
42
|
+
"usage fields."
|
|
43
|
+
),
|
|
44
|
+
"gemini": (
|
|
45
|
+
"Gemini context caching thresholds vary by model/platform; verify with "
|
|
46
|
+
"provider cached-content usage fields and override the threshold when "
|
|
47
|
+
"your model differs."
|
|
48
|
+
),
|
|
49
|
+
"generic": (
|
|
50
|
+
"Generic cache scoring uses a conservative threshold only; check your "
|
|
51
|
+
"provider documentation before claiming cache eligibility."
|
|
52
|
+
),
|
|
53
|
+
}
|
|
54
|
+
ALLOWED_FIRST_ABSOLUTE_SYMLINKS = {
|
|
55
|
+
"tmp": Path("/private/tmp"),
|
|
56
|
+
"var": Path("/private/var"),
|
|
57
|
+
}
|
|
58
|
+
MAX_JSON_PATH_SEGMENT_CHARS = 64
|
|
59
|
+
SAFE_JSON_PATH_SEGMENT_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]{0,63}$")
|
|
60
|
+
DYNAMIC_JSON_KEY_RE = re.compile(r"(?i)(request|trace|nonce|random|timestamp|created[_-]?at|updated[_-]?at|date)")
|
|
61
|
+
SENSITIVE_JSON_KEY_RE = re.compile(
|
|
62
|
+
r"(?i)(authorization|api[_-]?key|apikey|token|secret|password|passwd|pwd|client[_-]?secret|credential|signature|sig|private[_-]?key|privatekey|ssh[_-]?key|sshkey)"
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
DYNAMIC_MARKERS: tuple[tuple[str, re.Pattern[str]], ...] = (
|
|
66
|
+
("iso_timestamp", re.compile(r"\b20\d{2}-\d{2}-\d{2}[T ][0-2]\d:[0-5]\d(?::[0-5]\d(?:\.\d{1,9})?)?(?:Z|[+-][0-2]\d:?[0-5]\d)?\b")),
|
|
67
|
+
("uuid", re.compile(r"\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}\b")),
|
|
68
|
+
("unix_epoch_ms", re.compile(r"\b1[6-9]\d{11}\b")),
|
|
69
|
+
("request_id_key", re.compile(r"(?i)\b(?:request[_-]?id|trace[_-]?id|nonce|random[_-]?(?:id|seed)?|timestamp|created[_-]?at|updated[_-]?at|date_now)\b")),
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class CacheScoreError(ValueError):
|
|
74
|
+
"""User-facing fail-closed error."""
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def fail(message: str) -> NoReturn:
|
|
78
|
+
raise CacheScoreError(message)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def byte_len_text(text: str) -> int:
|
|
82
|
+
return len(text.encode("utf-8", errors="replace"))
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def json_bytes(data: Any, *, indent: int | None = None) -> str:
|
|
86
|
+
return json.dumps(data, ensure_ascii=False, sort_keys=True, separators=(",", ":") if indent is None else None, indent=indent)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def json_path_child(path: str, key: object) -> str:
|
|
90
|
+
"""Return a JSON warning path segment without echoing sensitive/dynamic keys."""
|
|
91
|
+
text = str(key)
|
|
92
|
+
if DYNAMIC_JSON_KEY_RE.search(text) or SENSITIVE_JSON_KEY_RE.search(text):
|
|
93
|
+
return f"{path}.[redacted-key]"
|
|
94
|
+
if SAFE_JSON_PATH_SEGMENT_RE.fullmatch(text):
|
|
95
|
+
return f"{path}.{text}"
|
|
96
|
+
if len(text) > MAX_JSON_PATH_SEGMENT_CHARS:
|
|
97
|
+
return f"{path}.[key:{len(text)} chars]"
|
|
98
|
+
return f"{path}.[key]"
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def bounded_int(value: object, *, default: int, minimum: int, maximum: int, name: str) -> int:
|
|
102
|
+
try:
|
|
103
|
+
number = int(default if value is None else value)
|
|
104
|
+
except (TypeError, ValueError, OverflowError):
|
|
105
|
+
fail(f"{name} must be an integer")
|
|
106
|
+
if number < minimum:
|
|
107
|
+
fail(f"{name} must be >= {minimum}")
|
|
108
|
+
if number > maximum:
|
|
109
|
+
fail(f"{name} must be <= {maximum}")
|
|
110
|
+
return number
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def normalized_link_target(parent: Path, raw_target: str) -> Path:
|
|
114
|
+
target = Path(raw_target)
|
|
115
|
+
if not target.is_absolute():
|
|
116
|
+
target = parent / target
|
|
117
|
+
return Path(os.path.normpath(str(target)))
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def normalize_allowed_first_absolute_symlink(path: Path) -> Path:
|
|
121
|
+
if not path.is_absolute() or len(path.parts) < 2:
|
|
122
|
+
return path
|
|
123
|
+
first = path.parts[1]
|
|
124
|
+
expected = ALLOWED_FIRST_ABSOLUTE_SYMLINKS.get(first)
|
|
125
|
+
if expected is None:
|
|
126
|
+
return path
|
|
127
|
+
link = Path(path.anchor) / first
|
|
128
|
+
try:
|
|
129
|
+
if not stat.S_ISLNK(os.lstat(link).st_mode):
|
|
130
|
+
return path
|
|
131
|
+
if normalized_link_target(Path(path.anchor), os.readlink(link)) != expected:
|
|
132
|
+
return path
|
|
133
|
+
except OSError:
|
|
134
|
+
return path
|
|
135
|
+
return expected.joinpath(*path.parts[2:])
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def reject_symlink_components(path: Path) -> None:
|
|
139
|
+
path = normalize_allowed_first_absolute_symlink(path)
|
|
140
|
+
current = Path(path.anchor) if path.is_absolute() else Path()
|
|
141
|
+
for part in path.parts:
|
|
142
|
+
if path.is_absolute() and part == path.anchor:
|
|
143
|
+
continue
|
|
144
|
+
current = current / part
|
|
145
|
+
try:
|
|
146
|
+
st = os.lstat(current)
|
|
147
|
+
except FileNotFoundError:
|
|
148
|
+
return
|
|
149
|
+
if stat.S_ISLNK(st.st_mode):
|
|
150
|
+
fail(f"refusing path with symlink component: {current}")
|
|
151
|
+
if not stat.S_ISDIR(st.st_mode) and current != path:
|
|
152
|
+
fail(f"refusing path through non-directory component: {current}")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def read_limited_path(path: Path, max_bytes: int) -> str:
|
|
156
|
+
reject_symlink_components(path)
|
|
157
|
+
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
|
|
158
|
+
try:
|
|
159
|
+
fd = os.open(str(path), flags)
|
|
160
|
+
except OSError as exc:
|
|
161
|
+
fail(f"input read failed: {exc}")
|
|
162
|
+
try:
|
|
163
|
+
st = os.fstat(fd)
|
|
164
|
+
if not stat.S_ISREG(st.st_mode):
|
|
165
|
+
fail("input must be a regular file")
|
|
166
|
+
if st.st_size > max_bytes:
|
|
167
|
+
fail(f"input exceeds --max-input-bytes: {st.st_size} > {max_bytes}")
|
|
168
|
+
data = os.read(fd, max_bytes + 1)
|
|
169
|
+
finally:
|
|
170
|
+
os.close(fd)
|
|
171
|
+
if len(data) > max_bytes:
|
|
172
|
+
fail(f"input exceeds --max-input-bytes: > {max_bytes}")
|
|
173
|
+
return data.decode("utf-8", errors="replace")
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def read_limited_stdin(max_bytes: int) -> str:
|
|
177
|
+
data = sys.stdin.buffer.read(max_bytes + 1)
|
|
178
|
+
if len(data) > max_bytes:
|
|
179
|
+
fail(f"input exceeds --max-input-bytes: > {max_bytes}")
|
|
180
|
+
return data.decode("utf-8", errors="replace")
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def estimate_tokens(text: str) -> int:
|
|
184
|
+
if not text:
|
|
185
|
+
return 0
|
|
186
|
+
return int(math.ceil(len(text) / TOKEN_PROXY_CHARS_PER_TOKEN))
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def first_dynamic_marker(text: str) -> tuple[int | None, str | None]:
|
|
190
|
+
best_offset: int | None = None
|
|
191
|
+
best_name: str | None = None
|
|
192
|
+
for name, pattern in DYNAMIC_MARKERS:
|
|
193
|
+
match = pattern.search(text)
|
|
194
|
+
if match and (best_offset is None or match.start() < best_offset):
|
|
195
|
+
best_offset = match.start()
|
|
196
|
+
best_name = name
|
|
197
|
+
return best_offset, best_name
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _walk_json(value: Any, path: str = "$") -> list[dict[str, Any]]:
|
|
201
|
+
warnings: list[dict[str, Any]] = []
|
|
202
|
+
if isinstance(value, dict):
|
|
203
|
+
keys = [str(key) for key in value]
|
|
204
|
+
if keys != sorted(keys):
|
|
205
|
+
warnings.append({
|
|
206
|
+
"code": "json_object_key_order_not_sorted",
|
|
207
|
+
"path": path,
|
|
208
|
+
"severity": "info",
|
|
209
|
+
"message": "Object keys are not in deterministic sorted order; keep generated JSON stable across runs.",
|
|
210
|
+
})
|
|
211
|
+
for key, item in value.items():
|
|
212
|
+
child_path = json_path_child(path, key)
|
|
213
|
+
if DYNAMIC_JSON_KEY_RE.search(str(key)):
|
|
214
|
+
warnings.append({
|
|
215
|
+
"code": "dynamic_json_key",
|
|
216
|
+
"path": child_path,
|
|
217
|
+
"severity": "warn",
|
|
218
|
+
"message": "Dynamic-looking JSON key appears in the prompt/request; place dynamic values after the reusable prefix.",
|
|
219
|
+
})
|
|
220
|
+
warnings.extend(_walk_json(item, child_path))
|
|
221
|
+
elif isinstance(value, list):
|
|
222
|
+
if path.endswith(".tools") and all(isinstance(item, dict) and "name" in item for item in value):
|
|
223
|
+
names = [str(item.get("name")) for item in value]
|
|
224
|
+
if names != sorted(names):
|
|
225
|
+
warnings.append({
|
|
226
|
+
"code": "tool_order_not_sorted",
|
|
227
|
+
"path": path,
|
|
228
|
+
"severity": "info",
|
|
229
|
+
"message": "Tool definitions are not sorted by name; deterministic ordering improves prefix reuse.",
|
|
230
|
+
})
|
|
231
|
+
for index, item in enumerate(value):
|
|
232
|
+
warnings.extend(_walk_json(item, f"{path}[{index}]"))
|
|
233
|
+
return warnings
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def json_shape_warnings(text: str) -> tuple[str, list[dict[str, Any]]]:
|
|
237
|
+
try:
|
|
238
|
+
data = json.loads(text)
|
|
239
|
+
except json.JSONDecodeError:
|
|
240
|
+
return "text", []
|
|
241
|
+
if not isinstance(data, (dict, list)):
|
|
242
|
+
return "json-scalar", []
|
|
243
|
+
warnings = _walk_json(data)
|
|
244
|
+
canonical = json_bytes(data, indent=2) + "\n"
|
|
245
|
+
if canonical != text:
|
|
246
|
+
warnings.append({
|
|
247
|
+
"code": "json_not_canonical",
|
|
248
|
+
"path": "$",
|
|
249
|
+
"severity": "info",
|
|
250
|
+
"message": "JSON input is parseable but not canonical sort-key formatting; generated prompt JSON should be byte-stable.",
|
|
251
|
+
})
|
|
252
|
+
return "json", warnings
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def score_prompt(text: str, *, provider: str, minimum_cacheable_tokens: int) -> dict[str, Any]:
|
|
256
|
+
prompt_kind, shape_warnings = json_shape_warnings(text)
|
|
257
|
+
dynamic_offset, dynamic_marker = first_dynamic_marker(text)
|
|
258
|
+
prefix_text = text if dynamic_offset is None else text[:dynamic_offset]
|
|
259
|
+
estimated = estimate_tokens(text)
|
|
260
|
+
prefix_estimated = estimate_tokens(prefix_text)
|
|
261
|
+
total_chars = len(text)
|
|
262
|
+
static_ratio = 1.0 if total_chars == 0 else len(prefix_text) / total_chars
|
|
263
|
+
warnings = list(shape_warnings)
|
|
264
|
+
if dynamic_offset is not None:
|
|
265
|
+
warnings.append({
|
|
266
|
+
"code": "dynamic_marker_in_prompt",
|
|
267
|
+
"severity": "warn",
|
|
268
|
+
"message": "Dynamic-looking content appears before the end of the prompt; move timestamps/request IDs/user-specific values later.",
|
|
269
|
+
"offset": dynamic_offset,
|
|
270
|
+
"marker": dynamic_marker,
|
|
271
|
+
})
|
|
272
|
+
if prefix_estimated < minimum_cacheable_tokens:
|
|
273
|
+
warnings.append({
|
|
274
|
+
"code": "below_minimum_cacheable_tokens",
|
|
275
|
+
"severity": "warn",
|
|
276
|
+
"message": "Static prefix token proxy is below the selected provider threshold.",
|
|
277
|
+
})
|
|
278
|
+
if provider == "anthropic" and "cache_control" not in text:
|
|
279
|
+
warnings.append({
|
|
280
|
+
"code": "anthropic_cache_control_not_detected",
|
|
281
|
+
"severity": "info",
|
|
282
|
+
"message": "Anthropic caching usually requires cache_control around the reusable prefix.",
|
|
283
|
+
})
|
|
284
|
+
|
|
285
|
+
return {
|
|
286
|
+
"tool": TOOL_NAME,
|
|
287
|
+
"schema_version": SCHEMA_VERSION,
|
|
288
|
+
"provider": provider,
|
|
289
|
+
"prompt_kind": prompt_kind,
|
|
290
|
+
"minimum_cacheable_tokens": minimum_cacheable_tokens,
|
|
291
|
+
"eligible": prefix_estimated >= minimum_cacheable_tokens,
|
|
292
|
+
"estimated_tokens": estimated,
|
|
293
|
+
"cacheable_prefix_tokens": prefix_estimated,
|
|
294
|
+
"token_estimate": {
|
|
295
|
+
"method": "char4_proxy",
|
|
296
|
+
"chars_per_token": TOKEN_PROXY_CHARS_PER_TOKEN,
|
|
297
|
+
"estimated_tokens": estimated,
|
|
298
|
+
"cacheable_prefix_tokens": prefix_estimated,
|
|
299
|
+
"label": "provider_tokenizer_free_proxy_not_billed_tokens",
|
|
300
|
+
},
|
|
301
|
+
"input_chars": total_chars,
|
|
302
|
+
"cacheable_prefix_chars": len(prefix_text),
|
|
303
|
+
"first_dynamic_offset": dynamic_offset,
|
|
304
|
+
"first_dynamic_marker": dynamic_marker,
|
|
305
|
+
"static_prefix_ratio": round(static_ratio, 6),
|
|
306
|
+
"warnings": warnings,
|
|
307
|
+
"provider_caveat": PROVIDER_CAVEATS[provider],
|
|
308
|
+
"raw_prompt_stored": False,
|
|
309
|
+
"claim_boundary": {
|
|
310
|
+
"advisory_only": True,
|
|
311
|
+
"provider_measured_cache_hit": False,
|
|
312
|
+
"hosted_api_token_or_cost_savings_claim_allowed": False,
|
|
313
|
+
"requires_provider_usage_fields_for_claims": True,
|
|
314
|
+
"token_estimate_is_provider_tokenizer_free_proxy": True,
|
|
315
|
+
},
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def render_text(report: dict[str, Any]) -> str:
|
|
320
|
+
status = "eligible" if report.get("eligible") else "not eligible"
|
|
321
|
+
warnings = report.get("warnings") if isinstance(report.get("warnings"), list) else []
|
|
322
|
+
warning_codes = ", ".join(str(item.get("code")) for item in warnings if isinstance(item, dict)) or "none"
|
|
323
|
+
return (
|
|
324
|
+
f"{TOOL_NAME}: {status} for {report['provider']} "
|
|
325
|
+
f"(static_prefix≈{report['cacheable_prefix_tokens']} char/4 tokens, "
|
|
326
|
+
f"minimum={report['minimum_cacheable_tokens']})\n"
|
|
327
|
+
f"warnings: {warning_codes}\n"
|
|
328
|
+
"claim boundary: advisory static lint only; not a measured provider cache hit or cost saving.\n"
|
|
329
|
+
)
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
333
|
+
parser = argparse.ArgumentParser(
|
|
334
|
+
description=(
|
|
335
|
+
"Static prompt cacheability lint. No provider calls, no pricing ledger, "
|
|
336
|
+
"and no measured cache-hit claims."
|
|
337
|
+
)
|
|
338
|
+
)
|
|
339
|
+
parser.add_argument("--input", help="prompt/request text or JSON path; stdin is used when omitted")
|
|
340
|
+
parser.add_argument("--provider", choices=sorted(PROVIDER_MINIMUM_CACHEABLE_TOKENS), default="generic")
|
|
341
|
+
parser.add_argument(
|
|
342
|
+
"--minimum-cacheable-tokens",
|
|
343
|
+
default=None,
|
|
344
|
+
help="override provider threshold for model/platform-specific cache minimums",
|
|
345
|
+
)
|
|
346
|
+
parser.add_argument("--max-input-bytes", default=DEFAULT_MAX_INPUT_BYTES, help=f"maximum input bytes (default: {DEFAULT_MAX_INPUT_BYTES})")
|
|
347
|
+
parser.add_argument("--json", action="store_true", help="emit stable JSON")
|
|
348
|
+
return parser
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def main(argv: list[str] | None = None) -> int:
|
|
352
|
+
parser = build_parser()
|
|
353
|
+
args = parser.parse_args(argv)
|
|
354
|
+
try:
|
|
355
|
+
max_input_bytes = bounded_int(args.max_input_bytes, default=DEFAULT_MAX_INPUT_BYTES, minimum=1, maximum=100_000_000, name="--max-input-bytes")
|
|
356
|
+
provider = str(args.provider)
|
|
357
|
+
default_minimum = PROVIDER_MINIMUM_CACHEABLE_TOKENS[provider]
|
|
358
|
+
minimum = bounded_int(
|
|
359
|
+
args.minimum_cacheable_tokens,
|
|
360
|
+
default=default_minimum,
|
|
361
|
+
minimum=1,
|
|
362
|
+
maximum=10_000_000,
|
|
363
|
+
name="--minimum-cacheable-tokens",
|
|
364
|
+
)
|
|
365
|
+
text = read_limited_path(Path(args.input), max_input_bytes) if args.input else read_limited_stdin(max_input_bytes)
|
|
366
|
+
report = score_prompt(text, provider=provider, minimum_cacheable_tokens=minimum)
|
|
367
|
+
if args.json:
|
|
368
|
+
sys.stdout.write(json_bytes(report, indent=2) + "\n")
|
|
369
|
+
else:
|
|
370
|
+
sys.stdout.write(render_text(report))
|
|
371
|
+
return 0
|
|
372
|
+
except CacheScoreError as exc:
|
|
373
|
+
print(f"{TOOL_NAME}: {exc}", file=sys.stderr)
|
|
374
|
+
return 1
|
|
375
|
+
except BrokenPipeError:
|
|
376
|
+
return 1
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
if __name__ == "__main__":
|
|
380
|
+
raise SystemExit(main())
|
|
@@ -28,6 +28,9 @@ MAX_MAX_BYTES = 100_000_000
|
|
|
28
28
|
# 메타데이터에 measurement="estimated" 로 명시해 관측 토큰 수와 혼동되지 않게 한다.
|
|
29
29
|
TOKEN_PROXY_CHARS_PER_TOKEN = 4
|
|
30
30
|
CONTENT_TYPES = ("json", "diff", "log", "search", "code", "prose")
|
|
31
|
+
COMPRESSION_MODES = ("conservative", "readable")
|
|
32
|
+
READABLE_COMPRESSION_SCHEMA_VERSION = "contextguard.compress-readable.v1"
|
|
33
|
+
READABLE_SENTENCE_LIMIT = 5
|
|
31
34
|
|
|
32
35
|
# diff 구조 라인(파일 헤더/헝크/변경)을 식별한다. 나머지 context 라인은 접어서 줄인다.
|
|
33
36
|
DIFF_FILE_HEADER_RE = re.compile(r"^(diff --git |index [0-9a-f]|--- |\+\+\+ |rename |similarity |new file|deleted file)")
|
|
@@ -93,6 +96,20 @@ PROTECTED_DENIED_TRANSFORMS = (
|
|
|
93
96
|
"path_rewrite",
|
|
94
97
|
"quoted_literal_rewrite",
|
|
95
98
|
)
|
|
99
|
+
READABLE_BLOCKING_PROTECTED_KEYS = (
|
|
100
|
+
"code_fence",
|
|
101
|
+
"diff",
|
|
102
|
+
"hash",
|
|
103
|
+
"path",
|
|
104
|
+
"stack_frame",
|
|
105
|
+
"numeric_constant",
|
|
106
|
+
"quoted_string",
|
|
107
|
+
"json_key",
|
|
108
|
+
)
|
|
109
|
+
PROMPT_LIKE_INSTRUCTION_RE = re.compile(
|
|
110
|
+
r"(?i)\b(ignore (?:all )?(?:previous|above) instructions|system prompt|developer message|"
|
|
111
|
+
r"you are chatgpt|act as (?:a|an)|do not follow|BEGIN (?:SYSTEM|DEVELOPER)|END (?:SYSTEM|DEVELOPER))\b"
|
|
112
|
+
)
|
|
96
113
|
|
|
97
114
|
|
|
98
115
|
def bounded_int(value: object, default: int, minimum: int, maximum: int) -> int:
|
|
@@ -301,6 +318,43 @@ def build_transform_policy(protected_policy: dict[str, object]) -> dict[str, obj
|
|
|
301
318
|
}
|
|
302
319
|
|
|
303
320
|
|
|
321
|
+
def build_readable_compression_metadata(
|
|
322
|
+
*,
|
|
323
|
+
content_type: str,
|
|
324
|
+
strategy_detail: dict[str, object],
|
|
325
|
+
lossy: bool,
|
|
326
|
+
) -> dict[str, object]:
|
|
327
|
+
blocking = strategy_detail.get("readable_blocking_signals", {})
|
|
328
|
+
if not isinstance(blocking, dict):
|
|
329
|
+
blocking = {}
|
|
330
|
+
applied = bool(strategy_detail.get("readable_applied"))
|
|
331
|
+
exact_fallback_required = bool(lossy or applied)
|
|
332
|
+
return {
|
|
333
|
+
"schema_version": READABLE_COMPRESSION_SCHEMA_VERSION,
|
|
334
|
+
"mode": "readable",
|
|
335
|
+
"preview_only": True,
|
|
336
|
+
"applied": applied,
|
|
337
|
+
"content_type": content_type,
|
|
338
|
+
"strategy": strategy_detail.get("strategy"),
|
|
339
|
+
"readable_strategy": strategy_detail.get("readable_strategy", "structural-preview"),
|
|
340
|
+
"omitted_reason": strategy_detail.get("readable_omitted_reason"),
|
|
341
|
+
"blocking_signal_counts": blocking,
|
|
342
|
+
"protected_spans_stored": False,
|
|
343
|
+
"source_verification": {
|
|
344
|
+
"exact_fallback_required": exact_fallback_required,
|
|
345
|
+
"recommended_command": "context-guard-artifact store --command 'readable-mode exact fallback' --json < sanitized-prose.txt",
|
|
346
|
+
"verify_before_edit_or_claim": True,
|
|
347
|
+
},
|
|
348
|
+
"claim_boundary": {
|
|
349
|
+
"deterministic_local_only": True,
|
|
350
|
+
"no_network_model_embedding_or_reranker": True,
|
|
351
|
+
"no_generated_semantic_rewrite": True,
|
|
352
|
+
"byte_and_token_counts_are_local_proxies": True,
|
|
353
|
+
"hosted_api_token_or_cost_savings_claim_allowed": False,
|
|
354
|
+
},
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
|
|
304
358
|
def _looks_like_json(stripped: str) -> bool:
|
|
305
359
|
if stripped[0] not in "{[":
|
|
306
360
|
return False
|
|
@@ -434,6 +488,64 @@ def compress_prose(text: str) -> tuple[str, dict[str, object]]:
|
|
|
434
488
|
return _whitespace_normalize(text, strategy="prose-whitespace", max_consecutive_blank=1)
|
|
435
489
|
|
|
436
490
|
|
|
491
|
+
def readable_blocking_signal_counts(text: str, content_type: str) -> dict[str, int]:
|
|
492
|
+
counts = protected_zone_counts(text)
|
|
493
|
+
blocking = {
|
|
494
|
+
key: int(counts.get(key, 0) or 0)
|
|
495
|
+
for key in READABLE_BLOCKING_PROTECTED_KEYS
|
|
496
|
+
if int(counts.get(key, 0) or 0) > 0
|
|
497
|
+
}
|
|
498
|
+
prompt_like = len(PROMPT_LIKE_INSTRUCTION_RE.findall(text))
|
|
499
|
+
if prompt_like:
|
|
500
|
+
blocking["prompt_like_instruction"] = prompt_like
|
|
501
|
+
if content_type != "prose":
|
|
502
|
+
blocking["non_prose_content"] = 1
|
|
503
|
+
return blocking
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def split_prose_sentences(text: str) -> list[str]:
|
|
507
|
+
compact = " ".join(text.split())
|
|
508
|
+
if not compact:
|
|
509
|
+
return []
|
|
510
|
+
sentences = re.split(r"(?<=[.!?])\s+", compact)
|
|
511
|
+
return [sentence.strip() for sentence in sentences if sentence.strip()]
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def compress_prose_readable(text: str) -> tuple[str, dict[str, object]]:
|
|
515
|
+
"""Readable opt-in sentence window for sanitized unprotected prose only."""
|
|
516
|
+
normalized, base_detail = compress_prose(text)
|
|
517
|
+
blocking = readable_blocking_signal_counts(normalized, "prose")
|
|
518
|
+
detail = dict(base_detail)
|
|
519
|
+
detail.update({
|
|
520
|
+
"readable_mode": True,
|
|
521
|
+
"readable_strategy": "sentence-window-preview",
|
|
522
|
+
"readable_blocking_signals": blocking,
|
|
523
|
+
})
|
|
524
|
+
if blocking:
|
|
525
|
+
detail["readable_applied"] = False
|
|
526
|
+
detail["readable_omitted_reason"] = "protected_or_prompt_like_signal"
|
|
527
|
+
return normalized, detail
|
|
528
|
+
sentences = split_prose_sentences(normalized)
|
|
529
|
+
if len(sentences) <= READABLE_SENTENCE_LIMIT:
|
|
530
|
+
detail["readable_applied"] = False
|
|
531
|
+
detail["readable_omitted_reason"] = "short_prose"
|
|
532
|
+
return normalized, detail
|
|
533
|
+
included_sentences = sentences[:3] + sentences[-1:]
|
|
534
|
+
kept = sentences[:3] + [f"[context-guard-readable] {len(sentences) - len(included_sentences)} sentence(s) omitted; retrieve exact source before relying on omitted detail."] + sentences[-1:]
|
|
535
|
+
preview = " ".join(kept)
|
|
536
|
+
if text.endswith("\n"):
|
|
537
|
+
preview += "\n"
|
|
538
|
+
detail.update({
|
|
539
|
+
"strategy": "prose-readable-window",
|
|
540
|
+
"lossy": True,
|
|
541
|
+
"readable_applied": True,
|
|
542
|
+
"sentences_original": len(sentences),
|
|
543
|
+
"sentences_included": len(included_sentences),
|
|
544
|
+
"sentences_omitted": len(sentences) - len(included_sentences),
|
|
545
|
+
})
|
|
546
|
+
return preview, detail
|
|
547
|
+
|
|
548
|
+
|
|
437
549
|
def _whitespace_normalize(text: str, *, strategy: str, max_consecutive_blank: int) -> tuple[str, dict[str, object]]:
|
|
438
550
|
out: list[str] = []
|
|
439
551
|
blank_run = 0
|
|
@@ -482,6 +594,7 @@ def build_metadata(
|
|
|
482
594
|
input_bytes: int,
|
|
483
595
|
max_bytes: int,
|
|
484
596
|
protected_policy_enabled: bool = False,
|
|
597
|
+
compression_mode: str = "conservative",
|
|
485
598
|
) -> dict[str, object]:
|
|
486
599
|
"""Assemble the compress receipt: observed byte/line counts plus an estimated token proxy.
|
|
487
600
|
|
|
@@ -550,6 +663,12 @@ def build_metadata(
|
|
|
550
663
|
"Protected lossy structural transform: store the full sanitized text with "
|
|
551
664
|
"`context-guard-artifact store` and retrieve exact slices before relying on omitted content."
|
|
552
665
|
)
|
|
666
|
+
if compression_mode == "readable":
|
|
667
|
+
metadata["readable_compression"] = build_readable_compression_metadata(
|
|
668
|
+
content_type=content_type,
|
|
669
|
+
strategy_detail=strategy_detail,
|
|
670
|
+
lossy=lossy,
|
|
671
|
+
)
|
|
553
672
|
return metadata
|
|
554
673
|
|
|
555
674
|
|
|
@@ -562,6 +681,7 @@ def compress_text(
|
|
|
562
681
|
input_bytes: int,
|
|
563
682
|
max_bytes: int,
|
|
564
683
|
protected_policy_enabled: bool = False,
|
|
684
|
+
compression_mode: str = "conservative",
|
|
565
685
|
) -> tuple[str, dict[str, object]]:
|
|
566
686
|
"""Sanitize first, then classify and compress, then build the receipt.
|
|
567
687
|
|
|
@@ -573,11 +693,24 @@ def compress_text(
|
|
|
573
693
|
content_type, type_source = forced_type, "override"
|
|
574
694
|
else:
|
|
575
695
|
content_type, type_source = classify_content(sanitized), "detected"
|
|
576
|
-
|
|
696
|
+
if compression_mode == "readable" and content_type == "prose":
|
|
697
|
+
compressed, strategy_detail = compress_prose_readable(sanitized)
|
|
698
|
+
else:
|
|
699
|
+
compressed, strategy_detail = STRATEGIES[content_type](sanitized)
|
|
700
|
+
if compression_mode == "readable":
|
|
701
|
+
strategy_detail["readable_mode"] = True
|
|
702
|
+
strategy_detail["readable_strategy"] = "sentence-window-preview"
|
|
703
|
+
strategy_detail["readable_applied"] = False
|
|
704
|
+
strategy_detail["readable_omitted_reason"] = "non_prose_content"
|
|
705
|
+
strategy_detail["readable_blocking_signals"] = {"non_prose_content": 1}
|
|
577
706
|
# 보수성 보장: 어떤 전략도 입력보다 큰 결과를 내보내지 않는다. 작은 입력에서
|
|
578
707
|
# 접기 마커가 원본보다 길어지는 경우 살균된 원본을 그대로 유지한다.
|
|
579
708
|
if byte_length(compressed) >= byte_length(sanitized):
|
|
580
709
|
compressed = sanitized
|
|
710
|
+
if compression_mode == "readable" and strategy_detail.get("readable_applied"):
|
|
711
|
+
strategy_detail["lossy"] = False
|
|
712
|
+
strategy_detail["readable_applied"] = False
|
|
713
|
+
strategy_detail["readable_omitted_reason"] = "not_smaller_than_input"
|
|
581
714
|
strategy_detail["reduced"] = False
|
|
582
715
|
else:
|
|
583
716
|
strategy_detail["reduced"] = True
|
|
@@ -592,6 +725,7 @@ def compress_text(
|
|
|
592
725
|
input_bytes=input_bytes,
|
|
593
726
|
max_bytes=max_bytes,
|
|
594
727
|
protected_policy_enabled=protected_policy_enabled,
|
|
728
|
+
compression_mode=compression_mode,
|
|
595
729
|
)
|
|
596
730
|
return compressed, metadata
|
|
597
731
|
|
|
@@ -623,6 +757,10 @@ def render_text_receipt(metadata: dict[str, object]) -> str:
|
|
|
623
757
|
def run_compress(args: argparse.Namespace) -> int:
|
|
624
758
|
"""Read stdin, compress, then emit JSON or (compressed text + stderr receipt)."""
|
|
625
759
|
max_bytes = bounded_int(args.max_bytes, DEFAULT_MAX_BYTES, 1, MAX_MAX_BYTES)
|
|
760
|
+
compression_mode = args.mode
|
|
761
|
+
if compression_mode not in COMPRESSION_MODES:
|
|
762
|
+
print(f"context-guard-compress: unknown --mode: {compression_mode}", file=sys.stderr)
|
|
763
|
+
return 2
|
|
626
764
|
raw_text, input_truncated, input_bytes = read_bounded_stdin(max_bytes)
|
|
627
765
|
forced_type = args.type
|
|
628
766
|
if forced_type is not None and forced_type not in STRATEGIES:
|
|
@@ -636,6 +774,7 @@ def run_compress(args: argparse.Namespace) -> int:
|
|
|
636
774
|
input_bytes=input_bytes,
|
|
637
775
|
max_bytes=max_bytes,
|
|
638
776
|
protected_policy_enabled=bool(args.protected_policy),
|
|
777
|
+
compression_mode=compression_mode,
|
|
639
778
|
)
|
|
640
779
|
if args.json:
|
|
641
780
|
payload = {"metadata": metadata, "content": compressed}
|
|
@@ -659,6 +798,12 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
659
798
|
default=None,
|
|
660
799
|
help="force a content type instead of auto-detecting (json/diff/log/search/code/prose)",
|
|
661
800
|
)
|
|
801
|
+
parser.add_argument(
|
|
802
|
+
"--mode",
|
|
803
|
+
choices=COMPRESSION_MODES,
|
|
804
|
+
default="conservative",
|
|
805
|
+
help="compression policy: conservative keeps existing deterministic strategies; readable adds opt-in readable preview/source-verification metadata",
|
|
806
|
+
)
|
|
662
807
|
parser.add_argument("--json", action="store_true", help="emit JSON with metadata and compressed content")
|
|
663
808
|
parser.add_argument(
|
|
664
809
|
"--protected-policy",
|