@inneranimalmedia/agentsam-sdk 1.8.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DEVELOPMENT.md +1 -1
- package/docs/RELEASES.iam-mirror.md +20 -0
- package/docs/RELEASES.md +4 -1
- package/package.json +2 -2
- package/python/agentsam_sdk/cli.py +57 -7
- package/python/agentsam_sdk/data/d1_bloat.py +279 -99
- package/python/agentsam_sdk/repository/__init__.py +26 -0
- package/python/agentsam_sdk/repository/inspect.py +496 -0
- package/python/docs/gaps.md +1 -1
- package/python/tests/test_d1_bloat.py +60 -24
- package/python/tests/test_repository_inspect.py +84 -0
package/DEVELOPMENT.md
CHANGED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# Agent Sam SDK — release pairings (IAM home)
|
|
2
|
+
|
|
3
|
+
**Status:** Evergreen receipts · **Product:** [README.md](./README.md)
|
|
4
|
+
|
|
5
|
+
Canonical npm-side table also lives in the SDK repo: `docs/RELEASES.md`.
|
|
6
|
+
**Keep both updated** when publishing (dual-home of the receipt).
|
|
7
|
+
|
|
8
|
+
| npm version | Published (UTC) | IAM git SHA (40) | Notes |
|
|
9
|
+
|-------------|-----------------|------------------|-------|
|
|
10
|
+
| 1.7.0 | 2026-07-14 | _(pre-receipt)_ | Prior publish |
|
|
11
|
+
| 1.9.0 | _(pending publish)_ | `fc1628505cb3c9946c149e4c468c2e264d6f381e` | `repository.inspect` + `--dupes`; mirrored from IAM agentsam-sdk |
|
|
12
|
+
| 1.8.0 | 2026-08-02T13:57:12.080Z | `580547301e370b77ec26bd72558979d335feedd9` | `python/` + `protocol/` in tarball; `repository.scan_bloat`; shell-kit under `packages/` |
|
|
13
|
+
|
|
14
|
+
**Latest pairing (ready to publish):** `iam@fc1628505cb3c9946c149e4c468c2e264d6f381e` ↔ `@inneranimalmedia/agentsam-sdk@1.9.0`
|
|
15
|
+
|
|
16
|
+
### Receipt rules
|
|
17
|
+
|
|
18
|
+
- Full 40-char SHA only (AGENTS.md §3)
|
|
19
|
+
- Record the IAM SHA that introduced or mirrored the consumer-facing change
|
|
20
|
+
- Tag SDK repo `vX.Y.Z` on publish
|
package/docs/RELEASES.md
CHANGED
|
@@ -3,4 +3,7 @@
|
|
|
3
3
|
| npm version | Published (UTC) | IAM git SHA (40) | Notes |
|
|
4
4
|
|-------------|-----------------|------------------|-------|
|
|
5
5
|
| 1.7.0 | 2026-07-14 | _(pre-receipt)_ | Prior publish |
|
|
6
|
-
| 1.
|
|
6
|
+
| 1.9.0 | _(pending publish)_ | `fc1628505cb3c9946c149e4c468c2e264d6f381e` | `python/agentsam_sdk/repository/inspect.py` (`--dupes`); CLI `agentsam repository inspect` |
|
|
7
|
+
| 1.8.0 | 2026-08-02T13:57:12.080Z | `580547301e370b77ec26bd72558979d335feedd9` | `python/` + `protocol/` in npm `files`; `repository.scan_bloat`; shell-kit folded to `packages/agentsam-shell-kit` (private). SDK git at publish: `ac52669b89ddb16fad87c42fe53f2c93ef13bd1a`. |
|
|
8
|
+
|
|
9
|
+
**Pairing (ready to publish):** `iam@fc1628505cb3c9946c149e4c468c2e264d6f381e` ↔ `@inneranimalmedia/agentsam-sdk@1.9.0`
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@inneranimalmedia/agentsam-sdk",
|
|
3
|
-
"version": "1.
|
|
4
|
-
"description": "Agent Sam is a full-stack AI agent SDK for autonomous task execution
|
|
3
|
+
"version": "1.9.0",
|
|
4
|
+
"description": "Agent Sam is a full-stack AI agent SDK for autonomous task execution \u2014 covering data management, creative workflows, design commands, and multi-step agentic pipelines.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.js",
|
|
7
7
|
"exports": {
|
|
@@ -6,6 +6,7 @@ convention: Python tooling here is stdlib-only).
|
|
|
6
6
|
agentsam repository inventory --repo-root .. --output-dir /tmp/scan --format json
|
|
7
7
|
agentsam repository inventory --repo-root .. --format json | jq '.data.totals'
|
|
8
8
|
agentsam repository scan-bloat --root src --min-kb 10 --top 30 --format json
|
|
9
|
+
agentsam repository inspect --repo-root . --json --dupes | jq '.duplicates'
|
|
9
10
|
"""
|
|
10
11
|
from __future__ import annotations
|
|
11
12
|
|
|
@@ -54,8 +55,7 @@ def _cmd_data_d1_bloat(args: argparse.Namespace) -> int:
|
|
|
54
55
|
mode=mode,
|
|
55
56
|
params={
|
|
56
57
|
"db": args.db, "config": args.config, "repo_root": args.repo_root,
|
|
57
|
-
"prefix": args.prefix, "workers": args.workers,
|
|
58
|
-
"count_only": args.count_only, "top": args.top,
|
|
58
|
+
"prefix": args.prefix, "workers": args.workers, "top": args.top,
|
|
59
59
|
},
|
|
60
60
|
output_dir=args.output_dir,
|
|
61
61
|
)
|
|
@@ -123,6 +123,36 @@ def _cmd_repository_scan_bloat(args: argparse.Namespace) -> int:
|
|
|
123
123
|
return 0 if result.ok else 1
|
|
124
124
|
|
|
125
125
|
|
|
126
|
+
def _cmd_repository_inspect(args: argparse.Namespace) -> int:
|
|
127
|
+
from agentsam_sdk.repository import inspect as repo_inspect
|
|
128
|
+
|
|
129
|
+
argv: list[str] = []
|
|
130
|
+
if args.repo_root:
|
|
131
|
+
argv.extend(["--repo-root", args.repo_root])
|
|
132
|
+
want_text = bool(args.text) or args.format in ("text", "markdown")
|
|
133
|
+
want_json = bool(args.json) or args.format == "json" or not want_text
|
|
134
|
+
if want_json and not want_text:
|
|
135
|
+
argv.append("--json")
|
|
136
|
+
if want_text and not want_json:
|
|
137
|
+
argv.append("--text")
|
|
138
|
+
if want_text and want_json:
|
|
139
|
+
# Explicit both → JSON wins (machine default) unless only --text
|
|
140
|
+
argv.append("--json")
|
|
141
|
+
if args.dupes:
|
|
142
|
+
argv.append("--dupes")
|
|
143
|
+
if args.all:
|
|
144
|
+
argv.append("--all")
|
|
145
|
+
if args.since:
|
|
146
|
+
argv.extend(["--since", args.since])
|
|
147
|
+
if args.recent is not None:
|
|
148
|
+
argv.extend(["--recent", str(args.recent)])
|
|
149
|
+
if args.largest is not None:
|
|
150
|
+
argv.extend(["--largest", str(args.largest)])
|
|
151
|
+
if args.out:
|
|
152
|
+
argv.extend(["--out", args.out])
|
|
153
|
+
return repo_inspect.main_cli(argv)
|
|
154
|
+
|
|
155
|
+
|
|
126
156
|
def build_parser() -> argparse.ArgumentParser:
|
|
127
157
|
ap = argparse.ArgumentParser(prog="agentsam", description="agentsam_sdk CLI")
|
|
128
158
|
sub = ap.add_subparsers(dest="group", required=True)
|
|
@@ -130,16 +160,20 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
130
160
|
data = sub.add_parser("data", help="D1 data audits")
|
|
131
161
|
data_sub = data.add_subparsers(dest="cmd", required=True)
|
|
132
162
|
|
|
133
|
-
bloat = data_sub.add_parser(
|
|
163
|
+
bloat = data_sub.add_parser(
|
|
164
|
+
"d1-bloat",
|
|
165
|
+
help="database-scoped D1 audit (--quick=counts, --full=text sizes)",
|
|
166
|
+
)
|
|
134
167
|
bloat.add_argument("--db", help="D1 database name (else AGENTSAM_D1_DB_NAME)")
|
|
135
168
|
bloat.add_argument("--config", help="wrangler config path (else AGENTSAM_WRANGLER_CONFIG)")
|
|
136
169
|
bloat.add_argument("--repo-root", help="repo root wrangler runs from")
|
|
137
|
-
bloat.add_argument("--quick", action="store_true", default=True
|
|
138
|
-
|
|
139
|
-
bloat.add_argument("--
|
|
170
|
+
bloat.add_argument("--quick", action="store_true", default=True,
|
|
171
|
+
help="All tables: COUNT(*) only (default)")
|
|
172
|
+
bloat.add_argument("--full", action="store_true",
|
|
173
|
+
help="All tables: row counts + text/JSON LENGTH estimates")
|
|
174
|
+
bloat.add_argument("--prefix", help="Only tables whose name starts with this prefix")
|
|
140
175
|
bloat.add_argument("--workers", type=int, default=6)
|
|
141
176
|
bloat.add_argument("--top", type=int, default=40)
|
|
142
|
-
bloat.add_argument("--count-only", action="store_true")
|
|
143
177
|
bloat.add_argument("--output-dir")
|
|
144
178
|
bloat.add_argument("--format", choices=["json", "markdown"], default="markdown")
|
|
145
179
|
bloat.set_defaults(func=_cmd_data_d1_bloat)
|
|
@@ -199,6 +233,22 @@ def build_parser() -> argparse.ArgumentParser:
|
|
|
199
233
|
)
|
|
200
234
|
sb.set_defaults(func=_cmd_repository_scan_bloat)
|
|
201
235
|
|
|
236
|
+
insp = repo_sub.add_parser(
|
|
237
|
+
"inspect",
|
|
238
|
+
help="file walk: sizes + dates (+ optional --dupes SHA-256 groups)",
|
|
239
|
+
)
|
|
240
|
+
insp.add_argument("--repo-root", default=None)
|
|
241
|
+
insp.add_argument("--format", choices=["json", "markdown", "text"], default="json")
|
|
242
|
+
insp.add_argument("--json", action="store_true")
|
|
243
|
+
insp.add_argument("--text", action="store_true")
|
|
244
|
+
insp.add_argument("--dupes", action="store_true")
|
|
245
|
+
insp.add_argument("--all", action="store_true")
|
|
246
|
+
insp.add_argument("--since", default=None)
|
|
247
|
+
insp.add_argument("--recent", type=int, default=50)
|
|
248
|
+
insp.add_argument("--largest", type=int, default=30)
|
|
249
|
+
insp.add_argument("--out", default=None)
|
|
250
|
+
insp.set_defaults(func=_cmd_repository_inspect)
|
|
251
|
+
|
|
202
252
|
return ap
|
|
203
253
|
|
|
204
254
|
|
|
@@ -1,54 +1,56 @@
|
|
|
1
1
|
"""agentsam_sdk.data.d1_bloat -- port of scripts/d1_bloat_audit.py.
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
3
|
+
Database-scoped D1 size audit (not tenant/workspace). Walks one CF D1 database.
|
|
4
|
+
|
|
5
|
+
Modes:
|
|
6
|
+
quick — every user table + COUNT(*) only
|
|
7
|
+
full — same tables + SUM(LENGTH(...)) on text-ish columns + briefing
|
|
8
|
+
|
|
9
|
+
D1 remote has no dbstat; sizes are LENGTH estimates, not exact page bytes.
|
|
5
10
|
|
|
6
11
|
Deferred from the legacy script (see docs/gaps.md): --email/Resend delivery.
|
|
7
|
-
This module writes json + markdown only
|
|
8
|
-
if still wanted -- keeps this module free of a RESEND_API_KEY dependency.
|
|
12
|
+
This module writes json + markdown only.
|
|
9
13
|
"""
|
|
10
14
|
from __future__ import annotations
|
|
11
15
|
|
|
16
|
+
import json
|
|
12
17
|
import re
|
|
13
18
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
14
|
-
from dataclasses import dataclass, field
|
|
19
|
+
from dataclasses import asdict, dataclass, field
|
|
15
20
|
from datetime import datetime, timezone
|
|
16
|
-
from pathlib import Path
|
|
17
21
|
from typing import Any, Optional
|
|
18
22
|
|
|
19
23
|
from agentsam_sdk.data.d1_adapter import D1Adapter, D1AdapterError
|
|
20
|
-
from agentsam_sdk.runtime.contract import ToolInput, ToolResult,
|
|
24
|
+
from agentsam_sdk.runtime.contract import ToolInput, ToolResult, start_timer, write_receipt
|
|
21
25
|
|
|
22
26
|
TOOL_NAME = "data.d1_bloat"
|
|
23
27
|
|
|
28
|
+
# Prefer payload-ish TEXT columns when measuring LENGTH (full mode).
|
|
24
29
|
BLOAT_COL_RE = re.compile(
|
|
25
30
|
r"(body|content|value|markdown|_json\b|schema|payload|output|prompt|message|"
|
|
26
31
|
r"text|config|metadata|description|notes|script|summary|arguments|result|"
|
|
27
32
|
r"attributes|events|resource|handler|input_|output_|sql\b|embedding|merged_)",
|
|
28
33
|
re.I,
|
|
29
34
|
)
|
|
30
|
-
SKIP_COL_RE = re.compile(
|
|
31
|
-
r"(^id$|_id$|_at$|_at_epoch$|_hash$|_key$|_uuid$|_ref$|_url$|_path$|_email$|"
|
|
32
|
-
r"_slug$|_name$|_type$|_status$|_mode$|_token$|tenant_id|workspace_id|user_id)",
|
|
33
|
-
re.I,
|
|
34
|
-
)
|
|
35
|
-
QUICK_TABLE_RE = re.compile(
|
|
36
|
-
r"^(agentsam_|otlp_|system_health|deployment|terminal_|cms_|worker_analytics|"
|
|
37
|
-
r"ai_api_test|dashboard_versions|semantic_search)",
|
|
38
|
-
re.I,
|
|
39
|
-
)
|
|
40
35
|
|
|
41
|
-
# Table-name -> rollup guidance. Naming convention hints, not identity/secrets;
|
|
42
|
-
# safe to ship. Extend freely -- this is documentation, not config.
|
|
43
36
|
ROLLUP_HINTS: dict[str, str] = {
|
|
44
37
|
"agentsam_tool_call_log": "Archive/purge output_json + input_json >30d; keep output_summary + ids.",
|
|
45
|
-
"agentsam_tool_chain": "result_json dominates
|
|
46
|
-
"agentsam_tool_cache": "
|
|
47
|
-
"
|
|
48
|
-
"
|
|
49
|
-
"
|
|
50
|
-
"
|
|
51
|
-
"
|
|
38
|
+
"agentsam_tool_chain": "result_json dominates — rollup to R2 or truncate; keep summaries.",
|
|
39
|
+
"agentsam_tool_cache": "Enforce TTL + max rows; output_json must not grow unbounded.",
|
|
40
|
+
"agentsam_mcp_tool_execution": "Archive old output_json; mirror tool_call_log policy.",
|
|
41
|
+
"agentsam_execution_steps": "Archive input/output JSON after workflow completes.",
|
|
42
|
+
"agentsam_workflow_runs": "Move step_results_json to R2; D1 row = pointer + status + cost.",
|
|
43
|
+
"agentsam_webhook_events": "Rollup payload_json; retain type + ts + external id.",
|
|
44
|
+
"agentsam_scripts": "body must stay empty; canonical source in R2 (source_stored=r2:…).",
|
|
45
|
+
"agentsam_skill": "Large SKILL.md → R2; D1 = metadata + retrieval_strategy=r2.",
|
|
46
|
+
"agentsam_memory": "Archive stale prose; vectors live in Supabase/Vectorize, not D1.",
|
|
47
|
+
"agentsam_rules_document": "body_markdown → R2; D1 = trigger + key + short summary.",
|
|
48
|
+
"agentsam_cron_runs": "Trim metadata_json on old runs.",
|
|
49
|
+
"agentsam_hook_execution": "Archive payload_json; keep hook id + status.",
|
|
50
|
+
"agentsam_eval_runs": "Cap grader notes; long artifacts → Supabase eval tables.",
|
|
51
|
+
"otlp_traces": "Retention on attributes_json; sample or export off-D1.",
|
|
52
|
+
"terminal_history_archive_431": "Archive scrollback to R2 or cap rows per connection.",
|
|
53
|
+
"system_health_snapshots": "Shorten retention or aggregate further.",
|
|
52
54
|
}
|
|
53
55
|
|
|
54
56
|
|
|
@@ -73,40 +75,55 @@ class TableStat:
|
|
|
73
75
|
if self.name in ROLLUP_HINTS:
|
|
74
76
|
return ROLLUP_HINTS[self.name]
|
|
75
77
|
if self.text_bytes > 500_000 and any(
|
|
76
|
-
c.name
|
|
78
|
+
c.name
|
|
79
|
+
in (
|
|
80
|
+
"body",
|
|
81
|
+
"content_markdown",
|
|
82
|
+
"value",
|
|
83
|
+
"output_json",
|
|
84
|
+
"result_json",
|
|
85
|
+
"payload_json",
|
|
86
|
+
)
|
|
77
87
|
for c in self.columns
|
|
78
88
|
):
|
|
79
|
-
return "Large text/JSON in D1
|
|
89
|
+
return "Large text/JSON in D1 — prefer R2 pointer + vector lanes for search."
|
|
80
90
|
return None
|
|
81
91
|
|
|
82
92
|
|
|
83
|
-
def
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
if
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
return
|
|
93
|
+
def _pick_measure_columns(cols: list[tuple[str, str]], max_cols: int = 12) -> list[str]:
|
|
94
|
+
"""TEXT/BLOB/JSON columns to LENGTH-scan. Prefer payload-ish names; else any text cols."""
|
|
95
|
+
textish = [
|
|
96
|
+
n
|
|
97
|
+
for n, typ in cols
|
|
98
|
+
if (typ or "TEXT").upper() in ("TEXT", "BLOB", "JSON") or "CHAR" in (typ or "").upper()
|
|
99
|
+
]
|
|
100
|
+
preferred = [n for n in textish if BLOAT_COL_RE.search(n)]
|
|
101
|
+
chosen = preferred if preferred else textish
|
|
102
|
+
return chosen[:max_cols]
|
|
93
103
|
|
|
94
104
|
|
|
95
105
|
def _scan_table(adapter: D1Adapter, table: str, analyze_text: bool) -> TableStat:
|
|
96
106
|
stat = TableStat(name=table)
|
|
97
107
|
try:
|
|
108
|
+
if not analyze_text:
|
|
109
|
+
stat.row_count = adapter.row_count(table)
|
|
110
|
+
stat.est_bytes = stat.row_count * 120
|
|
111
|
+
return stat
|
|
112
|
+
|
|
98
113
|
cols = adapter.table_columns(table)
|
|
99
|
-
|
|
100
|
-
if
|
|
101
|
-
parts = [f'SUM(LENGTH(COALESCE("{c}", \'\'))) AS "{c}"' for c in
|
|
102
|
-
max_parts = [f'MAX(LENGTH(COALESCE("{c}", \'\'))) AS "m_{c}"' for c in
|
|
114
|
+
measure = _pick_measure_columns(cols)
|
|
115
|
+
if measure:
|
|
116
|
+
parts = [f'SUM(LENGTH(COALESCE("{c}", \'\'))) AS "{c}"' for c in measure]
|
|
117
|
+
max_parts = [f'MAX(LENGTH(COALESCE("{c}", \'\'))) AS "m_{c}"' for c in measure]
|
|
103
118
|
sql = f'SELECT COUNT(*) AS rc, {", ".join(parts + max_parts)} FROM "{table}"'
|
|
104
119
|
row = adapter.query(sql)[0]
|
|
105
120
|
stat.row_count = int(row.get("rc") or 0)
|
|
106
|
-
for c in
|
|
121
|
+
for c in measure:
|
|
107
122
|
b = int(row.get(c) or 0)
|
|
108
123
|
if b:
|
|
109
|
-
stat.columns.append(
|
|
124
|
+
stat.columns.append(
|
|
125
|
+
ColStat(name=c, bytes=b, max_len=int(row.get(f"m_{c}") or 0))
|
|
126
|
+
)
|
|
110
127
|
stat.text_bytes = sum(c.bytes for c in stat.columns)
|
|
111
128
|
stat.est_bytes = stat.text_bytes if stat.text_bytes else stat.row_count * 120
|
|
112
129
|
else:
|
|
@@ -127,10 +144,45 @@ def _fmt_bytes(n: int) -> str:
|
|
|
127
144
|
return f"{n} B"
|
|
128
145
|
|
|
129
146
|
|
|
130
|
-
def
|
|
131
|
-
|
|
132
|
-
|
|
147
|
+
def _build_findings(stats: list[TableStat], mode: str) -> list[dict[str, Any]]:
|
|
148
|
+
findings: list[dict[str, Any]] = []
|
|
149
|
+
if mode == "quick":
|
|
150
|
+
ranked = sorted(stats, key=lambda s: s.row_count, reverse=True)
|
|
151
|
+
for s in ranked[:40]:
|
|
152
|
+
if s.error:
|
|
153
|
+
findings.append(
|
|
154
|
+
{
|
|
155
|
+
"severity": "medium",
|
|
156
|
+
"table": s.name,
|
|
157
|
+
"rows": s.row_count,
|
|
158
|
+
"est_bytes": 0,
|
|
159
|
+
"est_human": "—",
|
|
160
|
+
"why": f"scan_error: {s.error[:120]}",
|
|
161
|
+
"next_steps": ["Re-run --full for this table after fixing access."],
|
|
162
|
+
}
|
|
163
|
+
)
|
|
164
|
+
continue
|
|
165
|
+
if s.row_count < 50_000:
|
|
166
|
+
continue
|
|
167
|
+
sev = "high" if s.row_count >= 100_000 else "medium"
|
|
168
|
+
findings.append(
|
|
169
|
+
{
|
|
170
|
+
"severity": sev,
|
|
171
|
+
"table": s.name,
|
|
172
|
+
"rows": s.row_count,
|
|
173
|
+
"est_bytes": s.est_bytes,
|
|
174
|
+
"est_human": _fmt_bytes(s.est_bytes),
|
|
175
|
+
"why": f"High row count ({s.row_count:,}) — run --full to size text/JSON.",
|
|
176
|
+
"next_steps": [
|
|
177
|
+
f"agentsam data d1-bloat --full --prefix {s.name}",
|
|
178
|
+
"Confirm retention / archive policy for this table.",
|
|
179
|
+
],
|
|
180
|
+
}
|
|
181
|
+
)
|
|
182
|
+
return findings
|
|
183
|
+
|
|
133
184
|
total_text = sum(s.text_bytes for s in stats) or 1
|
|
185
|
+
ranked = sorted(stats, key=lambda s: s.text_bytes or s.est_bytes, reverse=True)
|
|
134
186
|
for s in ranked[:80]:
|
|
135
187
|
est = s.text_bytes or s.est_bytes
|
|
136
188
|
reasons: list[str] = []
|
|
@@ -144,121 +196,249 @@ def _flag_suspicious(stats: list[TableStat]) -> list[dict[str, Any]]:
|
|
|
144
196
|
reasons.append("known_rollup_candidate")
|
|
145
197
|
if s.error:
|
|
146
198
|
reasons.append(f"scan_error:{s.error[:80]}")
|
|
147
|
-
if reasons:
|
|
148
|
-
|
|
149
|
-
|
|
199
|
+
if not reasons:
|
|
200
|
+
continue
|
|
201
|
+
steps = []
|
|
202
|
+
if s.rollup_hint:
|
|
203
|
+
steps.append(s.rollup_hint)
|
|
204
|
+
else:
|
|
205
|
+
steps.append("Inspect top text columns; archive or move large payloads to R2.")
|
|
206
|
+
steps.append("Cap writers at source (result ceilings / retention).")
|
|
207
|
+
findings.append(
|
|
208
|
+
{
|
|
209
|
+
"severity": "high"
|
|
210
|
+
if est >= 5_000_000 or s.row_count >= 100_000
|
|
211
|
+
else "medium",
|
|
212
|
+
"table": s.name,
|
|
213
|
+
"rows": s.row_count,
|
|
214
|
+
"est_bytes": est,
|
|
150
215
|
"est_human": _fmt_bytes(est),
|
|
151
|
-
"
|
|
152
|
-
"
|
|
153
|
-
|
|
154
|
-
|
|
216
|
+
"why": "; ".join(reasons),
|
|
217
|
+
"next_steps": steps,
|
|
218
|
+
"top_columns": [
|
|
219
|
+
{"name": c.name, "bytes": c.bytes, "human": _fmt_bytes(c.bytes)}
|
|
220
|
+
for c in sorted(s.columns, key=lambda x: x.bytes, reverse=True)[:5]
|
|
221
|
+
],
|
|
222
|
+
}
|
|
223
|
+
)
|
|
224
|
+
return findings
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _build_doing_well(
|
|
228
|
+
stats: list[TableStat], findings: list[dict[str, Any]]
|
|
229
|
+
) -> list[dict[str, str]]:
|
|
230
|
+
flagged = {f["table"] for f in findings}
|
|
231
|
+
well: list[dict[str, str]] = []
|
|
232
|
+
small = [s for s in stats if not s.error and s.row_count < 1_000 and s.name not in flagged]
|
|
233
|
+
if small:
|
|
234
|
+
well.append(
|
|
235
|
+
{
|
|
236
|
+
"area": "small_tables",
|
|
237
|
+
"why": f"{len(small)} tables under 1k rows and not flagged — fine for registry/config.",
|
|
238
|
+
}
|
|
239
|
+
)
|
|
240
|
+
empty = sum(1 for s in stats if not s.error and s.row_count == 0)
|
|
241
|
+
if empty:
|
|
242
|
+
well.append(
|
|
243
|
+
{
|
|
244
|
+
"area": "empty_tables",
|
|
245
|
+
"why": f"{empty} empty tables (candidates to drop later, not urgent).",
|
|
246
|
+
}
|
|
247
|
+
)
|
|
248
|
+
if not findings:
|
|
249
|
+
well.append(
|
|
250
|
+
{"area": "no_flags", "why": "No high/medium bloat heuristics fired on this pass."}
|
|
251
|
+
)
|
|
252
|
+
return well
|
|
155
253
|
|
|
156
254
|
|
|
157
|
-
def
|
|
255
|
+
def _build_briefing(
|
|
256
|
+
stats: list[TableStat],
|
|
257
|
+
findings: list[dict[str, Any]],
|
|
258
|
+
doing_well: list[dict[str, str]],
|
|
259
|
+
db_name: str,
|
|
260
|
+
db_size: Optional[str],
|
|
261
|
+
mode: str,
|
|
262
|
+
table_total: int,
|
|
263
|
+
) -> str:
|
|
264
|
+
high = sum(1 for f in findings if f["severity"] == "high")
|
|
265
|
+
med = sum(1 for f in findings if f["severity"] == "medium")
|
|
266
|
+
verdict = "needs_attention" if high or med else "healthy"
|
|
158
267
|
lines = [
|
|
159
|
-
"# D1
|
|
268
|
+
f"# D1 health — {db_name}",
|
|
160
269
|
"",
|
|
161
270
|
f"- **Generated:** {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}",
|
|
271
|
+
f"- **Mode:** {mode} (database-scoped; not tenant/workspace)",
|
|
162
272
|
f"- **Reported DB size:** {db_size or 'unknown'}",
|
|
163
|
-
f"- **
|
|
164
|
-
f"- **
|
|
165
|
-
f"- **Tables scanned:** {scanned}",
|
|
166
|
-
f"- **Estimated text payload (scanned):** {_fmt_bytes(sum(s.text_bytes for s in stats))}",
|
|
167
|
-
"",
|
|
168
|
-
"> D1 remote has no `dbstat`. Sizes are `SUM(LENGTH(text_col))` estimates.",
|
|
169
|
-
"",
|
|
170
|
-
"## Top tables by estimated text bytes",
|
|
273
|
+
f"- **Tables:** {table_total} scanned",
|
|
274
|
+
f"- **Verdict:** {verdict} ({high} high / {med} medium findings)",
|
|
171
275
|
"",
|
|
172
|
-
"| Rank | Table | Rows | Text est. | Top columns | Rollup hint |",
|
|
173
|
-
"|------|-------|------|-----------|-------------|-------------|",
|
|
174
276
|
]
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
277
|
+
if mode == "quick":
|
|
278
|
+
lines.append(
|
|
279
|
+
"> Quick = row counts only. Run `--full` for text/JSON size estimates and rollup guidance."
|
|
280
|
+
)
|
|
281
|
+
lines.append("")
|
|
282
|
+
else:
|
|
283
|
+
lines.append(
|
|
284
|
+
f"> Full text estimate (scanned columns): {_fmt_bytes(sum(s.text_bytes for s in stats))}."
|
|
180
285
|
)
|
|
181
|
-
hint = (s.rollup_hint or "").replace("|", "/")[:80]
|
|
182
|
-
err = f" ⚠ {s.error}" if s.error else ""
|
|
183
286
|
lines.append(
|
|
184
|
-
|
|
185
|
-
f"{top_cols or '—'} | {hint or '—'}{err} |"
|
|
287
|
+
"> D1 remote has no `dbstat` — sizes are `SUM(LENGTH(...))`, not exact page bytes."
|
|
186
288
|
)
|
|
289
|
+
lines.append("")
|
|
290
|
+
|
|
291
|
+
lines.append("## What's fine")
|
|
292
|
+
lines.append("")
|
|
293
|
+
if doing_well:
|
|
294
|
+
for w in doing_well:
|
|
295
|
+
lines.append(f"- **{w['area']}:** {w['why']}")
|
|
296
|
+
else:
|
|
297
|
+
lines.append("- (nothing notable)")
|
|
298
|
+
lines.append("")
|
|
299
|
+
|
|
300
|
+
lines.append("## What's bloated / needs attention")
|
|
301
|
+
lines.append("")
|
|
302
|
+
if not findings:
|
|
303
|
+
lines.append("- None flagged on this pass.")
|
|
304
|
+
else:
|
|
305
|
+
for i, f in enumerate(findings[:25], 1):
|
|
306
|
+
lines.append(
|
|
307
|
+
f"{i}. **`{f['table']}`** [{f['severity']}] — "
|
|
308
|
+
f"rows={f['rows']:,}, est={f['est_human']}"
|
|
309
|
+
)
|
|
310
|
+
lines.append(f" - Why: {f['why']}")
|
|
311
|
+
for step in f.get("next_steps") or []:
|
|
312
|
+
lines.append(f" - Next: {step}")
|
|
313
|
+
lines.append("")
|
|
314
|
+
|
|
315
|
+
lines.append("## Next steps (global)")
|
|
316
|
+
lines.append("")
|
|
317
|
+
if mode == "quick":
|
|
318
|
+
lines.append("1. `agentsam data d1-bloat --full --format json` for size + column detail.")
|
|
319
|
+
lines.append("2. Prioritize tables with ≥100k rows from the inventory above.")
|
|
320
|
+
else:
|
|
321
|
+
lines.append("1. Act on high findings first (archive / R2 / writer caps).")
|
|
322
|
+
lines.append("2. Re-run `--quick` weekly for row-count drift; `--full` after large ingest.")
|
|
323
|
+
lines.append("3. Prefer D1 for pointers + state; R2 for bytes; Vectorize/Supabase for search.")
|
|
187
324
|
lines.append("")
|
|
188
325
|
return "\n".join(lines)
|
|
189
326
|
|
|
190
327
|
|
|
191
328
|
def run(tool_input: ToolInput) -> ToolResult:
|
|
192
329
|
started = start_timer()
|
|
330
|
+
tool_input.assert_read_only()
|
|
193
331
|
p = tool_input.params
|
|
194
332
|
mode = tool_input.mode if tool_input.mode in ("quick", "full") else "quick"
|
|
195
333
|
prefix = p.get("prefix")
|
|
196
334
|
workers = int(p.get("workers", 6))
|
|
197
|
-
count_only = bool(p.get("count_only", False))
|
|
198
335
|
top = int(p.get("top", 40))
|
|
336
|
+
analyze_text = mode == "full"
|
|
199
337
|
|
|
200
338
|
output_dir = tool_input.output_path()
|
|
201
339
|
|
|
202
340
|
try:
|
|
203
341
|
adapter = D1Adapter.from_env(
|
|
204
|
-
db_name=p.get("db"),
|
|
342
|
+
db_name=p.get("db"),
|
|
343
|
+
wrangler_config=p.get("config"),
|
|
344
|
+
repo_root=p.get("repo_root"),
|
|
205
345
|
)
|
|
206
346
|
all_tables = adapter.list_tables()
|
|
347
|
+
tables = all_tables
|
|
207
348
|
if prefix:
|
|
208
|
-
tables = [t for t in all_tables if t.lower().startswith(prefix.lower())]
|
|
209
|
-
elif mode == "quick":
|
|
210
|
-
tables = [t for t in all_tables if QUICK_TABLE_RE.match(t)]
|
|
211
|
-
else:
|
|
212
|
-
tables = all_tables
|
|
349
|
+
tables = [t for t in all_tables if t.lower().startswith(str(prefix).lower())]
|
|
213
350
|
|
|
214
351
|
db_size = adapter.database_size()
|
|
215
352
|
stats: list[TableStat] = []
|
|
216
353
|
with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
|
|
217
|
-
futures = {
|
|
354
|
+
futures = {
|
|
355
|
+
pool.submit(_scan_table, adapter, t, analyze_text): t for t in tables
|
|
356
|
+
}
|
|
218
357
|
for fut in as_completed(futures):
|
|
219
358
|
try:
|
|
220
359
|
stats.append(fut.result())
|
|
221
360
|
except Exception as e: # noqa: BLE001
|
|
222
361
|
stats.append(TableStat(name=futures[fut], error=str(e)))
|
|
223
362
|
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
363
|
+
if mode == "quick":
|
|
364
|
+
stats.sort(key=lambda s: s.row_count, reverse=True)
|
|
365
|
+
else:
|
|
366
|
+
stats.sort(key=lambda s: s.text_bytes or s.est_bytes, reverse=True)
|
|
367
|
+
|
|
368
|
+
findings = _build_findings(stats, mode)
|
|
369
|
+
doing_well = _build_doing_well(stats, findings)
|
|
370
|
+
high = sum(1 for f in findings if f["severity"] == "high")
|
|
371
|
+
med = sum(1 for f in findings if f["severity"] == "medium")
|
|
372
|
+
verdict = "needs_attention" if high or med else "healthy"
|
|
373
|
+
md = _build_briefing(
|
|
374
|
+
stats, findings, doing_well, adapter.db_name, db_size, mode, len(tables)
|
|
375
|
+
)
|
|
227
376
|
|
|
228
|
-
artifacts: list[str] = []
|
|
229
377
|
json_payload = {
|
|
230
378
|
"database": adapter.db_name,
|
|
231
|
-
"
|
|
379
|
+
"scope": "database",
|
|
232
380
|
"mode": mode,
|
|
381
|
+
"verdict": verdict,
|
|
382
|
+
"database_size": db_size,
|
|
233
383
|
"tables_total": len(all_tables),
|
|
234
384
|
"tables_scanned": len(tables),
|
|
235
385
|
"estimated_text_bytes": sum(s.text_bytes for s in stats),
|
|
236
|
-
"
|
|
386
|
+
"doing_well": doing_well,
|
|
387
|
+
"findings": findings,
|
|
388
|
+
"next_steps_global": (
|
|
389
|
+
[
|
|
390
|
+
"agentsam data d1-bloat --full --format json",
|
|
391
|
+
"Act on tables with ≥100k rows first",
|
|
392
|
+
]
|
|
393
|
+
if mode == "quick"
|
|
394
|
+
else [
|
|
395
|
+
"Act on high findings (archive / R2 / writer caps)",
|
|
396
|
+
"Re-run --quick weekly for row drift",
|
|
397
|
+
]
|
|
398
|
+
),
|
|
237
399
|
"tables": [
|
|
238
|
-
{
|
|
239
|
-
|
|
400
|
+
{
|
|
401
|
+
**{k: v for k, v in asdict(s).items() if k != "columns"},
|
|
402
|
+
"columns": [asdict(c) for c in s.columns],
|
|
403
|
+
"rollup_hint": s.rollup_hint,
|
|
404
|
+
}
|
|
240
405
|
for s in stats[:top]
|
|
241
406
|
],
|
|
242
407
|
}
|
|
408
|
+
|
|
409
|
+
artifacts: list[str] = []
|
|
243
410
|
if output_dir:
|
|
244
411
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
245
412
|
(output_dir / "d1-bloat.md").write_text(md, encoding="utf-8")
|
|
246
413
|
(output_dir / "d1-bloat.json").write_text(
|
|
247
|
-
|
|
414
|
+
json.dumps(json_payload, indent=2), encoding="utf-8"
|
|
248
415
|
)
|
|
249
416
|
artifacts = [str(output_dir / "d1-bloat.md"), str(output_dir / "d1-bloat.json")]
|
|
250
417
|
|
|
251
418
|
result = ToolResult(
|
|
252
|
-
ok=True,
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
419
|
+
ok=True,
|
|
420
|
+
tool=TOOL_NAME,
|
|
421
|
+
mode=mode,
|
|
422
|
+
request_id=tool_input.request_id,
|
|
423
|
+
started_at=started,
|
|
424
|
+
finished_at=start_timer(),
|
|
425
|
+
summary=(
|
|
426
|
+
f"Scanned {len(tables)}/{len(all_tables)} tables "
|
|
427
|
+
f"(mode={mode}, verdict={verdict}, findings={len(findings)})."
|
|
428
|
+
),
|
|
429
|
+
data=json_payload,
|
|
430
|
+
artifacts=artifacts,
|
|
256
431
|
)
|
|
257
432
|
except D1AdapterError as e:
|
|
258
433
|
result = ToolResult(
|
|
259
|
-
ok=False,
|
|
260
|
-
|
|
261
|
-
|
|
434
|
+
ok=False,
|
|
435
|
+
tool=TOOL_NAME,
|
|
436
|
+
mode=mode,
|
|
437
|
+
request_id=tool_input.request_id,
|
|
438
|
+
started_at=started,
|
|
439
|
+
finished_at=start_timer(),
|
|
440
|
+
summary="D1 adapter error",
|
|
441
|
+
error=str(e),
|
|
262
442
|
)
|
|
263
443
|
|
|
264
444
|
write_receipt(result, output_dir)
|