@cliwant/mcp-sam-gov 0.2.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (228) hide show
  1. package/LICENSE +21 -21
  2. package/README.ja.md +48 -16
  3. package/README.ko.md +48 -16
  4. package/README.md +279 -67
  5. package/dist/attachments.d.ts +63 -0
  6. package/dist/attachments.d.ts.map +1 -0
  7. package/dist/attachments.js +576 -0
  8. package/dist/attachments.js.map +1 -0
  9. package/dist/bls.d.ts +267 -0
  10. package/dist/bls.d.ts.map +1 -0
  11. package/dist/bls.js +1549 -0
  12. package/dist/bls.js.map +1 -0
  13. package/dist/cache.d.ts +42 -0
  14. package/dist/cache.d.ts.map +1 -0
  15. package/dist/cache.js +64 -0
  16. package/dist/cache.js.map +1 -0
  17. package/dist/census.d.ts +182 -0
  18. package/dist/census.d.ts.map +1 -0
  19. package/dist/census.js +576 -0
  20. package/dist/census.js.map +1 -0
  21. package/dist/ckan.d.ts +141 -0
  22. package/dist/ckan.d.ts.map +1 -0
  23. package/dist/ckan.js +393 -0
  24. package/dist/ckan.js.map +1 -0
  25. package/dist/clinicaltrials.d.ts +180 -0
  26. package/dist/clinicaltrials.d.ts.map +1 -0
  27. package/dist/clinicaltrials.js +730 -0
  28. package/dist/clinicaltrials.js.map +1 -0
  29. package/dist/cms.d.ts +140 -0
  30. package/dist/cms.d.ts.map +1 -0
  31. package/dist/cms.js +482 -0
  32. package/dist/cms.js.map +1 -0
  33. package/dist/coerce.d.ts +32 -0
  34. package/dist/coerce.d.ts.map +1 -0
  35. package/dist/coerce.js +50 -0
  36. package/dist/coerce.js.map +1 -0
  37. package/dist/datagov-catalog.d.ts +84 -0
  38. package/dist/datagov-catalog.d.ts.map +1 -0
  39. package/dist/datagov-catalog.js +233 -0
  40. package/dist/datagov-catalog.js.map +1 -0
  41. package/dist/datagov.d.ts +146 -0
  42. package/dist/datagov.d.ts.map +1 -0
  43. package/dist/datagov.js +689 -0
  44. package/dist/datagov.js.map +1 -0
  45. package/dist/datagovKey.d.ts +36 -0
  46. package/dist/datagovKey.d.ts.map +1 -0
  47. package/dist/datagovKey.js +60 -0
  48. package/dist/datagovKey.js.map +1 -0
  49. package/dist/datasource.d.ts +313 -0
  50. package/dist/datasource.d.ts.map +1 -0
  51. package/dist/datasource.js +551 -0
  52. package/dist/datasource.js.map +1 -0
  53. package/dist/disclosure.d.ts +55 -0
  54. package/dist/disclosure.d.ts.map +1 -0
  55. package/dist/disclosure.js +57 -0
  56. package/dist/disclosure.js.map +1 -0
  57. package/dist/ecfr.d.ts +4 -2
  58. package/dist/ecfr.d.ts.map +1 -1
  59. package/dist/ecfr.js +92 -17
  60. package/dist/ecfr.js.map +1 -1
  61. package/dist/echo.d.ts +143 -0
  62. package/dist/echo.d.ts.map +1 -0
  63. package/dist/echo.js +424 -0
  64. package/dist/echo.js.map +1 -0
  65. package/dist/edgar.d.ts +377 -0
  66. package/dist/edgar.d.ts.map +1 -0
  67. package/dist/edgar.js +2418 -0
  68. package/dist/edgar.js.map +1 -0
  69. package/dist/errors.d.ts +102 -0
  70. package/dist/errors.d.ts.map +1 -0
  71. package/dist/errors.js +247 -0
  72. package/dist/errors.js.map +1 -0
  73. package/dist/fac.d.ts +180 -0
  74. package/dist/fac.d.ts.map +1 -0
  75. package/dist/fac.js +416 -0
  76. package/dist/fac.js.map +1 -0
  77. package/dist/far.d.ts +170 -0
  78. package/dist/far.d.ts.map +1 -0
  79. package/dist/far.js +804 -0
  80. package/dist/far.js.map +1 -0
  81. package/dist/fdic.d.ts +599 -0
  82. package/dist/fdic.d.ts.map +1 -0
  83. package/dist/fdic.js +1624 -0
  84. package/dist/fdic.js.map +1 -0
  85. package/dist/federal-register.d.ts +139 -2
  86. package/dist/federal-register.d.ts.map +1 -1
  87. package/dist/federal-register.js +432 -15
  88. package/dist/federal-register.js.map +1 -1
  89. package/dist/fema.d.ts +181 -0
  90. package/dist/fema.d.ts.map +1 -0
  91. package/dist/fema.js +436 -0
  92. package/dist/fema.js.map +1 -0
  93. package/dist/fpds.d.ts +108 -0
  94. package/dist/fpds.d.ts.map +1 -0
  95. package/dist/fpds.js +519 -0
  96. package/dist/fpds.js.map +1 -0
  97. package/dist/gao.d.ts +64 -0
  98. package/dist/gao.d.ts.map +1 -0
  99. package/dist/gao.js +640 -0
  100. package/dist/gao.js.map +1 -0
  101. package/dist/govinfo.d.ts +111 -0
  102. package/dist/govinfo.d.ts.map +1 -0
  103. package/dist/govinfo.js +422 -0
  104. package/dist/govinfo.js.map +1 -0
  105. package/dist/grants.d.ts +27 -4
  106. package/dist/grants.d.ts.map +1 -1
  107. package/dist/grants.js +114 -11
  108. package/dist/grants.js.map +1 -1
  109. package/dist/gsa-csv.d.ts +249 -0
  110. package/dist/gsa-csv.d.ts.map +1 -0
  111. package/dist/gsa-csv.js +784 -0
  112. package/dist/gsa-csv.js.map +1 -0
  113. package/dist/integrity.d.ts +212 -0
  114. package/dist/integrity.d.ts.map +1 -0
  115. package/dist/integrity.js +707 -0
  116. package/dist/integrity.js.map +1 -0
  117. package/dist/meta.d.ts +165 -0
  118. package/dist/meta.d.ts.map +1 -0
  119. package/dist/meta.js +162 -0
  120. package/dist/meta.js.map +1 -0
  121. package/dist/nih.d.ts +117 -0
  122. package/dist/nih.d.ts.map +1 -0
  123. package/dist/nih.js +291 -0
  124. package/dist/nih.js.map +1 -0
  125. package/dist/nppes.d.ts +157 -0
  126. package/dist/nppes.d.ts.map +1 -0
  127. package/dist/nppes.js +648 -0
  128. package/dist/nppes.js.map +1 -0
  129. package/dist/nsf.d.ts +176 -0
  130. package/dist/nsf.d.ts.map +1 -0
  131. package/dist/nsf.js +554 -0
  132. package/dist/nsf.js.map +1 -0
  133. package/dist/nvd.d.ts +176 -0
  134. package/dist/nvd.d.ts.map +1 -0
  135. package/dist/nvd.js +912 -0
  136. package/dist/nvd.js.map +1 -0
  137. package/dist/ofac.d.ts +205 -0
  138. package/dist/ofac.d.ts.map +1 -0
  139. package/dist/ofac.js +919 -0
  140. package/dist/ofac.js.map +1 -0
  141. package/dist/pricing.d.ts +110 -0
  142. package/dist/pricing.d.ts.map +1 -0
  143. package/dist/pricing.js +843 -0
  144. package/dist/pricing.js.map +1 -0
  145. package/dist/sam-gov/client.d.ts +60 -2
  146. package/dist/sam-gov/client.d.ts.map +1 -1
  147. package/dist/sam-gov/client.js +320 -54
  148. package/dist/sam-gov/client.js.map +1 -1
  149. package/dist/sam-gov/index.d.ts +1 -1
  150. package/dist/sam-gov/index.d.ts.map +1 -1
  151. package/dist/sam-gov/index.js +1 -1
  152. package/dist/sam-gov/index.js.map +1 -1
  153. package/dist/sam-gov/types.d.ts +24 -0
  154. package/dist/sam-gov/types.d.ts.map +1 -1
  155. package/dist/sba.d.ts +72 -0
  156. package/dist/sba.d.ts.map +1 -0
  157. package/dist/sba.js +281 -0
  158. package/dist/sba.js.map +1 -0
  159. package/dist/server.d.ts +14 -2
  160. package/dist/server.d.ts.map +1 -1
  161. package/dist/server.js +3897 -295
  162. package/dist/server.js.map +1 -1
  163. package/dist/snapshot.d.ts +98 -0
  164. package/dist/snapshot.d.ts.map +1 -0
  165. package/dist/snapshot.js +146 -0
  166. package/dist/snapshot.js.map +1 -0
  167. package/dist/socrata.d.ts +157 -0
  168. package/dist/socrata.d.ts.map +1 -0
  169. package/dist/socrata.js +448 -0
  170. package/dist/socrata.js.map +1 -0
  171. package/dist/treasury.d.ts +143 -0
  172. package/dist/treasury.d.ts.map +1 -0
  173. package/dist/treasury.js +436 -0
  174. package/dist/treasury.js.map +1 -0
  175. package/dist/usaspending.d.ts +260 -65
  176. package/dist/usaspending.d.ts.map +1 -1
  177. package/dist/usaspending.js +1664 -228
  178. package/dist/usaspending.js.map +1 -1
  179. package/dist/usitc.d.ts +142 -0
  180. package/dist/usitc.d.ts.map +1 -0
  181. package/dist/usitc.js +339 -0
  182. package/dist/usitc.js.map +1 -0
  183. package/package.json +24 -2
  184. package/src/attachments.ts +652 -0
  185. package/src/bls.ts +1943 -0
  186. package/src/cache.ts +73 -0
  187. package/src/census.ts +735 -0
  188. package/src/ckan.ts +495 -0
  189. package/src/clinicaltrials.ts +923 -0
  190. package/src/cms.ts +634 -0
  191. package/src/coerce.ts +47 -0
  192. package/src/datagov-catalog.ts +296 -0
  193. package/src/datagov.ts +907 -0
  194. package/src/datagovKey.ts +68 -0
  195. package/src/datasource.ts +721 -0
  196. package/src/disclosure.ts +61 -0
  197. package/src/ecfr.ts +231 -127
  198. package/src/echo.ts +496 -0
  199. package/src/edgar.ts +3014 -0
  200. package/src/errors.ts +303 -0
  201. package/src/fac.ts +529 -0
  202. package/src/far.ts +1007 -0
  203. package/src/fdic.ts +2052 -0
  204. package/src/federal-register.ts +706 -191
  205. package/src/fema.ts +541 -0
  206. package/src/fpds.ts +620 -0
  207. package/src/gao.ts +744 -0
  208. package/src/govinfo.ts +497 -0
  209. package/src/grants.ts +290 -155
  210. package/src/gsa-csv.ts +992 -0
  211. package/src/integrity.ts +928 -0
  212. package/src/meta.ts +292 -0
  213. package/src/nih.ts +375 -0
  214. package/src/nppes.ts +834 -0
  215. package/src/nsf.ts +706 -0
  216. package/src/nvd.ts +1124 -0
  217. package/src/ofac.ts +1166 -0
  218. package/src/pricing.ts +1075 -0
  219. package/src/sam-gov/client.ts +345 -63
  220. package/src/sam-gov/index.ts +5 -1
  221. package/src/sam-gov/types.ts +22 -0
  222. package/src/sba.ts +357 -0
  223. package/src/server.ts +4559 -327
  224. package/src/snapshot.ts +192 -0
  225. package/src/socrata.ts +532 -0
  226. package/src/treasury.ts +575 -0
  227. package/src/usaspending.ts +2680 -925
  228. package/src/usitc.ts +420 -0
package/src/edgar.ts ADDED
@@ -0,0 +1,3014 @@
1
+ /**
2
+ * SEC EDGAR — company filings, XBRL financial facts, ticker→CIK, and full-text
3
+ * search (keyless). First capital-markets source (ADR-0003); 2nd consumer of the
4
+ * fetch/map/meta shape after treasury.ts.
5
+ *
6
+ * Fully PUBLIC, KEYLESS. Hosts: data.sec.gov, efts.sec.gov, www.sec.gov/files
7
+ * (+ www.sec.gov/Archives for constructed links). No API key; instead SEC
8
+ * requires a descriptive `User-Agent` on EVERY request and self-throttling to
9
+ * ≤10 req/s (a breach → ~10-minute IP block).
10
+ *
11
+ * Three layers (mirror treasury.ts):
12
+ * fetch — `getEdgar(url, label)`: sets the mandatory UA (+ gzip) on the init,
13
+ * serializes every EDGAR fetch through a per-process min-interval gate
14
+ * (~110ms → ≤~9 req/s), and reuses errors.ts retry/timeout/taxonomy.
15
+ * map — PURE columnar/curated mappers (zipRecent, companyfacts extraction,
16
+ * FTS hit map). `num(x)` → number|null (NEVER 0 for absent). `padCik`.
17
+ * meta — `withMeta(...)`: hands totalAvailable/returned/pagination/notes to
18
+ * meta.ts's buildMeta, which DERIVES complete/truncated.
19
+ *
20
+ * HONESTY / REVIEW FIXES (ADR-0003 "Review outcome (v2)", live-verified 2026-07-10):
21
+ * F1 — the committed default UA is an ORG contact on the project domain
22
+ * (`cliwant-mcp-sam-gov/1.0 (contact: opendata@cliwant.com)`), NOT a
23
+ * github.com URL (SEC 403s any UA containing `github.com`) and NOT a
24
+ * personal email. Override via EDGAR_USER_AGENT.
25
+ * F2 — `edgar_full_text_search` has NO `size` param: efts ignores it (5/20/100/
26
+ * 200 all return 100), so page size is a fixed 100. Pagination is by `from`.
27
+ * F3 — FTS window overflow: `from > 9900` (from+100 > 10000) is rejected as
28
+ * invalid_input BEFORE the fetch; and after r.json(), a missing `hits.hits`
29
+ * (SEC returns HTTP 200 + `{message:"Internal server error"}` past the
30
+ * window) is thrown as schema_drift, never crashed on.
31
+ * F4 — default curated concepts DO NOT include EarningsPerShareBasic (its unit
32
+ * is `USD/shares`, so the default `unit="USD"` would silently return
33
+ * nothing). A requested concept present only in another unit → a note,
34
+ * never a silent/fabricated 0.
35
+ * F5 — FTS `hits.total.relation === "gte"` → `totalIsLowerBound:true` passed to
36
+ * withMeta (machine-readable, not only a note). See meta.ts.
37
+ * F6 — getEdgar disambiguates a 403 by reading the body: "automated"/
38
+ * "Undeclared" ⇒ invalid_input (bad UA, don't retry); else ⇒ rate_limited,
39
+ * retryable, retryAfterSeconds 600 (the ~10-min block).
40
+ * F7 — FTS `_source` has NO primary-document filename; the output `filingIndexUrl`
41
+ * is the filing's ARCHIVE INDEX directory built from `adsh` (no fabricated
42
+ * doc URL). `edgar_company_filings` DOES use the real `primaryDocument`.
43
+ *
44
+ * CIK↔UEI JOIN CAVEAT (load-bearing, in every tool's `_meta.notes` +
45
+ * fieldsUnavailable): EDGAR keys on 10-digit SEC CIK, NOT SAM UEI/DUNS or a
46
+ * USAspending recipient id. No reliable programmatic CIK↔UEI join exists;
47
+ * bridging an EDGAR filer to a federal contracting entity is name/ticker-only
48
+ * (fuzzy) and MUST NOT be asserted as authoritative.
49
+ */
50
+
51
+ import { fetchWithRetry, ToolErrorCarrier, errorFromResponse } from "./errors.js";
52
+ import { throughGate, driftError } from "./datasource.js";
53
+ import { memoize } from "./cache.js";
54
+ import { withMeta, type MetaBundle, type ResponseMeta } from "./meta.js";
55
+
56
+ // ─── Hosts + the mandatory User-Agent (F1) ────────────────────────
57
+ const SUBMISSIONS_BASE = "https://data.sec.gov/submissions";
58
+ const XBRL_FACTS_BASE = "https://data.sec.gov/api/xbrl/companyfacts";
59
+ const TICKERS_URL = "https://www.sec.gov/files/company_tickers.json";
60
+ const FTS_URL = "https://efts.sec.gov/LATEST/search-index";
61
+ const ARCHIVES_BASE = "https://www.sec.gov/Archives/edgar/data";
62
+
63
+ /**
64
+ * The descriptive User-Agent SEC mandates on every request (Policy①). Default is
65
+ * an ORG contact on the project domain — live-verified 200 (F1). It must NOT
66
+ * contain `github.com` (SEC 403s that on all 3 hosts) and must NOT be a personal
67
+ * email. Operators override with the `EDGAR_USER_AGENT` env var (SEC's suggested
68
+ * form: "Sample Company AdminContact@example.com").
69
+ */
70
+ const EDGAR_UA =
71
+ process.env.EDGAR_USER_AGENT ??
72
+ "cliwant-mcp-sam-gov/1.0 (contact: opendata@cliwant.com)";
73
+
74
+ const EDGAR_HEADERS = {
75
+ "User-Agent": EDGAR_UA,
76
+ "Accept-Encoding": "gzip",
77
+ } as const;
78
+
79
+ const EDGAR_SOURCE = "data.sec.gov / efts.sec.gov (keyless)";
80
+ const EDGAR_FIELDS_UNAVAILABLE = ["uei", "duns", "sam_recipient_id"];
81
+ const CIK_UEI_CAVEAT =
82
+ "EDGAR identifies filers by 10-digit SEC CIK, NOT SAM UEI/DUNS or a USAspending recipient id. There is no reliable programmatic CIK↔UEI join — bridging an EDGAR filer to a federal contracting entity is name/ticker-only (fuzzy) and must NOT be treated as authoritative.";
83
+
84
+ // ─── Min-interval gate (self-throttle ≤10 req/s) ──────────────────
85
+ // A tool call can fan out (lookup→submissions), and the process may run many
86
+ // tools; serialize EVERY EDGAR fetch through one promise chain with a ≥110ms
87
+ // spacing (~9 req/s < the 10 req/s ceiling → no ~10-min block). The chain +
88
+ // spacing math now live in the shared `throughGate("edgar", 110, fn)` primitive
89
+ // (datasource.ts, ADR-0011 R2 deferred slice) — same single-chain, same
90
+ // lastAt-stamped-before-fn, same bare-`setTimeout` global (so the fault tests'
91
+ // timer-neutralizing patch keeps it instant offline). Behavior is byte-identical
92
+ // to the former module singleton this replaced.
93
+ const EDGAR_MIN_INTERVAL_MS = 110;
94
+
95
+ // ─── fetch layer ──────────────────────────────────────────────────
96
+ /**
97
+ * GET one EDGAR resource with the mandatory UA (+ gzip) and a 15s timeout,
98
+ * serialized through the min-interval gate. Reuses errors.ts for the retry/
99
+ * backoff on transient 429/5xx/network faults.
100
+ *
101
+ * F6 — a 403 from SEC is ambiguous (a missing/undeclared UA vs a rate-block).
102
+ * Read the body to disambiguate BEFORE the generic errorFromResponse mislabels
103
+ * it as a plain invalid_input:
104
+ * body ~ /automated|undeclared/i ⇒ invalid_input, non-retryable (fix the UA)
105
+ * otherwise ⇒ rate_limited, retryable, retryAfter 600s
106
+ * Common non-retryable statuses (404/400) throw immediately; 404 in particular
107
+ * is caught by the caller and turned into an honest `found:false`.
108
+ */
109
+ async function getEdgar(url: string, label: string): Promise<Response> {
110
+ const init: RequestInit = {
111
+ headers: { ...EDGAR_HEADERS },
112
+ signal: AbortSignal.timeout(15_000),
113
+ };
114
+ return throughGate("edgar", EDGAR_MIN_INTERVAL_MS, async () => {
115
+ let r: Response;
116
+ try {
117
+ r = await fetch(url, init);
118
+ } catch {
119
+ // Network-level fault (DNS/reset/timeout). Reuse the shared retry/backoff
120
+ // with a fresh AbortSignal per attempt.
121
+ return fetchWithRetry(
122
+ url,
123
+ { headers: { ...EDGAR_HEADERS }, signal: AbortSignal.timeout(15_000) },
124
+ label,
125
+ );
126
+ }
127
+ if (r.ok) return r;
128
+ if (r.status === 403) {
129
+ const body = await r.text().catch(() => "");
130
+ if (/automated|undeclared/i.test(body)) {
131
+ throw new ToolErrorCarrier({
132
+ kind: "invalid_input",
133
+ message: `SEC EDGAR rejected the request as automated/undeclared (HTTP 403) at ${label}. SEC requires a descriptive, non-github User-Agent — set EDGAR_USER_AGENT (e.g. "Sample Company AdminContact@example.com"). Upstream note: ${body.slice(0, 160)}`,
134
+ retryable: false,
135
+ upstreamStatus: 403,
136
+ upstreamEndpoint: label,
137
+ });
138
+ }
139
+ throw new ToolErrorCarrier({
140
+ kind: "rate_limited",
141
+ message: `SEC EDGAR returned HTTP 403 at ${label} — likely the 10 req/s limit / a ~10-minute IP block. Slow down and retry after ~10 minutes.`,
142
+ retryable: true,
143
+ retryAfterSeconds: 600,
144
+ upstreamStatus: 403,
145
+ upstreamEndpoint: label,
146
+ });
147
+ }
148
+ const err = errorFromResponse(r, label);
149
+ if (err.retryable) {
150
+ // 429 / 5xx — reuse the shared retry/backoff machinery.
151
+ return fetchWithRetry(
152
+ url,
153
+ { headers: { ...EDGAR_HEADERS }, signal: AbortSignal.timeout(15_000) },
154
+ label,
155
+ );
156
+ }
157
+ // 404 / 400 (non-retryable) — throw now; callers translate 404 → found:false.
158
+ throw new ToolErrorCarrier(err);
159
+ });
160
+ }
161
+
162
+ // ─── shared coercions ─────────────────────────────────────────────
163
+ /**
164
+ * A CIK (int or string, padded or not) → the canonical 10-digit zero-padded
165
+ * form EDGAR's data.sec.gov paths require. Strips non-digits first (accepts
166
+ * "CIK320193", "320193", 320193). e.g. 320193 → "0000320193".
167
+ */
168
+ export function padCik(x: string | number): string {
169
+ const digits = String(x).replace(/\D/g, "");
170
+ return digits.padStart(10, "0");
171
+ }
172
+
173
+ /** The un-padded CIK for Archives URLs (SEC drops leading zeros there). */
174
+ function unpadCik(cik10: string): string {
175
+ return cik10.replace(/^0+/, "") || "0";
176
+ }
177
+
178
+ /**
179
+ * Coerce an XBRL value to number|null. Returns **null (never 0)** for absent —
180
+ * a missing fact is an honest "unknown", never a fabricated zero. XBRL `val` is
181
+ * already a JSON number, but guard strings/nullish/non-finite defensively.
182
+ */
183
+ export function num(x: unknown): number | null {
184
+ if (x === null || x === undefined) return null;
185
+ if (typeof x === "number") return Number.isFinite(x) ? x : null;
186
+ if (typeof x === "string") {
187
+ const s = x.trim();
188
+ if (s === "" || s.toLowerCase() === "null") return null;
189
+ const n = Number(s);
190
+ return Number.isFinite(n) ? n : null;
191
+ }
192
+ return null;
193
+ }
194
+
195
+ /** null for absent (null/undefined/""), else the trimmed string. */
196
+ function str(x: unknown): string | null {
197
+ if (x === null || x === undefined) return null;
198
+ const s = String(x).trim();
199
+ return s === "" ? null : s;
200
+ }
201
+
202
+ // ─── ticker → CIK map (memoized ~6h) ──────────────────────────────
203
+ export type TickerEntry = { cik: string; ticker: string; title: string };
204
+
205
+ async function fetchTickerMap(): Promise<TickerEntry[]> {
206
+ const r = await getEdgar(TICKERS_URL, "edgar:tickers");
207
+ const d = (await r.json()) as unknown;
208
+ if (!d || typeof d !== "object") {
209
+ throw new ToolErrorCarrier({
210
+ kind: "schema_drift",
211
+ message:
212
+ "edgar:tickers returned an unexpected shape (company_tickers.json should be a keyed dict of {cik_str,ticker,title}).",
213
+ retryable: false,
214
+ upstreamEndpoint: "edgar:tickers",
215
+ });
216
+ }
217
+ const out: TickerEntry[] = [];
218
+ for (const v of Object.values(d as Record<string, unknown>)) {
219
+ const e = v as { cik_str?: unknown; ticker?: unknown; title?: unknown };
220
+ if (e && e.cik_str != null && e.ticker != null) {
221
+ out.push({
222
+ cik: padCik(e.cik_str as string | number),
223
+ ticker: String(e.ticker),
224
+ title: e.title == null ? "" : String(e.title),
225
+ });
226
+ }
227
+ }
228
+ return out;
229
+ }
230
+
231
+ /** The full ticker→CIK table, memoized ~6h (slow-changing reference read). */
232
+ async function tickerMap(): Promise<TickerEntry[]> {
233
+ return memoize("edgar:tickers", fetchTickerMap, 6 * 60 * 60 * 1000);
234
+ }
235
+
236
+ /**
237
+ * Resolve a `cikOrTicker` argument to a 10-digit CIK. Pure digits (or "CIK…")
238
+ * → padCik directly (no network). Otherwise look up the memoized ticker map:
239
+ * exact ticker match first, then a case-insensitive title substring. null when
240
+ * nothing matches (caller → honest found:false).
241
+ */
242
+ async function resolveCik(
243
+ cikOrTicker: string,
244
+ ): Promise<{ cik: string; ticker: string | null; title: string | null } | null> {
245
+ const raw = cikOrTicker.trim();
246
+ if (/^(cik)?\s*\d+$/i.test(raw)) {
247
+ return { cik: padCik(raw), ticker: null, title: null };
248
+ }
249
+ const map = await tickerMap();
250
+ const upper = raw.toUpperCase();
251
+ const exact = map.find((e) => e.ticker.toUpperCase() === upper);
252
+ if (exact) return { cik: exact.cik, ticker: exact.ticker, title: exact.title };
253
+ const byName = map.find((e) => e.title.toUpperCase().includes(upper));
254
+ if (byName) return { cik: byName.cik, ticker: byName.ticker, title: byName.title };
255
+ return null;
256
+ }
257
+
258
+ // ─── meta helper ──────────────────────────────────────────────────
259
+ /**
260
+ * Build a partial `_meta` with the EDGAR source + the mandatory CIK↔UEI caveat
261
+ * appended to every tool's notes and `fieldsUnavailable` set. Tool-specific
262
+ * signals (returned/totalAvailable/pagination/totalIsLowerBound/complete) are
263
+ * merged in via `extra`; buildMeta DERIVES complete/truncated from them.
264
+ */
265
+ function edgarMeta(extra: Partial<ResponseMeta>): Partial<ResponseMeta> {
266
+ return {
267
+ source: EDGAR_SOURCE,
268
+ keylessMode: true,
269
+ filtersApplied: extra.filtersApplied ?? [],
270
+ filtersDropped: [],
271
+ fieldsUnavailable: EDGAR_FIELDS_UNAVAILABLE,
272
+ ...extra,
273
+ notes: [...(extra.notes ?? []), CIK_UEI_CAVEAT],
274
+ };
275
+ }
276
+
277
+ /** Honest empty for an unresolved/unknown filer: a definitive answer (complete). */
278
+ function notFoundBundle(identifier: string, note: string): MetaBundle {
279
+ return withMeta(
280
+ { found: false, identifier },
281
+ edgarMeta({
282
+ returned: 0,
283
+ totalAvailable: 0,
284
+ complete: true,
285
+ notes: [note],
286
+ }),
287
+ );
288
+ }
289
+
290
+ // ─── Tool 1: edgar_lookup_cik ─────────────────────────────────────
291
+ /**
292
+ * Map a company query (exact ticker, else title substring) to its 10-digit CIK
293
+ * via the memoized company_tickers.json. Returns up to 50 matches; `found:false`
294
+ * on none. The CIK is the join key the other three tools take.
295
+ */
296
+ export async function lookupCik(args: { query: string }): Promise<MetaBundle> {
297
+ const q = args.query.trim();
298
+ const map = await tickerMap();
299
+ const upper = q.toUpperCase();
300
+ const exact = map.filter((e) => e.ticker.toUpperCase() === upper);
301
+ const matched = exact.length
302
+ ? exact
303
+ : map.filter((e) => e.title.toUpperCase().includes(upper));
304
+ const LIMIT = 50;
305
+ const results = matched
306
+ .slice(0, LIMIT)
307
+ .map((e) => ({ cik: e.cik, ticker: e.ticker, title: e.title }));
308
+ const notes: string[] = [];
309
+ if (matched.length === 0) {
310
+ notes.push(
311
+ `No company in SEC's ticker registry matched "${q}" (exact ticker or title substring). Not every SEC filer has a ticker; try edgar_full_text_search by name.`,
312
+ );
313
+ } else {
314
+ notes.push(
315
+ exact.length
316
+ ? `Matched by exact ticker "${q.toUpperCase()}".`
317
+ : `Matched by title substring "${q}" (case-insensitive).`,
318
+ );
319
+ if (matched.length > LIMIT) {
320
+ notes.push(
321
+ `${matched.length} companies matched; returning the first ${LIMIT}. Refine the query for a specific filer.`,
322
+ );
323
+ }
324
+ }
325
+ return withMeta(
326
+ { found: results.length > 0, query: q, results },
327
+ edgarMeta({
328
+ returned: results.length,
329
+ totalAvailable: matched.length,
330
+ filtersApplied: ["query"],
331
+ notes,
332
+ }),
333
+ );
334
+ }
335
+
336
+ // ─── Tool 2: edgar_company_filings ────────────────────────────────
337
+ export type Filing = {
338
+ accession: string | null;
339
+ form: string | null;
340
+ filingDate: string | null;
341
+ reportDate: string | null;
342
+ primaryDocument: string | null;
343
+ primaryDocDescription: string | null;
344
+ primaryDocUrl: string | null;
345
+ isXBRL: boolean;
346
+ };
347
+
348
+ type SubmissionsRecent = Record<string, unknown[]>;
349
+ type SubmissionsShard = {
350
+ name?: string;
351
+ filingCount?: number;
352
+ filingFrom?: string;
353
+ filingTo?: string;
354
+ };
355
+
356
+ /**
357
+ * The fixed grammar for a `filings.files[].name` older-submissions shard (ADR-0019).
358
+ * Live-confirmed 2026-07-12: `CIK0000320193-submissions-001.json`,
359
+ * `CIK0000019617-submissions-068.json`. This is the FIRST-pass SSRF grammar guard
360
+ * (SEC-format check); `fetchShard` ALSO builds a stronger CIK-BOUND regex from the
361
+ * resolved parent CIK (M3) so a name for ANY OTHER CIK is refused before any fetch.
362
+ * The `\d{1,4}` shard-number quantifier is bounded (a giant number just 404s); the
363
+ * `%`/`/`/`.`/`\`/`..` traversal characters are simply not in the allowed classes.
364
+ */
365
+ const SHARD_NAME_RE = /^CIK\d{10}-submissions-\d{1,4}\.json$/;
366
+
367
+ /**
368
+ * Unroll the COLUMNAR `filings.recent` (parallel arrays; index i = one filing)
369
+ * into `Filing[]`, constructing the primary-document ARCHIVE URL from the real
370
+ * accession + primaryDocument. Alignment is strictly by index i.
371
+ */
372
+ function zipRecent(recent: SubmissionsRecent, cik10: string): Filing[] {
373
+ const accs = (recent.accessionNumber as string[]) ?? [];
374
+ const cikUnpadded = unpadCik(cik10);
375
+ const out: Filing[] = [];
376
+ for (let i = 0; i < accs.length; i++) {
377
+ const accession = str(accs[i]);
378
+ const primaryDocument = str(recent.primaryDocument?.[i]);
379
+ let primaryDocUrl: string | null = null;
380
+ if (accession && primaryDocument) {
381
+ const accNoDash = accession.replace(/-/g, "");
382
+ primaryDocUrl = `${ARCHIVES_BASE}/${cikUnpadded}/${accNoDash}/${primaryDocument}`;
383
+ }
384
+ out.push({
385
+ accession,
386
+ form: str(recent.form?.[i]),
387
+ filingDate: str(recent.filingDate?.[i]),
388
+ reportDate: str(recent.reportDate?.[i]),
389
+ primaryDocument,
390
+ primaryDocDescription: str(recent.primaryDocDescription?.[i]),
391
+ primaryDocUrl,
392
+ isXBRL: recent.isXBRL?.[i] === 1 || recent.isXBRL?.[i] === true,
393
+ });
394
+ }
395
+ return out;
396
+ }
397
+
398
+ async function fetchSubmissions(cik10: string): Promise<{
399
+ name: string | null;
400
+ recent: SubmissionsRecent;
401
+ files: SubmissionsShard[];
402
+ recentCount: number;
403
+ }> {
404
+ const url = `${SUBMISSIONS_BASE}/CIK${cik10}.json`;
405
+ const r = await getEdgar(url, "edgar:submissions");
406
+ const d = (await r.json()) as {
407
+ name?: string;
408
+ filings?: { recent?: SubmissionsRecent; files?: SubmissionsShard[] };
409
+ };
410
+ const recent = d?.filings?.recent;
411
+ if (!recent || !Array.isArray(recent.accessionNumber)) {
412
+ throw new ToolErrorCarrier({
413
+ kind: "schema_drift",
414
+ message: `edgar:submissions returned an unexpected shape for CIK ${cik10} (filings.recent.accessionNumber array missing).`,
415
+ retryable: false,
416
+ upstreamEndpoint: "edgar:submissions",
417
+ });
418
+ }
419
+ const files = Array.isArray(d.filings?.files) ? d.filings!.files! : [];
420
+ return {
421
+ name: str(d.name),
422
+ recent,
423
+ files,
424
+ recentCount: recent.accessionNumber.length,
425
+ };
426
+ }
427
+
428
+ /**
429
+ * The 7 parallel-array columns `zipRecent` reads by index i. The INTRA-shard
430
+ * shape guard (M1 — the REAL alignment guard, the `pts===data.length` analogue):
431
+ * a shard 200 body whose `accessionNumber` is missing/non-array, or any of these
432
+ * 7 arrays is not `=== accessionNumber.length`, would let `zipRecent` borrow a
433
+ * value from a shorter/absent column at index i → a fabricated (form,date,accession)
434
+ * tuple. So a non-columnar/ragged shard THROWS schema_drift (never emits rows).
435
+ */
436
+ const SHARD_ZIP_COLUMNS = [
437
+ "accessionNumber",
438
+ "form",
439
+ "filingDate",
440
+ "reportDate",
441
+ "primaryDocument",
442
+ "primaryDocDescription",
443
+ "isXBRL",
444
+ ] as const;
445
+
446
+ /**
447
+ * Fetch ONE older-submissions shard and return its `Filing[]` (ADR-0019). The
448
+ * shard `name` is SERVER-PROVIDED (from the parent `filings.files[].name`); the
449
+ * caller never constructs it from user input. Belt-and-suspenders, mirroring
450
+ * `buildFramesUrl` (S1/S2):
451
+ * - **[M3] CIK-BIND + grammar validate BEFORE any URL is built.** The name must
452
+ * match BOTH the fixed SEC grammar (`SHARD_NAME_RE`) AND a regex bound to THIS
453
+ * filer's resolved `cik10` — so a drifted/hostile/MITM'd parent whose name
454
+ * embeds a DIFFERENT CIK (`CIK9999999999-…`) or a traversal (`../`, `%2F`) is
455
+ * NEVER fetched (thrown as `not_found` → the caller skips + discloses PARTIAL).
456
+ * - Fixed host: build `${SUBMISSIONS_BASE}/${name}` and assert the parsed URL is
457
+ * `https://data.sec.gov` (host-escape/downgrade guard).
458
+ * - `getEdgar` → the shard body; **[M1]** assert the columnar shape (SHARD_ZIP_COLUMNS
459
+ * all equal-length) or THROW schema_drift; then `zipRecent`.
460
+ * Throw taxonomy the caller relies on: `not_found` (bad/cross-CIK/host name OR a
461
+ * genuine 404) → skip+PARTIAL; `schema_drift`/`rate_limited`/other → re-throw loud.
462
+ */
463
+ async function fetchShard(name: string, cik10: string): Promise<Filing[]> {
464
+ // [M3] CIK-bound grammar — ONLY a shard whose embedded CIK === this filer's CIK
465
+ // is eligible. cik10 is 10 zero-padded DIGITS (padCik strips non-digits), so it
466
+ // is safe to interpolate into a RegExp source (no metacharacters).
467
+ const cikBoundRe = new RegExp(`^CIK${cik10}-submissions-\\d{1,4}\\.json$`);
468
+ if (!SHARD_NAME_RE.test(name) || !cikBoundRe.test(name)) {
469
+ throw new ToolErrorCarrier({
470
+ kind: "not_found",
471
+ message: `edgar:submissions shard name ${JSON.stringify(name)} did not match the CIK-bound grammar ^CIK${cik10}-submissions-\\d{1,4}\\.json$ (SEC format change, or a cross-entity/hostile name) — refused before any fetch, counted as a skipped shard.`,
472
+ retryable: false,
473
+ upstreamEndpoint: "edgar:submissions",
474
+ });
475
+ }
476
+ const built = `${SUBMISSIONS_BASE}/${name}`;
477
+ let parsed: URL;
478
+ try {
479
+ parsed = new URL(built);
480
+ } catch {
481
+ throw new ToolErrorCarrier({
482
+ kind: "not_found",
483
+ message: `edgar:submissions shard URL could not be constructed from ${JSON.stringify(name)} — refused before any fetch.`,
484
+ retryable: false,
485
+ upstreamEndpoint: "edgar:submissions",
486
+ });
487
+ }
488
+ if (parsed.protocol !== "https:" || parsed.hostname !== "data.sec.gov") {
489
+ throw new ToolErrorCarrier({
490
+ kind: "not_found",
491
+ message: `edgar:submissions shard URL host/scheme is not https://data.sec.gov (${parsed.protocol}//${parsed.hostname}) — refused before any fetch (fixed-host assertion).`,
492
+ retryable: false,
493
+ upstreamEndpoint: "edgar:submissions",
494
+ });
495
+ }
496
+ const r = await getEdgar(built, "edgar:submissions");
497
+ const shard = (await r.json()) as SubmissionsRecent;
498
+ // [M1] Intra-shard columnar-shape guard (the real alignment guard).
499
+ const acc = shard?.accessionNumber;
500
+ if (!Array.isArray(acc)) {
501
+ throw new ToolErrorCarrier({
502
+ kind: "schema_drift",
503
+ message: `edgar:submissions shard ${name} returned HTTP 200 without a columnar accessionNumber[] array — the submissions shard envelope changed.`,
504
+ retryable: false,
505
+ upstreamEndpoint: "edgar:submissions",
506
+ });
507
+ }
508
+ for (const col of SHARD_ZIP_COLUMNS) {
509
+ const arr = shard[col];
510
+ if (!Array.isArray(arr) || arr.length !== acc.length) {
511
+ throw new ToolErrorCarrier({
512
+ kind: "schema_drift",
513
+ message: `edgar:submissions shard ${name} is not columnar: '${col}' is ${Array.isArray(arr) ? `length ${arr.length}` : "missing/non-array"} ≠ accessionNumber length ${acc.length}. Emitting index-aligned rows would fabricate (form,date,accession) tuples — refusing.`,
514
+ retryable: false,
515
+ upstreamEndpoint: "edgar:submissions",
516
+ });
517
+ }
518
+ }
519
+ return zipRecent(shard, cik10);
520
+ }
521
+
522
+ /**
523
+ * A company's SEC filings. By default (`fullHistory` off) returns the recent
524
+ * window (from `filings.recent` — up to 1 year OR 1000 filings, whichever is
525
+ * more), optionally narrowed to specific `forms`, with offset pagination. HONESTY:
526
+ * the response is COMPLETE only when `filings.files[]` (older shards) is empty;
527
+ * when shards exist, `totalAvailable` is the grand total (recent + Σ shard counts),
528
+ * `hasMore:true`, and a note discloses that only the recent window was searched.
529
+ * With `fullHistory:true`, the older `files[]` shards are fetched (newest-first up
530
+ * to `maxShards`, default 10) and assembled (recent ++ shard001..N, descending
531
+ * preserved, NO re-sort) into the COMPLETE history — a capped/failed fan-out is
532
+ * disclosed as PARTIAL (never a capped set claimed complete). `totalAvailable`
533
+ * stays the grand total regardless of the cap (buildMeta forces complete:false
534
+ * when returned < total).
535
+ */
536
+ export async function companyFilings(args: {
537
+ cikOrTicker: string;
538
+ forms?: string[];
539
+ limit?: number;
540
+ offset?: number;
541
+ fullHistory?: boolean;
542
+ maxShards?: number;
543
+ }): Promise<MetaBundle> {
544
+ const resolved = await resolveCik(args.cikOrTicker);
545
+ if (!resolved) {
546
+ return notFoundBundle(
547
+ args.cikOrTicker,
548
+ `Could not resolve "${args.cikOrTicker}" to a CIK (not a numeric CIK and no exact-ticker/title match in company_tickers.json).`,
549
+ );
550
+ }
551
+ const cik = resolved.cik;
552
+ let subm;
553
+ try {
554
+ subm = await fetchSubmissions(cik);
555
+ } catch (e) {
556
+ if (e instanceof ToolErrorCarrier && e.toolError.kind === "not_found") {
557
+ return notFoundBundle(
558
+ cik,
559
+ `No SEC submissions found for CIK ${cik} (HTTP 404) — the CIK does not exist or has no filings. Not fabricated.`,
560
+ );
561
+ }
562
+ throw e;
563
+ }
564
+
565
+ const fullHistory = args.fullHistory ?? false;
566
+ const maxShards = args.maxShards ?? 10;
567
+
568
+ const hasShards = subm.files.length > 0;
569
+ const totalShards = subm.files.length;
570
+ // Σ ALL files[].filingCount — the grand shard total, INCLUDING un-fetched shards.
571
+ // Authoritative from the PARENT payload alone; never recomputed down (M1).
572
+ const shardCount = subm.files.reduce((s, f) => s + (f.filingCount ?? 0), 0);
573
+
574
+ // ── Shard fan-out (gated STRICTLY on fullHistory && there are shards) ──
575
+ // Assemble `recent ++ shard001 ++ … ++ shardNNN` — descending date order is
576
+ // preserved by construction (recent newest, then each shard newest-first), so
577
+ // there is NO re-sort (a blind re-sort would risk breaking SEC's own tie order).
578
+ let assembled = zipRecent(subm.recent, cik);
579
+ let fetchedShards = 0;
580
+ let failedShards = 0; // shards ATTEMPTED (within the cap) that 404'd / were refused.
581
+ let failedFilings = 0; // Σ declared filingCount of those FAILED shards.
582
+ let shortfallFilings = 0; // Σ (declared − actual) for SHORT (but fetched) shards.
583
+ const shardNotes: string[] = [];
584
+ const fanoutRan = fullHistory && hasShards;
585
+ if (fanoutRan) {
586
+ // newest-first up to maxShards (files[] is ordered shard-001 newest → shard-NNN oldest).
587
+ const toFetch = subm.files.slice(0, maxShards);
588
+ for (const shardEntry of toFetch) {
589
+ const name = shardEntry.name ?? "";
590
+ const declared = shardEntry.filingCount ?? 0;
591
+ try {
592
+ const rows = await fetchShard(name, cik);
593
+ fetchedShards++;
594
+ assembled = assembled.concat(rows);
595
+ // [M1] filingCount !== length is INTER-document (parent vs a later shard
596
+ // fetch) and can be legit (recent→shard roll / upstream lag). Do NOT throw;
597
+ // use the ACTUAL fetched count and disclose the shortfall as partial.
598
+ if (rows.length < declared) {
599
+ shortfallFilings += declared - rows.length;
600
+ shardNotes.push(
601
+ `shard ${name} returned ${rows.length} of ${declared} declared filings (recent→shard roll or upstream lag); not fabricated.`,
602
+ );
603
+ }
604
+ } catch (e) {
605
+ const kind = e instanceof ToolErrorCarrier ? e.toolError.kind : "unknown";
606
+ // not_found = a genuine 404 OR a refused bad/cross-CIK/host name (0 fetch).
607
+ // Degrade to PARTIAL and continue; the declared filings are missing. This is
608
+ // a FAILURE of an ATTEMPTED shard — NOT the maxShards cap (see cappedShards).
609
+ if (kind === "not_found") {
610
+ failedShards++;
611
+ failedFilings += declared;
612
+ continue;
613
+ }
614
+ // schema_drift / rate_limited / invalid_input / upstream_unavailable →
615
+ // fail LOUD (a shape break or a systemic UA/rate/outage fault is not a
616
+ // per-shard degrade — never a fake-partial).
617
+ throw e;
618
+ }
619
+ }
620
+ }
621
+ // CAP remainder — the older shards the maxShards cap NEVER ATTEMPTED (totalShards >
622
+ // maxShards). These are the ONLY shards reachable by RAISING maxShards; a shard that
623
+ // was attempted-and-404'd is a FAILURE (failedShards), NOT a cap remainder, and
624
+ // raising the cap will NOT recover it. Keeping the two causes separate is the fix.
625
+ const cappedShards = fanoutRan && totalShards > maxShards ? totalShards - maxShards : 0;
626
+ const cappedFilings =
627
+ cappedShards > 0
628
+ ? subm.files.slice(maxShards).reduce((s, f) => s + (f.filingCount ?? 0), 0)
629
+ : 0;
630
+
631
+ const forms = args.forms?.map((f) => f.trim().toUpperCase()).filter(Boolean);
632
+ const filtered =
633
+ forms && forms.length
634
+ ? assembled.filter((f) => f.form != null && forms.includes(f.form.toUpperCase()))
635
+ : assembled;
636
+
637
+ const limit = args.limit ?? 20;
638
+ const offset = args.offset ?? 0;
639
+ const page = filtered.slice(offset, offset + limit);
640
+ const returned = page.length;
641
+
642
+ const moreInWindow = offset + returned < filtered.length;
643
+ // [M2/M4] Two "more" axes. When NOT fanning out, the beyond-window axis is
644
+ // today's "shards exist but were not fetched" (hasShards). When fanning out, it
645
+ // is the CAP remainder OR any FAILED shard (cappedShards>0 || failedShards>0 —
646
+ // equivalent to fetchedShards<totalShards but attributed to its actual cause),
647
+ // NOT the plain hasShards boolean. A mutation that sets hasMore from moreInWindow
648
+ // alone → RED.
649
+ const beyondWindow = fanoutRan
650
+ ? cappedShards > 0 || failedShards > 0
651
+ : hasShards;
652
+ const hasMore = moreInWindow || beyondWindow;
653
+ // nextOffset walks ONLY the assembled+fetched window; the older un-fetched shards
654
+ // are UNREACHABLE via paging (mirror the FTS beyond-window pattern: hasMore true
655
+ // while nextOffset is null).
656
+ const nextOffset = moreInWindow ? offset + returned : null;
657
+ // [M1] totalAvailable UNCHANGED — grand total when shards exist (incl un-fetched),
658
+ // else the filtered recent length. Never recomputed down to fetched rows.
659
+ const totalAvailable = hasShards ? subm.recentCount + shardCount : filtered.length;
660
+
661
+ const filtersApplied: string[] = [];
662
+ if (forms && forms.length) filtersApplied.push("forms");
663
+
664
+ const notes: string[] = [];
665
+ if (fanoutRan) {
666
+ // The THREE independent PARTIAL causes, attributed + disclosed SEPARATELY (each
667
+ // only when it actually applies): (1) the maxShards CAP skipped older shards
668
+ // (cappedShards) → RAISE maxShards; (2) an ATTEMPTED shard FAILED/404'd
669
+ // (failedShards) → retry may recover, raising maxShards will NOT; (3) a fetched
670
+ // shard was SHORT (shortfallFilings) → benign roll/lag. "complete" needs all three
671
+ // absent. Misattributing a failure to the cap (and advising "raise maxShards") is
672
+ // the defect this split fixes.
673
+ const complete = cappedShards === 0 && failedShards === 0 && shortfallFilings === 0;
674
+ if (complete) {
675
+ notes.push(
676
+ `COMPLETE filing history: fetched all ${totalShards} older shard(s) plus the recent window (${subm.recentCount}) = ${totalAvailable} filings.`,
677
+ );
678
+ } else {
679
+ if (cappedShards > 0) {
680
+ // PARTIAL-BY-CAP — ONLY the un-attempted older shards the cap skipped; the
681
+ // "RAISE maxShards" advice appears ONLY here (it recovers cap remainder, not
682
+ // a 404). The counts are the CAPPED counts, never a failed-shard count.
683
+ notes.push(
684
+ `PARTIAL history (maxShards cap ${maxShards}): the cap limited the fan-out to the newest ${maxShards} of ${totalShards} shard(s); ${cappedShards} older shard(s) (${cappedFilings} filings) were NOT attempted. Paginate via nextOffset for MORE OF THIS fetched window (the fetched shard(s) + the recent ${subm.recentCount}); pagination will NOT reach the un-attempted older shard(s) — RAISE maxShards (max 100) to fetch them.`,
685
+ );
686
+ }
687
+ if (failedShards > 0) {
688
+ // PARTIAL-BY-FAILURE — shards that WERE attempted (within the cap) but 404'd /
689
+ // errored. NOT a cap issue → do NOT advise raising maxShards (a re-fetch hits
690
+ // the same 404); a retry may recover a transient failure.
691
+ notes.push(
692
+ `PARTIAL history: ${failedShards} attempted shard(s) could not be fetched (HTTP 404 / bad-or-cross-CIK name / transient); ${failedFilings} filing(s) are missing from this history. Not fabricated — a retry may recover a transient failure (raising maxShards will NOT recover a shard that 404'd).`,
693
+ );
694
+ }
695
+ if (shortfallFilings > 0) {
696
+ // A fetched shard returned fewer rows than its parent-declared filingCount
697
+ // (recent→shard roll / upstream lag) — a benign shortfall, per-shard below.
698
+ notes.push(
699
+ `${shortfallFilings} declared filing(s) were absent from otherwise-fetched shard body(ies) (recent→shard roll or upstream lag); disclosed per-shard below and NOT fabricated.`,
700
+ );
701
+ }
702
+ }
703
+ if (forms && forms.length) {
704
+ notes.push(
705
+ complete
706
+ ? "The form filter now spans the COMPLETE fetched history (recent + all shards); totalAvailable stays the grand UNFILTERED total, so complete may read false — an under-claim, NOT missing older matches."
707
+ : "Form filtering was applied across the fetched window (recent + fetched shards) — older matching filings may exist in the un-attempted/failed shards.",
708
+ );
709
+ }
710
+ // [M4] Shared-throttle contention: the fan-out serializes N shard GETs through
711
+ // the module-level edgar gate SHARED across all edgar tools.
712
+ notes.push(
713
+ `fullHistory serialized ${fetchedShards + failedShards} shard GET(s) through the shared EDGAR throttle gate (~${EDGAR_MIN_INTERVAL_MS}ms spacing, shared across ALL edgar tools) — a large fan-out adds latency to concurrent edgar tool calls.`,
714
+ );
715
+ notes.push(...shardNotes);
716
+ } else if (hasShards) {
717
+ // fullHistory OFF (default) with shards → today's INCOMPLETE note VERBATIM
718
+ // (backward-compat: byte-identical output when the flag is absent).
719
+ notes.push(
720
+ `INCOMPLETE HISTORY: only the ${subm.recentCount} most-recent filings (filings.recent) were fetched. ${subm.files.length} older shard(s) (~${shardCount} filings, filings.files[]) were NOT fetched, so this is NOT the full filing history.` +
721
+ (forms && forms.length
722
+ ? " Form filtering was applied to the recent window only — older matching filings may exist in the un-fetched shards."
723
+ : ""),
724
+ );
725
+ } else {
726
+ notes.push(
727
+ `Complete filing history: filings.files[] is empty, so filings.recent (${subm.recentCount}) is the full set.` +
728
+ (moreInWindow ? " This page is a subset; paginate via nextOffset for the rest." : ""),
729
+ );
730
+ }
731
+ if (resolved.title) notes.push(`Resolved "${args.cikOrTicker}" → ${resolved.title} (CIK ${cik}).`);
732
+
733
+ return withMeta(
734
+ {
735
+ cik,
736
+ entityName: subm.name,
737
+ filings: page,
738
+ },
739
+ edgarMeta({
740
+ returned,
741
+ totalAvailable,
742
+ filtersApplied,
743
+ pagination: {
744
+ offset,
745
+ limit,
746
+ hasMore,
747
+ nextOffset,
748
+ },
749
+ notes,
750
+ }),
751
+ );
752
+ }
753
+
754
+ // ─── Tool 3: edgar_company_facts ──────────────────────────────────
755
+ /**
756
+ * The default curated us-gaap concepts (F4 — NO EarningsPerShareBasic; its unit
757
+ * is USD/shares, so the default unit="USD" would silently return nothing). The
758
+ * two revenue tags cover the same logical concept — filers report under one or
759
+ * the other — so this is the 6 curated USD concepts. Extracting only these
760
+ * avoids returning the full ~500-concept companyfacts payload.
761
+ */
762
+ export const DEFAULT_FACT_CONCEPTS = [
763
+ "Revenues",
764
+ "RevenueFromContractWithCustomerExcludingAssessedTax",
765
+ "Assets",
766
+ "Liabilities",
767
+ "StockholdersEquity",
768
+ "NetIncomeLoss",
769
+ "CashAndCashEquivalentsAtCarryingValue",
770
+ ];
771
+
772
+ type FactPoint = {
773
+ start?: string; // present (ISO date) for DURATION concepts; absent for INSTANT ones
774
+ end: string;
775
+ val: number;
776
+ accn?: string;
777
+ fy?: number;
778
+ fp?: string;
779
+ form?: string;
780
+ filed?: string;
781
+ };
782
+ type FactNode = {
783
+ label?: string;
784
+ description?: string;
785
+ units?: Record<string, FactPoint[]>;
786
+ };
787
+ type FactsDoc = {
788
+ cik?: number;
789
+ entityName?: string;
790
+ facts?: Record<string, Record<string, FactNode>>;
791
+ };
792
+
793
+ async function fetchFacts(cik10: string): Promise<FactsDoc> {
794
+ return memoize(
795
+ `edgar:facts:${cik10}`,
796
+ async () => {
797
+ const url = `${XBRL_FACTS_BASE}/CIK${cik10}.json`;
798
+ const r = await getEdgar(url, "edgar:companyfacts");
799
+ const d = (await r.json()) as FactsDoc;
800
+ if (!d || typeof d.facts !== "object" || d.facts === null) {
801
+ throw new ToolErrorCarrier({
802
+ kind: "schema_drift",
803
+ message: `edgar:companyfacts returned an unexpected shape for CIK ${cik10} (facts object missing).`,
804
+ retryable: false,
805
+ upstreamEndpoint: "edgar:companyfacts",
806
+ });
807
+ }
808
+ return d;
809
+ },
810
+ 60 * 60 * 1000,
811
+ );
812
+ }
813
+
814
+ /**
815
+ * Curated XBRL financial facts for a filer. Extracts only the requested (or the
816
+ * 6 default) concepts in the requested `unit` (default USD), from the memoized
817
+ * companyfacts doc. HONESTY: a concept ABSENT for this filer is OMITTED and
818
+ * listed in a note (NEVER surfaced as 0); a concept present only in a DIFFERENT
819
+ * unit (e.g. EPS in USD/shares) is listed under `wrongUnit` with a note, never a
820
+ * silent 0. `latest` reduces each concept to its single most-recent data point.
821
+ */
822
+ export async function companyFacts(args: {
823
+ cikOrTicker: string;
824
+ concepts?: string[];
825
+ unit?: string;
826
+ latest?: boolean;
827
+ }): Promise<MetaBundle> {
828
+ const resolved = await resolveCik(args.cikOrTicker);
829
+ if (!resolved) {
830
+ return notFoundBundle(
831
+ args.cikOrTicker,
832
+ `Could not resolve "${args.cikOrTicker}" to a CIK (not a numeric CIK and no exact-ticker/title match in company_tickers.json).`,
833
+ );
834
+ }
835
+ const cik = resolved.cik;
836
+ let doc: FactsDoc;
837
+ try {
838
+ doc = await fetchFacts(cik);
839
+ } catch (e) {
840
+ if (e instanceof ToolErrorCarrier && e.toolError.kind === "not_found") {
841
+ return notFoundBundle(
842
+ cik,
843
+ `No XBRL company facts for CIK ${cik} (HTTP 404) — the CIK does not exist or has filed no XBRL financial data. Not fabricated.`,
844
+ );
845
+ }
846
+ throw e;
847
+ }
848
+
849
+ const unit = (args.unit ?? "USD").trim();
850
+ const latest = args.latest ?? false;
851
+ const requested =
852
+ args.concepts && args.concepts.length ? args.concepts : DEFAULT_FACT_CONCEPTS;
853
+
854
+ const usGaap = doc.facts?.["us-gaap"] ?? {};
855
+ const dei = doc.facts?.["dei"] ?? {};
856
+
857
+ const concepts: Array<{
858
+ concept: string;
859
+ label: string | null;
860
+ unit: string;
861
+ points: Array<{
862
+ start: string | null; // null for INSTANT concepts; the ISO period-start for DURATION concepts (mirrors companyConcept M1)
863
+ end: string | null;
864
+ val: number | null;
865
+ accn: string | null;
866
+ fy: number | null;
867
+ fp: string | null;
868
+ form: string | null;
869
+ filed: string | null;
870
+ }>;
871
+ }> = [];
872
+ const absent: string[] = [];
873
+ const wrongUnit: Array<{ concept: string; availableUnits: string[] }> = [];
874
+ // Concepts where ≥2 points share an `end` with DIFFERING `start` (a quarter AND a
875
+ // YTD/annual figure coexist under one `end`) — surfaced in a note so a consumer
876
+ // never conflates a Q4 figure with a full-year one.
877
+ const sameEndMultiDuration: string[] = [];
878
+
879
+ for (const concept of requested) {
880
+ const node: FactNode | undefined = usGaap[concept] ?? dei[concept];
881
+ if (!node) {
882
+ absent.push(concept);
883
+ continue;
884
+ }
885
+ const units = node.units ?? {};
886
+ const series = units[unit];
887
+ if (!series || !Array.isArray(series)) {
888
+ wrongUnit.push({ concept, availableUnits: Object.keys(units) });
889
+ continue;
890
+ }
891
+ let points = series.map((p) => ({
892
+ start: str(p.start), // null for INSTANT concepts; ISO period-start for DURATION (M1)
893
+ end: str(p.end),
894
+ val: num(p.val),
895
+ accn: str(p.accn),
896
+ fy: typeof p.fy === "number" ? p.fy : null,
897
+ fp: str(p.fp),
898
+ form: str(p.form),
899
+ filed: str(p.filed),
900
+ }));
901
+ // Detect same-`end` different-`start` coexistence on the FULL series (before any
902
+ // `latest` reduction): group by `end`, flag when one `end` carries ≥2 distinct
903
+ // `start` values (a shorter period AND a longer one under the same end date).
904
+ const startsByEnd = new Map<string, Set<string>>();
905
+ for (const p of points) {
906
+ const endKey = p.end ?? "";
907
+ let set = startsByEnd.get(endKey);
908
+ if (!set) {
909
+ set = new Set<string>();
910
+ startsByEnd.set(endKey, set);
911
+ }
912
+ set.add(p.start ?? "");
913
+ }
914
+ if ([...startsByEnd.values()].some((set) => set.size >= 2)) {
915
+ sameEndMultiDuration.push(concept);
916
+ }
917
+ if (latest && points.length > 0) {
918
+ // Most-recent by period end (fallback: filed date), keep a single point.
919
+ points = [
920
+ points.reduce((best, p) =>
921
+ (p.end ?? "") > (best.end ?? "") ||
922
+ ((p.end ?? "") === (best.end ?? "") && (p.filed ?? "") > (best.filed ?? ""))
923
+ ? p
924
+ : best,
925
+ ),
926
+ ];
927
+ }
928
+ concepts.push({ concept, label: str(node.label), unit, points });
929
+ }
930
+
931
+ const notes: string[] = [];
932
+ if (absent.length) {
933
+ notes.push(
934
+ `Concept(s) not reported by this filer (OMITTED, not zero): ${absent.join(", ")}.`,
935
+ );
936
+ }
937
+ if (wrongUnit.length) {
938
+ for (const w of wrongUnit) {
939
+ notes.push(
940
+ `Concept "${w.concept}" exists but not in unit "${unit}" (available: ${w.availableUnits.join(", ") || "none"}). Re-request with the correct unit — NOT reported as 0.`,
941
+ );
942
+ }
943
+ }
944
+ notes.push(
945
+ "Each point carries `start`: null for INSTANT (balance-sheet) concepts and the ISO period-start for DURATION (flow) concepts. Period identity is the (start,end) PAIR — a same-`end` different-`start` pair is a different-duration fact (a 3-month quarter vs the 12-month year), NOT a revision.",
946
+ );
947
+ if (sameEndMultiDuration.length) {
948
+ notes.push(
949
+ `Concept(s) with MULTIPLE durations sharing the same period-end (a shorter quarter AND a longer YTD/annual figure coexist under one \`end\`): ${sameEndMultiDuration.join(", ")}. Each point's \`start\` distinguishes them — do NOT compare or sum values across differing \`start\` (e.g. never read a Q4 figure as the full year) without checking \`start\`.`,
950
+ );
951
+ }
952
+ notes.push(
953
+ latest
954
+ ? "latest=true: each concept reduced to its single most-recent data point (by period end). The selected point retains its `start`, so a duration value is never presented unqualified — a same-`end` shorter/longer period may have been dropped (see the (start,end) note)."
955
+ : "Full reported time series per concept (curated to the requested concepts only; companyfacts holds ~hundreds of concepts).",
956
+ );
957
+
958
+ // A definitive curated extraction of the requested concepts (no upstream
959
+ // pagination) — complete for the concepts that ARE present in this unit.
960
+ return withMeta(
961
+ { cik, entityName: str(doc.entityName), unit, concepts, absent, wrongUnit },
962
+ edgarMeta({
963
+ returned: concepts.length,
964
+ totalAvailable: concepts.length,
965
+ complete: true,
966
+ filtersApplied: latest ? ["concepts", "unit", "latest"] : ["concepts", "unit"],
967
+ notes,
968
+ }),
969
+ );
970
+ }
971
+
972
+ // ─── Tool 4: edgar_full_text_search ───────────────────────────────
973
+ export type FtsResult = {
974
+ accession: string | null;
975
+ form: string | null;
976
+ filingDate: string | null;
977
+ entityNames: string[];
978
+ ciks: string[];
979
+ filingIndexUrl: string | null;
980
+ };
981
+
982
+ /**
983
+ * Map one efts hit → a stable output row. F7: `_source` has NO primary-document
984
+ * filename, so `filingIndexUrl` is the filing's ARCHIVE INDEX DIRECTORY built
985
+ * from `adsh` (accession) + the first CIK — a real, resolvable URL, with NO
986
+ * fabricated document filename appended.
987
+ */
988
+ function mapFtsHit(hit: {
989
+ _id?: string;
990
+ _source?: {
991
+ ciks?: string[];
992
+ display_names?: string[];
993
+ form?: string;
994
+ file_date?: string;
995
+ adsh?: string;
996
+ };
997
+ }): FtsResult {
998
+ const src = hit._source ?? {};
999
+ const ciks = Array.isArray(src.ciks) ? src.ciks.map(String) : [];
1000
+ const adsh =
1001
+ str(src.adsh) ??
1002
+ (typeof hit._id === "string" ? str(hit._id.split(":")[0]) : null);
1003
+ let filingIndexUrl: string | null = null;
1004
+ if (adsh && ciks[0]) {
1005
+ filingIndexUrl = `${ARCHIVES_BASE}/${unpadCik(padCik(ciks[0]))}/${adsh.replace(/-/g, "")}/`;
1006
+ }
1007
+ return {
1008
+ accession: adsh,
1009
+ form: str(src.form),
1010
+ filingDate: str(src.file_date),
1011
+ entityNames: Array.isArray(src.display_names) ? src.display_names.map(String) : [],
1012
+ ciks,
1013
+ filingIndexUrl,
1014
+ };
1015
+ }
1016
+
1017
+ const FTS_PAGE_SIZE = 100; // F2 — efts ignores `size`; it is fixed at 100/page.
1018
+ const FTS_WINDOW = 10_000; // upstream hard cap: from + 100 must be ≤ 10000.
1019
+ const FTS_MAX_FROM = FTS_WINDOW - FTS_PAGE_SIZE; // 9900 → from > 9900 is invalid (9900 is the valid final page).
1020
+
1021
+ /**
1022
+ * Full-text search across EDGAR filings (2001-present). F2 — NO `size` param
1023
+ * (efts always returns 100/page); pagination is by `from`. F3 — `from > 9900`
1024
+ * is rejected as invalid_input BEFORE the fetch (from+100 would exceed the 10000
1025
+ * window, which efts answers with HTTP 200 + an error body; from=9900 itself is a
1026
+ * VALID final page), and a response missing `hits.hits` is thrown as schema_drift
1027
+ * (never crashed on). F5 — `hits.total.relation === "gte"` (true total unknown,
1028
+ * ≥10000) surfaces as `totalIsLowerBound:true`. ADR-0018 — optional `ciks` (pin
1029
+ * filings BY entities, exact 10-digit CIK) + `entityName` (fuzzy filer-name)
1030
+ * narrowing filters; a no-digit/CIK-0 `ciks` entry is rejected pre-fetch (M1).
1031
+ */
1032
+ export async function fullTextSearch(args: {
1033
+ q: string;
1034
+ forms?: string[];
1035
+ startdt?: string;
1036
+ enddt?: string;
1037
+ ciks?: string[];
1038
+ entityName?: string;
1039
+ from?: number;
1040
+ }): Promise<MetaBundle> {
1041
+ const from = args.from ?? 0;
1042
+ // Off-by-one fix (ADR-0018): `from=9900` is a VALID final page (from+100=10000,
1043
+ // the exact ES window boundary — live-confirmed). Only `from > 9900` overflows.
1044
+ if (from > FTS_MAX_FROM) {
1045
+ throw new ToolErrorCarrier({
1046
+ kind: "invalid_input",
1047
+ message: `EDGAR full-text search 'from' (${from}) is out of range: the paging window is from+100 ≤ ${FTS_WINDOW}, so 'from' must be <= ${FTS_MAX_FROM}. Narrow the query with forms/startdt/enddt instead of paging past ${FTS_WINDOW} results.`,
1048
+ retryable: false,
1049
+ upstreamEndpoint: "edgar:fts",
1050
+ });
1051
+ }
1052
+
1053
+ // ── Entity filters (ADR-0018) — normalize BEFORE any fetch. ──
1054
+ // M1 (fake-empty landmine): each `ciks` entry is reduced to digits; an entry
1055
+ // that yields NO digits (a ticker/garbage like "AAPL"/"../") OR an all-zeros
1056
+ // value ("0"/"0000000000") pads to the NON-EXISTENT CIK 0000000000, which efts
1057
+ // answers with a LIVE 0/eq fake-empty. Reject BOTH explicitly here (0 fetch) —
1058
+ // NEVER `.map(padCik).filter(Boolean)` (padCik("AAPL")="0000000000" is truthy).
1059
+ // Whitespace-only/empty entries are silently dropped (probe 10 — an empty CIK
1060
+ // is ignored upstream anyway).
1061
+ const ciks: string[] = [];
1062
+ for (const raw of args.ciks ?? []) {
1063
+ const s = raw == null ? "" : String(raw).trim();
1064
+ if (s === "") continue; // drop empty/whitespace entries
1065
+ const digits = s.replace(/\D/g, "");
1066
+ if (digits.replace(/0/g, "") === "") {
1067
+ throw new ToolErrorCarrier({
1068
+ kind: "invalid_input",
1069
+ message: `EDGAR full-text search 'ciks' entry ${JSON.stringify(raw)} is not a valid SEC CIK: 'ciks' takes NUMERIC CIKs only (each is zero-padded to 10 digits; CIK 0 / a ticker / a company name is rejected — it would silently return 0 matches). For a ticker or company name, use 'entityName' or resolve the CIK first with edgar_lookup_cik (by name/ticker).`,
1070
+ retryable: false,
1071
+ upstreamEndpoint: "edgar:fts",
1072
+ });
1073
+ }
1074
+ ciks.push(digits.padStart(10, "0"));
1075
+ }
1076
+ const entityName = args.entityName?.trim() ?? "";
1077
+
1078
+ const params = new URLSearchParams();
1079
+ params.set("q", args.q);
1080
+ const forms = args.forms?.map((f) => f.trim()).filter(Boolean);
1081
+ if (forms && forms.length) params.set("forms", forms.join(","));
1082
+ if (args.startdt || args.enddt) {
1083
+ params.set("dateRange", "custom");
1084
+ if (args.startdt) params.set("startdt", args.startdt);
1085
+ if (args.enddt) params.set("enddt", args.enddt);
1086
+ }
1087
+ // Whitelisted entity filters — set (and echo in filtersApplied) ONLY when a
1088
+ // live-honored value was actually sent (silent-drop guard; probe 7).
1089
+ if (ciks.length) params.set("ciks", ciks.join(","));
1090
+ if (entityName) params.set("entityName", entityName);
1091
+ if (from > 0) params.set("from", String(from));
1092
+
1093
+ const r = await getEdgar(`${FTS_URL}?${params.toString()}`, "edgar:fts");
1094
+ const d = (await r.json()) as {
1095
+ hits?: { total?: { value?: number; relation?: string }; hits?: unknown[] };
1096
+ };
1097
+ // F3 — window overflow / any error body arrives as HTTP 200 with no hits.hits.
1098
+ if (!d || !d.hits || !Array.isArray(d.hits.hits)) {
1099
+ throw new ToolErrorCarrier({
1100
+ kind: "schema_drift",
1101
+ message:
1102
+ "edgar:fts returned HTTP 200 without hits.hits — SEC answers a paging-window overflow (or a malformed query) with 200 + an error body. Narrow the query or reduce 'from'.",
1103
+ retryable: false,
1104
+ upstreamEndpoint: "edgar:fts",
1105
+ });
1106
+ }
1107
+
1108
+ const total = d.hits.total ?? {};
1109
+ const totalAvailable = typeof total.value === "number" ? total.value : 0;
1110
+ const isLowerBound = total.relation === "gte";
1111
+ const results = (d.hits.hits as Parameters<typeof mapFtsHit>[0][]).map(mapFtsHit);
1112
+ const returned = results.length;
1113
+
1114
+ const nextFrom = from + FTS_PAGE_SIZE;
1115
+ // More results exist AND the next page is still inside the 10000 window.
1116
+ const hasMore = from + returned < totalAvailable && nextFrom < FTS_WINDOW;
1117
+ // The next page is only reachable if it does not trip the `from > 9900` guard.
1118
+ // Off-by-one fix (ADR-0018): `<=` so pagination can reach the final `from=9900`
1119
+ // page (nextFrom=9900) instead of stopping one page short.
1120
+ const nextOffset = hasMore && nextFrom <= FTS_MAX_FROM ? nextFrom : null;
1121
+
1122
+ const filtersApplied: string[] = ["q"];
1123
+ if (forms && forms.length) filtersApplied.push("forms");
1124
+ if (args.startdt || args.enddt) filtersApplied.push("dateRange");
1125
+ if (ciks.length) filtersApplied.push("ciks");
1126
+ if (entityName) filtersApplied.push("entityName");
1127
+
1128
+ const notes: string[] = [
1129
+ "EDGAR full-text search covers 2001-present only (earlier filings are not indexed).",
1130
+ "Page size is fixed at 100 (the efts `size` param is ignored); paginate via `from`.",
1131
+ ];
1132
+ // Entity-filter semantics (ADR-0018) — emitted only when the filter was applied.
1133
+ if (ciks.length) {
1134
+ notes.push(
1135
+ "Results are pinned to the supplied CIK(s) (exact 10-digit match). A 0-result set may mean the CIK is wrong OR the entity has no matching filings in the query/form/date window — it is NOT proof of absence; verify the CIK via edgar_lookup_cik (by name/ticker) or on SEC EDGAR directly.",
1136
+ );
1137
+ }
1138
+ if (entityName) {
1139
+ notes.push(
1140
+ "entityName is a FUZZY filer-name filter (it can match related entities, e.g. multiple 'Apple*' filers) — it is NOT a CIK-exact pin; combine with ciks for an exact-entity result.",
1141
+ );
1142
+ }
1143
+ if (returned === 0 && (ciks.length || entityName)) {
1144
+ notes.push(
1145
+ "0 results with ciks/entityName applied is NOT proof of absence — a wrong CIK, a too-fuzzy/mismatched entityName, or a genuine no-match are indistinguishable here; verify the identifier via edgar_lookup_cik (by name/ticker) or on SEC EDGAR directly.",
1146
+ );
1147
+ }
1148
+ if (isLowerBound) {
1149
+ notes.push(
1150
+ `totalAvailable is a LOWER BOUND: SEC reported hits.total.relation="gte" with the value pinned at ${totalAvailable}; the true match count is UNKNOWN and ≥ ${totalAvailable}. See totalIsLowerBound. Narrow the query for an exact count.`,
1151
+ );
1152
+ }
1153
+ if (hasMore && nextOffset === null) {
1154
+ notes.push(
1155
+ `More matches exist but are beyond EDGAR's ${FTS_WINDOW}-result full-text window and are UNREACHABLE via paging — narrow the query with forms/date filters to retrieve them.`,
1156
+ );
1157
+ }
1158
+ const meta: Partial<ResponseMeta> = {
1159
+ returned,
1160
+ totalAvailable,
1161
+ filtersApplied,
1162
+ pagination: { offset: from, limit: FTS_PAGE_SIZE, hasMore, nextOffset },
1163
+ notes,
1164
+ };
1165
+ if (isLowerBound) meta.totalIsLowerBound = true;
1166
+
1167
+ return withMeta({ query: args.q, results }, edgarMeta(meta));
1168
+ }
1169
+
1170
+ // ─── Tool 5: edgar_xbrl_frames ────────────────────────────────────
1171
+ // ADR-0017 (v1 + v2 AUTHORITATIVE). A COMPLETE keyless cross-filer cross-section:
1172
+ // every XBRL filer's reported value for ONE concept in ONE calendar period, in a
1173
+ // single call — the peer-benchmarking / distribution primitive. Reuses getEdgar
1174
+ // VERBATIM (no new host/gate/UA); the frames path puts caller-supplied segments
1175
+ // (taxonomy/tag/unit/period) as RAW PATH SEGMENTS, so the load-bearing control is
1176
+ // the pre-fetch enum/regex validation + fixed-host assertion (§SSRF, S1/S2).
1177
+
1178
+ /** The whitelisted frames host + base (same data.sec.gov the other edgar tools use). */
1179
+ const FRAMES_BASE = "https://data.sec.gov/api/xbrl/frames";
1180
+
1181
+ /**
1182
+ * The `taxonomy` path-segment enum — the SSRF guard for that segment (no free
1183
+ * value reaches the host). Only members LIVE-CONFIRMED to resolve a real frame
1184
+ * (per-segment live-verify discipline, ADR-0017 Open-Q5) are shipped. Maker
1185
+ * probes 2026-07-12: us-gaap (Assets/Revenues/NetIncomeLoss/EPS) + dei
1186
+ * (EntityCommonStockSharesOutstanding/EntityPublicFloat) → 200; the guessed
1187
+ * srt/invest/us-ins tags → 404, so they are DROPPED (conservative floor).
1188
+ */
1189
+ export const FRAMES_TAXONOMIES = ["us-gaap", "dei"] as const;
1190
+
1191
+ // Segment grammars (belt-and-suspenders re-check in the builder — S1). These are
1192
+ // the SAME regexes/enum the server Zod schema applies; re-running them here does
1193
+ // NOT rely on Zod (a direct call could bypass it) nor on the hostname assertion
1194
+ // alone (a same-host `../` traversal normalizes to host=data.sec.gov and PASSES a
1195
+ // hostname check). They are allowlists: `%`, `/`, `.`, `\`, `..`, `%2F`, `%2E`,
1196
+ // `%00` all fail (S2 — those characters are simply not in the allowed classes).
1197
+ const FRAMES_TAG_RE = /^[A-Za-z0-9]+$/;
1198
+ const FRAMES_UNIT_RE = /^[A-Za-z0-9-]+$/;
1199
+ const FRAMES_PERIOD_RE = /^CY\d{4}(Q[1-4]I?)?$/;
1200
+
1201
+ /** Throw the pre-fetch injection-guard error (invalid_input, 0 fetch). */
1202
+ function framesInvalid(message: string): never {
1203
+ throw new ToolErrorCarrier({
1204
+ kind: "invalid_input",
1205
+ message,
1206
+ retryable: false,
1207
+ upstreamEndpoint: "edgar:frames",
1208
+ });
1209
+ }
1210
+
1211
+ /**
1212
+ * Build the frames URL from validated path segments (S1/S2). BELT-AND-SUSPENDERS:
1213
+ * re-run the enum + the three regexes on each segment and hard-throw invalid_input
1214
+ * (0 fetch) on any mismatch — do NOT trust that Zod already ran, and do NOT rely on
1215
+ * the hostname assertion alone (it passes same-host traversal). THEN assert the
1216
+ * built URL is https on the fixed data.sec.gov host (guards host-escape/downgrade).
1217
+ */
1218
+ function buildFramesUrl(
1219
+ taxonomy: string,
1220
+ tag: string,
1221
+ unit: string,
1222
+ period: string,
1223
+ ): string {
1224
+ if (!(FRAMES_TAXONOMIES as readonly string[]).includes(taxonomy)) {
1225
+ framesInvalid(
1226
+ `edgar_xbrl_frames: taxonomy ${JSON.stringify(taxonomy)} is not one of {${FRAMES_TAXONOMIES.join(", ")}} — refused before any fetch (path-segment injection guard).`,
1227
+ );
1228
+ }
1229
+ if (!FRAMES_TAG_RE.test(tag)) {
1230
+ framesInvalid(
1231
+ `edgar_xbrl_frames: tag ${JSON.stringify(tag)} must match ^[A-Za-z0-9]+$ (XBRL tags are alphanumeric; slash/dot/percent/backslash/'..' are rejected) — refused before any fetch (path-segment injection guard).`,
1232
+ );
1233
+ }
1234
+ if (!FRAMES_UNIT_RE.test(unit)) {
1235
+ framesInvalid(
1236
+ `edgar_xbrl_frames: unit ${JSON.stringify(unit)} must match ^[A-Za-z0-9-]+$ (hyphen allowed, e.g. USD-per-shares; slash/dot/percent forbidden — never the 'USD/shares' companyfacts key form) — refused before any fetch (path-segment injection guard).`,
1237
+ );
1238
+ }
1239
+ if (!FRAMES_PERIOD_RE.test(period)) {
1240
+ framesInvalid(
1241
+ `edgar_xbrl_frames: period ${JSON.stringify(period)} must match ^CY\\d{4}(Q[1-4]I?)?$ (e.g. CY2023, CY2023Q1, CY2023Q4I) — refused before any fetch (path-segment injection guard).`,
1242
+ );
1243
+ }
1244
+ const built = `${FRAMES_BASE}/${taxonomy}/${tag}/${unit}/${period}.json`;
1245
+ let parsed: URL;
1246
+ try {
1247
+ parsed = new URL(built);
1248
+ } catch {
1249
+ framesInvalid(
1250
+ `edgar_xbrl_frames: could not construct a valid URL from the segments — refused before any fetch.`,
1251
+ );
1252
+ }
1253
+ if (parsed.protocol !== "https:" || parsed.hostname !== "data.sec.gov") {
1254
+ framesInvalid(
1255
+ `edgar_xbrl_frames: constructed URL host/scheme is not https://data.sec.gov (${parsed.protocol}//${parsed.hostname}) — refused before any fetch (fixed-host assertion).`,
1256
+ );
1257
+ }
1258
+ return built;
1259
+ }
1260
+
1261
+ /** One filer's row in a frame. `start` is present ONLY for duration concepts. */
1262
+ export type FrameRow = {
1263
+ accn: string | null;
1264
+ cik: string | null;
1265
+ entityName: string | null;
1266
+ loc: string | null;
1267
+ end: string | null;
1268
+ val: number | null;
1269
+ start?: string | null;
1270
+ };
1271
+
1272
+ /**
1273
+ * Summary distribution over the FULL cross-section (M2/M3). Computed over the
1274
+ * FINITE `val`s only (num() → null for absent/blank/"null"/non-finite, which are
1275
+ * excluded and counted in `nonFiniteExcluded`; a real 0 survives). Percentiles use
1276
+ * LINEAR INTERPOLATION on the ascending-sorted finite vals. `count===0` (no finite
1277
+ * vals) ⇒ EVERY stat field null (never 0/NaN/Infinity — the null-never-0 row rule
1278
+ * lifted onto the aggregate).
1279
+ */
1280
+ export type FrameStats = {
1281
+ count: number;
1282
+ min: number | null;
1283
+ max: number | null;
1284
+ sum: number | null;
1285
+ mean: number | null;
1286
+ median: number | null;
1287
+ p25: number | null;
1288
+ p75: number | null;
1289
+ nonFiniteExcluded: number;
1290
+ };
1291
+
1292
+ /** Compute FrameStats over the full data[] (all rows, before any client slice). */
1293
+ function framesStats(data: Array<{ val?: unknown }>): FrameStats {
1294
+ const finite: number[] = [];
1295
+ let nonFiniteExcluded = 0;
1296
+ for (const d of data) {
1297
+ const v = num(d.val); // null for absent/""/"null"/non-finite; real 0 survives.
1298
+ if (v === null) nonFiniteExcluded++;
1299
+ else finite.push(v);
1300
+ }
1301
+ const count = finite.length;
1302
+ // M3 — no finite values ⇒ no distribution computable ⇒ every field null.
1303
+ if (count === 0) {
1304
+ return {
1305
+ count: 0,
1306
+ min: null,
1307
+ max: null,
1308
+ sum: null,
1309
+ mean: null,
1310
+ median: null,
1311
+ p25: null,
1312
+ p75: null,
1313
+ nonFiniteExcluded,
1314
+ };
1315
+ }
1316
+ finite.sort((a, b) => a - b);
1317
+ const sum = finite.reduce((s, v) => s + v, 0);
1318
+ // M2 — linear interpolation: pos=q*(n-1), interpolate between the bracketing
1319
+ // sorted values. median = q=0.5 (mean of the two middle values for even n).
1320
+ const quantile = (q: number): number => {
1321
+ const pos = q * (count - 1);
1322
+ const lo = Math.floor(pos);
1323
+ const hi = Math.ceil(pos);
1324
+ const vLo = finite[lo] as number;
1325
+ const vHi = finite[hi] as number;
1326
+ return pos === lo ? vLo : vLo + (pos - lo) * (vHi - vLo);
1327
+ };
1328
+ return {
1329
+ count,
1330
+ min: finite[0] as number,
1331
+ max: finite[count - 1] as number,
1332
+ sum,
1333
+ mean: sum / count,
1334
+ median: quantile(0.5),
1335
+ p25: quantile(0.25),
1336
+ p75: quantile(0.75),
1337
+ nonFiniteExcluded,
1338
+ };
1339
+ }
1340
+
1341
+ /**
1342
+ * Keyless cross-filer XBRL cross-section. In ONE call, return every filer's
1343
+ * reported value for a single us-gaap/dei concept in a single calendar period —
1344
+ * the complete cross-section — for peer benchmarking + distribution stats.
1345
+ *
1346
+ * HONESTY (ADR-0017 v2):
1347
+ * - `totalAvailable` = SEC's own `pts` (NEVER a page length). Drift guards THROW
1348
+ * schema_drift on a non-frames shape (data not array / pts non-numeric) or a
1349
+ * `pts !== data.length` mismatch (a truncation frames has no way to page past,
1350
+ * so refusing is the honest move — Open-Q2 resolved: THROW is the DEFAULT).
1351
+ * - The upstream frame is fetched in FULL; `limit`/`offset` is a CLIENT-SIDE
1352
+ * window disclosed as such (M1 — the completeness note never calls a subset
1353
+ * page "complete"; buildMeta derives complete/truncated from returned/total/
1354
+ * hasMore, mirroring edgar_company_filings — NO forced complete:true).
1355
+ * - A 404 (tag/unit/period/taxonomy quadruple did not match) ⇒ honest found:false
1356
+ * with the semantic note (absence ≠ 0). NEVER a fabricated val:0.
1357
+ * - Row `val` is num()-coerced (null-never-0). `start` appears only on duration
1358
+ * rows. `uom` echoes SEC's OWN unit (e.g. requested 'USD-per-shares' ⇒ 'USD/shares').
1359
+ * - `includeStats` computes over the FULL data[] (all rows, before the slice).
1360
+ */
1361
+ export async function xbrlFrames(args: {
1362
+ tag: string;
1363
+ period: string;
1364
+ taxonomy?: string;
1365
+ unit?: string;
1366
+ limit?: number;
1367
+ offset?: number;
1368
+ includeStats?: boolean;
1369
+ }): Promise<MetaBundle> {
1370
+ const taxonomy = args.taxonomy ?? "us-gaap";
1371
+ const unit = args.unit ?? "USD";
1372
+ const limit = args.limit ?? 100;
1373
+ const offset = args.offset ?? 0;
1374
+ const includeStats = args.includeStats ?? false;
1375
+
1376
+ // S1/S2 — build (and re-validate) the path BEFORE any fetch. Throws
1377
+ // invalid_input with 0 fetches on any bad segment or a non-fixed-host URL.
1378
+ const url = buildFramesUrl(taxonomy, args.tag, unit, args.period);
1379
+
1380
+ type FramesBody = {
1381
+ tag?: unknown;
1382
+ uom?: unknown;
1383
+ label?: unknown;
1384
+ description?: unknown;
1385
+ pts?: unknown;
1386
+ data?: unknown;
1387
+ };
1388
+ let body: FramesBody;
1389
+ try {
1390
+ const r = await getEdgar(url, "edgar:frames");
1391
+ body = (await r.json()) as FramesBody;
1392
+ } catch (e) {
1393
+ // 404 ⇒ the quadruple did not match a frame ⇒ honest found:false (NEVER 0).
1394
+ if (e instanceof ToolErrorCarrier && e.toolError.kind === "not_found") {
1395
+ return notFoundBundle(
1396
+ `${taxonomy}/${args.tag}/${unit}/${args.period}`,
1397
+ `No XBRL frame matched taxonomy=${taxonomy} tag=${args.tag} unit=${unit} period=${args.period} (HTTP 404). The concept was NOT reported under that exact tag/unit/calendar-frame — this is NOT a value of 0. Check: instant (balance-sheet) concepts need the trailing 'I' (e.g. CY2023Q4I); EPS uses unit 'USD-per-shares' (hyphen), never 'USD/shares'.`,
1398
+ );
1399
+ }
1400
+ throw e;
1401
+ }
1402
+
1403
+ // Drift guards (schema_drift THROW — never a fabricated empty). Order matters:
1404
+ // (1) data[] must be an array, (2) pts must be a finite number, (3) the
1405
+ // load-bearing invariant pts === data.length (a mismatch ⇒ truncation/shape
1406
+ // change; frames has NO page param to recover the rest, so we refuse).
1407
+ const data = body.data;
1408
+ if (!Array.isArray(data)) {
1409
+ throw new ToolErrorCarrier({
1410
+ kind: "schema_drift",
1411
+ message: `edgar:frames returned HTTP 200 without a data[] array (taxonomy=${taxonomy} tag=${args.tag} unit=${unit} period=${args.period}) — the frames envelope changed.`,
1412
+ retryable: false,
1413
+ upstreamEndpoint: "edgar:frames",
1414
+ });
1415
+ }
1416
+ if (typeof body.pts !== "number" || !Number.isFinite(body.pts)) {
1417
+ throw new ToolErrorCarrier({
1418
+ kind: "schema_drift",
1419
+ message: `edgar:frames returned a non-numeric 'pts' (${JSON.stringify(body.pts)}) — SEC's own data-point count is missing; the envelope changed.`,
1420
+ retryable: false,
1421
+ upstreamEndpoint: "edgar:frames",
1422
+ });
1423
+ }
1424
+ if (body.pts !== data.length) {
1425
+ throw new ToolErrorCarrier({
1426
+ kind: "schema_drift",
1427
+ message: `SEC frames pts (${body.pts}) ≠ data.length (${data.length}) — the cross-section may be truncated or the envelope changed. Frames has no pagination to recover the rest, so refusing rather than presenting a partial set as complete.`,
1428
+ retryable: false,
1429
+ upstreamEndpoint: "edgar:frames",
1430
+ });
1431
+ }
1432
+ // SEC's own count; validated === data.length. NEVER a page length (mutation a).
1433
+ const totalAvailable = body.pts;
1434
+
1435
+ // Stats over the FULL data[] (ALL rows, BEFORE the client-side slice) — M2/M3.
1436
+ const stats = includeStats
1437
+ ? framesStats(data as Array<{ val?: unknown }>)
1438
+ : undefined;
1439
+
1440
+ // Map every row: null-never-0 via num() on val; start only on duration rows.
1441
+ const allRows: FrameRow[] = (data as Array<Record<string, unknown>>).map((d) => {
1442
+ const row: FrameRow = {
1443
+ accn: str(d.accn),
1444
+ cik: d.cik == null ? null : padCik(d.cik as string | number),
1445
+ entityName: str(d.entityName),
1446
+ loc: str(d.loc),
1447
+ end: str(d.end),
1448
+ val: num(d.val),
1449
+ };
1450
+ if ("start" in d) row.start = str(d.start);
1451
+ return row;
1452
+ });
1453
+
1454
+ // Client-side window over the already-fully-fetched cross-section (M1).
1455
+ const page = allRows.slice(offset, offset + limit);
1456
+ const returned = page.length;
1457
+ const hasMore = offset + returned < totalAvailable;
1458
+ const uom = str(body.uom);
1459
+ const found = data.length > 0;
1460
+
1461
+ const notes: string[] = [];
1462
+ // SEMANTIC (the CIK-caveat analogue) — absence ≠ 0.
1463
+ notes.push(
1464
+ "A frame contains ONLY filers who reported this EXACT tag for this EXACT calendar period, with SEC selecting one best-fit fact per entity. Absence from the frame ≠ 0 and ≠ 'the company has none' — it means the filer did not report under that tag/calendar-frame.",
1465
+ );
1466
+ // COMPLETENESS — M1 two-clause split (the word "complete" never describes a
1467
+ // subset page): the upstream-frame-fetched-in-full sense and the this-page-is-a-
1468
+ // subset sense are stated distinctly.
1469
+ notes.push(
1470
+ `The upstream frame was fetched in FULL server-side (pts=${totalAvailable} = the entire cross-section for this taxonomy/tag/unit/period, uncapped and unpaginated by SEC). THIS response page contains ${returned} of ${totalAvailable} rows; when hasMore is true, page via nextOffset to retrieve the remaining filers.`,
1471
+ );
1472
+ // DISCOVERABILITY — period grammar + hyphen unit + 404-not-zero.
1473
+ notes.push(
1474
+ "Period grammar: CY2023 (annual flow) · CY2023Q1 (quarterly flow, no I) · CY2023Q4I (instant, trailing I). EPS uses unit 'USD-per-shares' (hyphen), never 'USD/shares'. A tag/unit/period mismatch returns found:false (a 404), NOT an empty zero.",
1475
+ );
1476
+ // val null-never-0.
1477
+ notes.push(
1478
+ "Each row 'val' is coerced to number-or-null: a real reported 0 survives as 0, but an absent/blank/non-finite value is null — never a fabricated 0.",
1479
+ );
1480
+ // uom echo (minor) — path-form vs response-form.
1481
+ if (uom && uom !== unit) {
1482
+ notes.push(
1483
+ `SEC's response uom is '${uom}', which differs from the requested unit path-segment '${unit}' (path-form vs response-form — e.g. 'USD-per-shares' is reported as 'USD/shares'). The 'uom' field echoes SEC's own value.`,
1484
+ );
1485
+ }
1486
+ // Stats disclosure — M2 method / M3 no-finite.
1487
+ if (stats) {
1488
+ notes.push(
1489
+ stats.count === 0
1490
+ ? `stats: no finite values across the full cross-section (nonFiniteExcluded=${stats.nonFiniteExcluded}) — no distribution computable, so every stat field (min/max/sum/mean/median/p25/p75) is null, never 0/NaN/Infinity.`
1491
+ : `stats: min/max/sum/mean and linear-interpolated p25/median/p75 computed over the ${stats.count} FINITE vals (nulls/non-finite excluded; nonFiniteExcluded=${stats.nonFiniteExcluded}) across the FULL cross-section of ${totalAvailable} rows, sorted ascending — not a robust estimator for small frames.`,
1492
+ );
1493
+ }
1494
+
1495
+ const outData: Record<string, unknown> = {
1496
+ found,
1497
+ taxonomy,
1498
+ tag: str(body.tag) ?? args.tag,
1499
+ unit,
1500
+ uom,
1501
+ period: args.period,
1502
+ label: str(body.label),
1503
+ description: str(body.description),
1504
+ rows: page,
1505
+ };
1506
+ if (stats) outData.stats = stats;
1507
+
1508
+ return withMeta(
1509
+ outData,
1510
+ edgarMeta({
1511
+ returned,
1512
+ totalAvailable,
1513
+ filtersApplied: ["taxonomy", "tag", "unit", "period"],
1514
+ pagination: {
1515
+ offset,
1516
+ limit,
1517
+ hasMore,
1518
+ nextOffset: hasMore ? offset + returned : null,
1519
+ },
1520
+ notes,
1521
+ }),
1522
+ );
1523
+ }
1524
+
1525
+ // ─── Tool 6: edgar_filing_index (ADR-0026) ────────────────────────
1526
+ // A KEYLESS BULK cross-filer capability on the EXISTING edgar source: read the
1527
+ // SEC EDGAR quarterly FULL-INDEX (www.sec.gov/Archives/edgar/full-index/<year>/
1528
+ // QTR<n>/master.idx — a COMPLETE cross-filer index of EVERY filer's EVERY filing
1529
+ // in a quarter, pipe-delimited, ~33MB / ~370K rows), FULL-SCAN it, apply
1530
+ // CLIENT-SIDE filters, and return offset-paginated filings with an EXACT total.
1531
+ // The per-filer edgar tools require you ALREADY hold a CIK; this is the
1532
+ // bulk-enumeration primitive ("every 8-K in 2024 Q1", "every filing by CIK X in
1533
+ // Q1", "every filer named '…'"). Reuses getEdgar VERBATIM (UA + Accept-Encoding:
1534
+ // gzip + the ≤10 req/s throughGate — UNTOUCHED, so the other 5 edgar tools stay
1535
+ // byte-identical), `.text()`, padCik, the module-local `str`, `driftError`,
1536
+ // `buildMeta`/`withMeta`. Copies (does NOT import) the frames buildFramesUrl
1537
+ // S1/S2 path-segment idiom.
1538
+ //
1539
+ // LIVE-VERIFIED 2026-07-12 (org UA `cliwant-mcp-sam-gov/1.0`):
1540
+ // - 2024/QTR1/master.idx → 200, 33,206,408 bytes, 370,304 data rows (all 5-field);
1541
+ // exact 8-K count 16,997, exact 10-K count 4,980.
1542
+ // - `Range: bytes=0-1000` → 200 FULL body (Range IGNORED, no 206) → a byte-cap
1543
+ // would truncate the CIK-sorted tail → under-count; FULL-SCAN → EXACT
1544
+ // totalAvailable is the ONLY honest model (byte-cap FORBIDDEN).
1545
+ // - 2026/QTR4 (current-YEAR future quarter) → 200, header+dashes, 0 data rows →
1546
+ // GENUINE-EMPTY (M1: NOT drift). 2026/QTR3 (current quarter) → 200, ~33K rows,
1547
+ // GROWING daily (short cache TTL + a point-in-time-snapshot note).
1548
+ // - 2024/QTR5 → 403 `AccessDenied` XML (does NOT match getEdgar F6's /automated|
1549
+ // undeclared/ → getEdgar mislabels it `rate_limited`) → TOOL-LOCAL reclassify.
1550
+ // - 2027/QTR1 (future YEAR) → 403, but the year bound blocks it PRE-FETCH
1551
+ // (invalid_input, 0 fetch — a future year has no published quarter index).
1552
+
1553
+ const FULLINDEX_HOST = "www.sec.gov";
1554
+ const FULLINDEX_BASE = "https://www.sec.gov/Archives/edgar/full-index";
1555
+ const FULLINDEX_FILE = "master.idx"; // the single backing file (pipe-delimited)
1556
+ const FULLINDEX_LABEL = "edgar:full-index"; // host+path only; keyless ⇒ no token can appear
1557
+ const FULLINDEX_HEADER = "CIK|Company Name|Form Type|Date Filed|Filename";
1558
+ const EDGAR_FULLINDEX_START_YEAR = 1993; // EDGAR full-index begins 1993 Q1
1559
+ const FILING_ARCHIVE_BASE = "https://www.sec.gov/Archives/"; // prefix for row.filename → a resolvable URL
1560
+ // Safety ceiling ABOVE the live ~370K rows (2024Q1 = 370,304). If a body EXCEEDS
1561
+ // it (drift/hostile/giant), the scan stops + `totalIsLowerBound` is set + a note —
1562
+ // NEVER a silent truncation. The normal path never reaches it, so totals stay EXACT.
1563
+ // Exported (with the `maxRows` param on parseFullIndex) so the fault suite can drive
1564
+ // the ceiling with a COMPACT fixture (A2) instead of a 500K-row body.
1565
+ export const MAX_INDEX_ROWS = 500_000;
1566
+
1567
+ /**
1568
+ * Fresh UTC year at CALL time (M2/S1) — NOT a module-load constant, so the upper
1569
+ * year bound survives a year rollover in a long-running process.
1570
+ */
1571
+ function currentUtcYear(): number {
1572
+ return new Date().getUTCFullYear();
1573
+ }
1574
+
1575
+ /**
1576
+ * True iff (year,quarter) is the CURRENT calendar quarter (UTC). The current
1577
+ * quarter GROWS daily (short cache TTL + a point-in-time-snapshot note); a closed
1578
+ * past quarter is immutable (long TTL). NOTE: this is a freshness/cache signal
1579
+ * ONLY — it is deliberately NOT a pre-fetch guard (M2: a same-year future quarter
1580
+ * returns a well-formed empty 200, so it must be reachable, not refused).
1581
+ */
1582
+ function isCurrentQuarter(year: number, quarter: number): boolean {
1583
+ const now = new Date();
1584
+ return (
1585
+ year === now.getUTCFullYear() &&
1586
+ quarter === Math.floor(now.getUTCMonth() / 3) + 1
1587
+ );
1588
+ }
1589
+
1590
+ /** Throw the pre-fetch bounds/injection guard error (invalid_input, 0 fetch). */
1591
+ function fullIndexInvalid(message: string): never {
1592
+ throw new ToolErrorCarrier({
1593
+ kind: "invalid_input",
1594
+ message,
1595
+ retryable: false,
1596
+ upstreamEndpoint: FULLINDEX_LABEL,
1597
+ });
1598
+ }
1599
+
1600
+ /**
1601
+ * Build the full-index URL from the ONLY two caller-influenced path segments
1602
+ * (year, quarter) — the frames `buildFramesUrl` S1/S2 idiom, with the segments as
1603
+ * BOUNDED INTEGERS (a range check, not a regex: `%`/`/`/`.`/`\`/`..`/`%2F` cannot
1604
+ * appear in an integer, so there is no path-segment-injection surface). This is
1605
+ * BELT-AND-SUSPENDERS behind Zod (a direct handler call could bypass Zod):
1606
+ * - year ∈ [1993, currentUtcYear()] else invalid_input (0 fetch) — the upper
1607
+ * bound is CALL-TIME fresh (M2); a future YEAR has no published quarter index.
1608
+ * - quarter ∈ {1,2,3,4} else invalid_input (0 fetch).
1609
+ * There is NO `≤currentCalendarQuarter` guard (M2): a same-year FUTURE quarter
1610
+ * returns a well-formed EMPTY 200 (2026/QTR4 live), so refusing it would be a false
1611
+ * "not available"; the two honest terminal paths (200+header+0rows → genuine-empty;
1612
+ * a bounds-valid 403 → the ambiguous reclassify) handle everything else.
1613
+ * THEN assert the built URL is https on the fixed www.sec.gov host.
1614
+ */
1615
+ function buildFullIndexUrl(year: number, quarter: number): string {
1616
+ const maxYear = currentUtcYear();
1617
+ if (
1618
+ !Number.isInteger(year) ||
1619
+ year < EDGAR_FULLINDEX_START_YEAR ||
1620
+ year > maxYear
1621
+ ) {
1622
+ fullIndexInvalid(
1623
+ `edgar_filing_index: year ${JSON.stringify(year)} must be an integer in [${EDGAR_FULLINDEX_START_YEAR}, ${maxYear}] (EDGAR full-index begins 1993 Q1; a future year has no published quarter index) — refused before any fetch (path-segment bounds guard).`,
1624
+ );
1625
+ }
1626
+ if (!Number.isInteger(quarter) || quarter < 1 || quarter > 4) {
1627
+ fullIndexInvalid(
1628
+ `edgar_filing_index: quarter ${JSON.stringify(quarter)} must be an integer in {1,2,3,4} — refused before any fetch (path-segment bounds guard).`,
1629
+ );
1630
+ }
1631
+ const built = `${FULLINDEX_BASE}/${year}/QTR${quarter}/${FULLINDEX_FILE}`;
1632
+ let parsed: URL;
1633
+ try {
1634
+ parsed = new URL(built);
1635
+ } catch {
1636
+ fullIndexInvalid(
1637
+ `edgar_filing_index: could not construct a valid URL from year=${year} quarter=${quarter} — refused before any fetch.`,
1638
+ );
1639
+ }
1640
+ if (parsed.protocol !== "https:" || parsed.hostname !== FULLINDEX_HOST) {
1641
+ fullIndexInvalid(
1642
+ `edgar_filing_index: constructed URL host/scheme is not https://${FULLINDEX_HOST} (${parsed.protocol}//${parsed.hostname}) — refused before any fetch (fixed-host assertion).`,
1643
+ );
1644
+ }
1645
+ return built;
1646
+ }
1647
+
1648
+ /** One filing row from master.idx. CIK stays a STRING (never num-coerced). */
1649
+ export type FilingIndexRow = {
1650
+ cik: string | null; // raw CIK from the row (e.g. "1000045")
1651
+ cikPadded: string | null; // padCik(cik) canonical 10-digit — the join key to the per-filer edgar tools
1652
+ companyName: string | null;
1653
+ formType: string | null;
1654
+ dateFiled: string | null; // "YYYY-MM-DD"
1655
+ filename: string | null; // raw archive path "edgar/data/<cik>/<accession>.txt"
1656
+ filingUrl: string | null; // FILING_ARCHIVE_BASE + filename → a resolvable URL (null if filename null)
1657
+ };
1658
+
1659
+ type ParsedFullIndex = {
1660
+ rows: FilingIndexRow[];
1661
+ malformedRows: number; // data rows that did NOT split into 5 pipe fields (skipped + counted)
1662
+ totalIsLowerBound: boolean; // true iff the MAX_INDEX_ROWS safety ceiling was hit
1663
+ };
1664
+
1665
+ /**
1666
+ * Split `line` on its FIRST `n` occurrences of `|`; the remainder (which may
1667
+ * itself contain `|`) is the FINAL element (the BOUNDED pipe-split — minor fix). For
1668
+ * n=4 a well-formed row yields exactly 5 fields, and a row carrying an EXTRA `|`
1669
+ * still yields 5 fields (the extra pipe stays inside the rejoined tail) rather than
1670
+ * being dropped as "malformed" — so `totalAvailable` is never SILENTLY under-counted
1671
+ * by an over-split. A row with FEWER than 4 pipes returns <5 elements → malformed.
1672
+ */
1673
+ function splitOnFirstPipes(line: string, n: number): string[] {
1674
+ const out: string[] = [];
1675
+ let start = 0;
1676
+ for (let k = 0; k < n; k++) {
1677
+ const idx = line.indexOf("|", start);
1678
+ if (idx === -1) break;
1679
+ out.push(line.slice(start, idx));
1680
+ start = idx + 1;
1681
+ }
1682
+ out.push(line.slice(start));
1683
+ return out;
1684
+ }
1685
+
1686
+ /**
1687
+ * Parse the raw master.idx body into `FilingIndexRow[]`.
1688
+ *
1689
+ * DRIFT keys on the ABSENCE of the `CIK|Company Name|Form Type|Date Filed|Filename`
1690
+ * header + the `----` dashes boundary ONLY (M1) — a non-index / error / format-changed
1691
+ * body served with HTTP 200 (e.g. an S3 error HTML page) → THROW `driftError`. A
1692
+ * body WITH the header+dashes but ZERO data rows is a GENUINE-EMPTY quarter (2026/QTR4
1693
+ * live) → returned to the caller (NOT thrown). A body WITH the header+dashes whose
1694
+ * EVERY data row fails the 5-field split → THROW `driftError` (all-malformed = format
1695
+ * drift). The fixed preamble (`Description:` …) BEFORE the header is skipped.
1696
+ */
1697
+ export function parseFullIndex(
1698
+ body: string,
1699
+ maxRows: number = MAX_INDEX_ROWS,
1700
+ ): ParsedFullIndex {
1701
+ const lines = body.split("\n");
1702
+ // Locate the header line immediately followed by the `----` dashes boundary.
1703
+ let dashesIdx = -1;
1704
+ for (let i = 0; i < lines.length - 1; i++) {
1705
+ if ((lines[i] ?? "").replace(/\r$/, "").trim() === FULLINDEX_HEADER) {
1706
+ const next = (lines[i + 1] ?? "").replace(/\r$/, "");
1707
+ if (/^-{5,}\s*$/.test(next.trim())) {
1708
+ dashesIdx = i + 1;
1709
+ break;
1710
+ }
1711
+ }
1712
+ }
1713
+ if (dashesIdx === -1) {
1714
+ throw driftError(
1715
+ FULLINDEX_LABEL,
1716
+ `edgar:full-index body is missing the '${FULLINDEX_HEADER}' header / '----' dashes boundary (a non-index / error / format-changed body served with HTTP 200) — refusing to report an empty result (schema drift, NOT a genuine-empty quarter).`,
1717
+ );
1718
+ }
1719
+ const rows: FilingIndexRow[] = [];
1720
+ let malformedRows = 0;
1721
+ let totalIsLowerBound = false;
1722
+ for (let i = dashesIdx + 1; i < lines.length; i++) {
1723
+ const line = (lines[i] ?? "").replace(/\r$/, "");
1724
+ if (line.trim() === "") continue; // blank line (e.g. the trailing newline) — skip
1725
+ // Safety ceiling (never a SILENT truncation): once the scanned data-row count
1726
+ // reaches MAX_INDEX_ROWS, stop + set totalIsLowerBound. The live ~370K is well
1727
+ // under 500K, so the normal path never trips it and totals stay EXACT.
1728
+ if (rows.length + malformedRows >= maxRows) {
1729
+ totalIsLowerBound = true;
1730
+ break;
1731
+ }
1732
+ const parts = splitOnFirstPipes(line, 4);
1733
+ if (parts.length < 5) {
1734
+ malformedRows++;
1735
+ continue;
1736
+ }
1737
+ const cik = str(parts[0]);
1738
+ const filename = str(parts[4]);
1739
+ rows.push({
1740
+ cik,
1741
+ cikPadded: cik === null ? null : padCik(cik),
1742
+ companyName: str(parts[1]),
1743
+ formType: str(parts[2]),
1744
+ dateFiled: str(parts[3]),
1745
+ filename,
1746
+ filingUrl: filename === null ? null : FILING_ARCHIVE_BASE + filename,
1747
+ });
1748
+ }
1749
+ // M1 — header+dashes PRESENT but EVERY data row failed the split ⇒ drift (a
1750
+ // format change). 0 data rows total (rows.length===0 && malformedRows===0) ⇒
1751
+ // GENUINE-EMPTY (returned by the caller, NOT thrown).
1752
+ if (rows.length === 0 && malformedRows > 0) {
1753
+ throw driftError(
1754
+ FULLINDEX_LABEL,
1755
+ `edgar:full-index has the header/dashes boundary but ALL ${malformedRows} data row(s) failed the 5-field pipe split (format drift) — refusing to report an empty result.`,
1756
+ );
1757
+ }
1758
+ return { rows, malformedRows, totalIsLowerBound };
1759
+ }
1760
+
1761
+ // A1 — a BOUNDED LRU for the full-index RAW TEXT. DELIBERATELY NOT the shared,
1762
+ // UNBOUNDED `memoize` (cache.ts): as the review flagged, memoizing ~370K parsed
1763
+ // row-objects per (year,quarter) with no eviction would OOM when a client sweeps
1764
+ // many quarters. This caps at FULLINDEX_CACHE_MAX distinct quarters (a HARD size
1765
+ // bound), caches the RAW TEXT (leaner than parsed rows) re-parsed per call (cheap),
1766
+ // and TTL-splits: the CURRENT quarter grows daily → short TTL; a CLOSED quarter is
1767
+ // immutable → long TTL. Map insertion order = LRU recency; the oldest quarter is
1768
+ // evicted past the cap. A repeat call for the same quarter (different filters/
1769
+ // offset) re-uses the cached text — NO second ~33MB download.
1770
+ const FULLINDEX_CACHE_MAX = 3;
1771
+ const FULLINDEX_TTL_CURRENT_MS = 5 * 60 * 1000; // current quarter: 5 min (grows daily)
1772
+ const FULLINDEX_TTL_CLOSED_MS = 6 * 60 * 60 * 1000; // closed quarter: 6h (immutable)
1773
+ const fullIndexCache = new Map<string, { text: string; expiresAt: number }>();
1774
+
1775
+ /** Fetch the whole quarter's master.idx text (getEdgar VERBATIM + `.text()`),
1776
+ * through the bounded LRU (A1). getEdgar sets the mandatory UA + gzip + the ≤10
1777
+ * req/s gate + the 15s timeout; a slow body-read → abort → getEdgar's honest
1778
+ * upstream_unavailable throw (never a fake-empty). */
1779
+ async function fetchFullIndexText(
1780
+ built: string,
1781
+ year: number,
1782
+ quarter: number,
1783
+ ): Promise<string> {
1784
+ const key = `${year}:${quarter}`;
1785
+ const now = Date.now();
1786
+ const hit = fullIndexCache.get(key);
1787
+ if (hit && hit.expiresAt > now) {
1788
+ fullIndexCache.delete(key); // LRU touch → re-insert as the newest
1789
+ fullIndexCache.set(key, hit);
1790
+ return hit.text;
1791
+ }
1792
+ if (hit) fullIndexCache.delete(key); // expired
1793
+ const r = await getEdgar(built, FULLINDEX_LABEL);
1794
+ const text = await r.text();
1795
+ const ttl = isCurrentQuarter(year, quarter)
1796
+ ? FULLINDEX_TTL_CURRENT_MS
1797
+ : FULLINDEX_TTL_CLOSED_MS;
1798
+ fullIndexCache.set(key, { text, expiresAt: now + ttl });
1799
+ while (fullIndexCache.size > FULLINDEX_CACHE_MAX) {
1800
+ const oldest = fullIndexCache.keys().next().value as string | undefined;
1801
+ if (oldest === undefined) break;
1802
+ fullIndexCache.delete(oldest);
1803
+ }
1804
+ return text;
1805
+ }
1806
+
1807
+ /** For tests: evict the full-index bounded LRU (mirrors cache.ts `_clearCache` for
1808
+ * this dedicated cache, which the shared `_clearCache` does not touch). */
1809
+ export function _resetFullIndexCache(): void {
1810
+ fullIndexCache.clear();
1811
+ }
1812
+
1813
+ /**
1814
+ * Read the SEC EDGAR quarterly full-index for (year, quarter) and return the
1815
+ * filings matching the given CLIENT-SIDE filters (form / CIK / company substring /
1816
+ * date range), offset-paginated, with the EXACT total match count for the quarter.
1817
+ *
1818
+ * HONESTY (ADR-0026 v2):
1819
+ * - FULL-SCAN → `totalAvailable` is the EXACT filtered match count across the WHOLE
1820
+ * quarter (never a page length, never a byte-capped under-count — SEC ignores Range).
1821
+ * - A bounds-valid but unpublished quarter 403s (getEdgar mislabels it rate_limited);
1822
+ * TOOL-LOCAL reclassify to an AMBIGUOUS both-causes error (unpublished quarter OR
1823
+ * the 10 req/s rate-block) — never a bare rate-limit, never a fake-empty.
1824
+ * - Drift on header/dashes ABSENCE or an all-malformed body (THROW); header+0-rows ⇒
1825
+ * genuine-empty (complete:true). A future year / bad quarter ⇒ invalid_input, 0 fetch.
1826
+ * - CIK stays a STRING; every column via the module-local `str` (null-never-"").
1827
+ * - `companyContains` is a LITERAL case-insensitive substring (C110 N/A — no token split).
1828
+ */
1829
+ export async function filingIndex(args: {
1830
+ year: number;
1831
+ quarter: number;
1832
+ formType?: string;
1833
+ cik?: string | number;
1834
+ companyContains?: string;
1835
+ dateFrom?: string;
1836
+ dateTo?: string;
1837
+ limit?: number;
1838
+ offset?: number;
1839
+ }): Promise<MetaBundle> {
1840
+ const { year, quarter } = args;
1841
+ // S1 — build + re-validate the path BEFORE any fetch (0 fetch on a bad year/quarter).
1842
+ const built = buildFullIndexUrl(year, quarter);
1843
+
1844
+ // Fetch the WHOLE quarter (getEdgar VERBATIM + `.text()`, bounded-LRU cached). A
1845
+ // bounds-valid 403 (an unpublished quarter) is reclassified TOOL-LOCAL — getEdgar
1846
+ // is UNTOUCHED (the other 5 edgar tools stay byte-identical).
1847
+ let body: string;
1848
+ try {
1849
+ body = await fetchFullIndexText(built, year, quarter);
1850
+ } catch (e) {
1851
+ if (
1852
+ e instanceof ToolErrorCarrier &&
1853
+ e.toolError.kind === "rate_limited" &&
1854
+ e.toolError.upstreamStatus === 403
1855
+ ) {
1856
+ // 403-not-404 reclassify (fact #2): a bad/unpublished full-index quarter
1857
+ // returns 403 `AccessDenied` (NOT 404); getEdgar's F6 can't match /automated|
1858
+ // undeclared/ so it labels it rate_limited. Re-surface an AMBIGUOUS both-causes
1859
+ // error — NEVER a bare "rate limited", NEVER a fake-empty. A REAL 429 (status
1860
+ // 429, not 403) is NOT caught here → stays an honest rate_limited.
1861
+ throw new ToolErrorCarrier({
1862
+ kind: "upstream_unavailable",
1863
+ message: `SEC returned HTTP 403 for the full-index path ${FULLINDEX_BASE}/${year}/QTR${quarter}/${FULLINDEX_FILE}. This is AMBIGUOUS: EITHER the ${year} QTR${quarter} index is not published yet (a too-early / non-existent quarter returns 403, not 404) OR SEC is rate-limiting this IP at the 10 req/s ceiling (~10-minute block). NOT fabricated as empty. Verify the quarter is a real past/current EDGAR quarter and retry after ~10 minutes.`,
1864
+ retryable: true,
1865
+ retryAfterSeconds: 600,
1866
+ upstreamStatus: 403,
1867
+ upstreamEndpoint: FULLINDEX_LABEL,
1868
+ });
1869
+ }
1870
+ throw e; // schema_drift / invalid_input(UA) / upstream_unavailable / real 429 — loud
1871
+ }
1872
+
1873
+ // FULL-SCAN → parse ALL rows past the preamble. Drift (header-absent / all-malformed)
1874
+ // THROWS; header+0-rows ⇒ genuine-empty (below).
1875
+ const parsed = parseFullIndex(body);
1876
+ const all = parsed.rows;
1877
+
1878
+ // CLIENT-SIDE filters — ZERO query string (none reach the URL). cik via padCik
1879
+ // both-sides; companyContains a LITERAL case-insensitive substring (no token split);
1880
+ // dates are ISO string compares (the column is already YYYY-MM-DD).
1881
+ const formNeedle = args.formType?.trim()
1882
+ ? args.formType.trim().toLowerCase()
1883
+ : null;
1884
+ const cikFilter =
1885
+ args.cik != null && String(args.cik).trim() !== ""
1886
+ ? padCik(args.cik)
1887
+ : null;
1888
+ const companyNeedle = args.companyContains?.trim()
1889
+ ? args.companyContains.trim().toLowerCase()
1890
+ : null;
1891
+ const dateFrom = args.dateFrom;
1892
+ const dateTo = args.dateTo;
1893
+
1894
+ const matches = all.filter((row) => {
1895
+ if (formNeedle !== null) {
1896
+ if (row.formType === null || row.formType.toLowerCase() !== formNeedle)
1897
+ return false;
1898
+ }
1899
+ if (cikFilter !== null) {
1900
+ if (row.cikPadded !== cikFilter) return false;
1901
+ }
1902
+ if (companyNeedle !== null) {
1903
+ if (
1904
+ row.companyName === null ||
1905
+ !row.companyName.toLowerCase().includes(companyNeedle)
1906
+ )
1907
+ return false;
1908
+ }
1909
+ if (dateFrom !== undefined) {
1910
+ if (row.dateFiled === null || row.dateFiled < dateFrom) return false;
1911
+ }
1912
+ if (dateTo !== undefined) {
1913
+ if (row.dateFiled === null || row.dateFiled > dateTo) return false;
1914
+ }
1915
+ return true;
1916
+ });
1917
+
1918
+ const limit = args.limit ?? 100;
1919
+ const offset = args.offset ?? 0;
1920
+ const totalAvailable = matches.length; // EXACT (full-scan) — never a page length
1921
+ const page = matches.slice(offset, offset + limit);
1922
+ const returned = page.length;
1923
+ const hasMore = offset + returned < totalAvailable;
1924
+ const nextOffset = hasMore ? offset + returned : null;
1925
+
1926
+ const filtersApplied: string[] = [];
1927
+ if (formNeedle !== null) filtersApplied.push("formType");
1928
+ if (cikFilter !== null) filtersApplied.push("cik");
1929
+ if (companyNeedle !== null) filtersApplied.push("companyContains");
1930
+ if (dateFrom !== undefined) filtersApplied.push("dateFrom");
1931
+ if (dateTo !== undefined) filtersApplied.push("dateTo");
1932
+
1933
+ const notes: string[] = [];
1934
+ // Index-vs-live-submissions caveat (always).
1935
+ notes.push(
1936
+ `The quarterly full-index (master.idx) is a POINT-IN-TIME snapshot of ${year} QTR${quarter}'s EDGAR dissemination feed — it covers ONLY that quarter. For the LATEST or complete-history filings of a KNOWN filer use edgar_company_filings (it fans out older shards); for a text query across 2001-present use edgar_full_text_search.`,
1937
+ );
1938
+ // Full-scan / exact-total disclosure + the ~33MB size caveat.
1939
+ notes.push(
1940
+ `The whole ${year} QTR${quarter} index was downloaded and FULL-SCANNED (~33MB / hundreds of thousands of rows); totalAvailable (${totalAvailable}) is the EXACT count of filings matching the filters across the ENTIRE quarter (not a page length, not a byte-capped subset — SEC ignores HTTP Range). This page contains ${returned} of ${totalAvailable}; page via _meta.pagination.nextOffset for the rest.`,
1941
+ );
1942
+ if (isCurrentQuarter(year, quarter)) {
1943
+ notes.push(
1944
+ `${year} QTR${quarter} is the CURRENT calendar quarter — it GROWS daily as new filings disseminate, so totalAvailable is EXACT AS-OF this (short-cached) snapshot, not exact-forever. A closed past quarter is immutable.`,
1945
+ );
1946
+ }
1947
+ if (companyNeedle !== null) {
1948
+ notes.push(
1949
+ `companyContains is a case-insensitive LITERAL substring match on the Company Name column — a multi-word value matches as ONE contiguous string, NOT AND/OR-tokenized.`,
1950
+ );
1951
+ }
1952
+ if (formNeedle !== null) {
1953
+ notes.push(
1954
+ `formType is a case-insensitive EXACT match on the Form Type column ("${args.formType?.trim()}" matches that form only — e.g. "8-K" does NOT match "8-K/A"). Pass each amendment variant separately.`,
1955
+ );
1956
+ }
1957
+ if (parsed.malformedRows > 0) {
1958
+ notes.push(
1959
+ `${parsed.malformedRows} row(s) did not split into 5 pipe fields and were skipped (tolerated as stray malformed rows; a body with ZERO valid rows would instead be refused as schema drift).`,
1960
+ );
1961
+ }
1962
+ if (parsed.totalIsLowerBound) {
1963
+ notes.push(
1964
+ `The scan hit the MAX_INDEX_ROWS safety ceiling (${MAX_INDEX_ROWS}); totals are a LOWER BOUND — the quarter index is larger than expected (possible format drift). See totalIsLowerBound.`,
1965
+ );
1966
+ }
1967
+ if (totalAvailable === 0) {
1968
+ notes.push(
1969
+ filtersApplied.length
1970
+ ? `0 filings in ${year} QTR${quarter} matched the filters (${filtersApplied.join(", ")}). This is an EXACT ZERO over the full quarter index — NOT a truncation, NOT an outage.`
1971
+ : `0 filings in ${year} QTR${quarter} (the quarter index has no data rows). This is an EXACT ZERO over the full quarter index — NOT a truncation, NOT an outage.`,
1972
+ );
1973
+ }
1974
+
1975
+ const meta: Partial<ResponseMeta> = {
1976
+ returned,
1977
+ totalAvailable,
1978
+ filtersApplied,
1979
+ pagination: { offset, limit, hasMore, nextOffset },
1980
+ notes,
1981
+ };
1982
+ if (parsed.totalIsLowerBound) meta.totalIsLowerBound = true;
1983
+
1984
+ return withMeta(
1985
+ {
1986
+ year,
1987
+ quarter,
1988
+ indexFile: FULLINDEX_FILE,
1989
+ returned,
1990
+ totalAvailable,
1991
+ filings: page,
1992
+ },
1993
+ edgarMeta(meta),
1994
+ );
1995
+ }
1996
+
1997
+ // ─── Tool 7: edgar_daily_filing_index (ADR-0027) ──────────────────
1998
+ // The per-DAY sibling of edgar_filing_index: read the SEC EDGAR DAILY-index
1999
+ // (www.sec.gov/Archives/edgar/daily-index/<year>/QTR<n>/master.YYYYMMDD.idx — a
2000
+ // COMPLETE cross-filer index of EVERY filer's EVERY filing on ONE calendar day,
2001
+ // pipe-delimited, ~0.5–1.2MB / ~8K rows — ~30× smaller than the quarterly file),
2002
+ // FULL-SCAN it, apply CLIENT-SIDE filters, and return offset-paginated filings
2003
+ // with an EXACT total. It answers the monitoring/alerting question the quarterly
2004
+ // tool cannot without a whole-quarter download ("every 8-K filed on 2024-01-03").
2005
+ // Reuses getEdgar VERBATIM (UA + gzip + the ≤10 req/s throughGate — UNTOUCHED, so
2006
+ // the other 6 edgar tools stay byte-identical), `.text()`, padCik, the module-local
2007
+ // `str`, `driftError`, `buildMeta`/`withMeta`, `edgarMeta`, `splitOnFirstPipes`,
2008
+ // `MAX_INDEX_ROWS`, `currentUtcYear`, `FILING_ARCHIVE_BASE`, `FULLINDEX_HOST`. A
2009
+ // DEDICATED daily parser (do NOT reuse parseFullIndex — the daily header is
2010
+ // `…|Date Filed|File Name` WITH A SPACE, not the full-index `Filename`).
2011
+ //
2012
+ // LIVE-VERIFIED 2026-07-12 (SEC UA `cliwant-mcp-sam-gov research (…)`):
2013
+ // - master.20240103.idx (Wed) → 200; header VERBATIM `CIK|Company Name|Form Type|
2014
+ // Date Filed|File Name` (File Name WITH a space), then `----`; Date Filed column
2015
+ // compact `20240103`; 8426 data rows; 8-K = 220, 8-K/A = 5 (distinct/exact).
2016
+ // - master.20240106.idx (Sat) → 403 AccessDenied XML; index.json 2024/QTR1 does
2017
+ // NOT list it and its newest master is 20240329 ⇒ TRUE genuine-absent (found:false,
2018
+ // complete:true) — a real gap INSIDE the covered range.
2019
+ // - ★M1 recency (live Sun 2026-07-12): 2026/QTR3 index.json lists master ONLY through
2020
+ // master.20260709 (Thu); Fri 20260710 (a normal trading day) is UNLISTED and its
2021
+ // .idx 403s ⇒ requestedYyyymmdd > maxListedMasterDate ⇒ NOT-YET-DISSEMINATED
2022
+ // (found:false, complete:FALSE), NEVER a confident genuine-absent complete:true.
2023
+ //
2024
+ // ★HONESTY: the 403-disambiguation is DESIGN (b) — fetch the .idx FIRST (the happy
2025
+ // path pays ZERO oracle cost); consult the index.json existence oracle ONLY on a 403,
2026
+ // RECENCY-AWARE per M1 (maxListedMasterDate: newer-than-listed ⇒ not-yet-disseminated
2027
+ // complete:FALSE; listed-range gap ⇒ true genuine-absent complete:true; listed-but-403
2028
+ // ⇒ honest rate_limited; oracle-inconclusive ⇒ ambiguous both-causes upstream_unavailable).
2029
+ // The 403-reclassify is TOOL-LOCAL (getEdgar UNTOUCHED). M2: an EXACT date round-trip
2030
+ // PRE-fetch rejects Feb-30 / day-40 / non-leap-Feb-29 / a future date (invalid_input, 0 GET).
2031
+
2032
+ const DAILYINDEX_BASE = "https://www.sec.gov/Archives/edgar/daily-index";
2033
+ const DAILYINDEX_FILE = (yyyymmdd: string): string => `master.${yyyymmdd}.idx`; // the single backing file (pipe-delimited)
2034
+ const DAILYINDEX_LABEL = "edgar:daily-index"; // host+path only; keyless ⇒ no token can appear
2035
+ const DAILYINDEX_ORACLE_LABEL = "edgar:daily-index:index.json";
2036
+ // ★ File Name WITH A SPACE (the daily header) — the full-index uses `Filename` (no
2037
+ // space); a parser keying drift on `Filename` would FALSE-DRIFT on every valid daily index.
2038
+ const DAILYINDEX_HEADER = "CIK|Company Name|Form Type|Date Filed|File Name";
2039
+ const EDGAR_DAILY_START_YEAR = 1994; // EDGAR daily-index begins 1994 Q1 (conservative lower bound)
2040
+
2041
+ // A daily-specific bounded LRU of the RAW TEXT keyed by yyyymmdd (mirrors the shipped
2042
+ // fullIndexCache; NOT the shared unbounded `memoize`, which a many-day sweep would OOM).
2043
+ // Days are tiny (~0.5–1.2MB) → a larger cap than full-index's 3 is cheap. TTL splits:
2044
+ // TODAY may still be posting until ~22:00 ET → short TTL; a PAST day is immutable →
2045
+ // long TTL. The index.json oracle is fetched ONLY on the rare 403 path and is NOT cached.
2046
+ const DAILYINDEX_CACHE_MAX = 8;
2047
+ const DAILYINDEX_TTL_TODAY_MS = 5 * 60 * 1000; // today may still be posting → short TTL
2048
+ const DAILYINDEX_TTL_CLOSED_MS = 24 * 60 * 60 * 1000; // a past day is immutable → long TTL
2049
+ const dailyIndexCache = new Map<string, { text: string; expiresAt: number }>();
2050
+
2051
+ /** Throw the pre-fetch bounds/injection guard error (invalid_input, 0 fetch). */
2052
+ function dailyIndexInvalid(message: string): never {
2053
+ throw new ToolErrorCarrier({
2054
+ kind: "invalid_input",
2055
+ message,
2056
+ retryable: false,
2057
+ upstreamEndpoint: DAILYINDEX_LABEL,
2058
+ });
2059
+ }
2060
+
2061
+ /** Today's date as compact `YYYYMMDD` in UTC (call-time fresh — survives a day rollover). */
2062
+ function todayUtcYyyymmdd(): string {
2063
+ const now = new Date();
2064
+ const y = now.getUTCFullYear();
2065
+ const m = now.getUTCMonth() + 1;
2066
+ const d = now.getUTCDate();
2067
+ return `${y}${String(m).padStart(2, "0")}${String(d).padStart(2, "0")}`;
2068
+ }
2069
+
2070
+ /** True iff the compact `yyyymmdd` is TODAY (UTC) — the freshness/cache-TTL signal. */
2071
+ function isTodayUtc(yyyymmdd: string): boolean {
2072
+ return yyyymmdd === todayUtcYyyymmdd();
2073
+ }
2074
+
2075
+ /**
2076
+ * Normalize the row's compact `Date Filed` (`YYYYMMDD`, fact #3) to ISO `YYYY-MM-DD`
2077
+ * for the `dateFiled` output (so it matches edgar_filing_index / edgar_company_filings
2078
+ * and is human-readable). A non-8-digit value is kept as-is (defensive; via `str`).
2079
+ */
2080
+ function normDailyDate(raw: unknown): string | null {
2081
+ const s = str(raw);
2082
+ if (s === null) return null;
2083
+ if (/^\d{8}$/.test(s)) return `${s.slice(0, 4)}-${s.slice(4, 6)}-${s.slice(6, 8)}`;
2084
+ return s;
2085
+ }
2086
+
2087
+ /**
2088
+ * Build the daily-index URL from the caller-influenced segments (year, quarter,
2089
+ * yyyymmdd — all DERIVED from one validated ISO date). BELT-AND-SUSPENDERS behind Zod
2090
+ * (a direct handler call could bypass it): re-run the integer/range guards + the
2091
+ * 8-digit-tied-to-year regex, then assert the built URL is https on the fixed
2092
+ * www.sec.gov host. The 8-digit regex admits NO `%`/`/`/`.`/`\`/`..`/`%2F` (JS `\d`
2093
+ * is ASCII-only). This does NOT reject a well-formed-but-NONEXISTENT day (Feb-30) —
2094
+ * the pre-fetch date round-trip (M2, in the handler) is the SOLE defense for that.
2095
+ */
2096
+ function buildDailyIndexUrl(year: number, quarter: number, yyyymmdd: string): string {
2097
+ const maxYear = currentUtcYear();
2098
+ if (!Number.isInteger(year) || year < EDGAR_DAILY_START_YEAR || year > maxYear) {
2099
+ dailyIndexInvalid(
2100
+ `edgar_daily_filing_index: year ${JSON.stringify(year)} must be an integer in [${EDGAR_DAILY_START_YEAR}, ${maxYear}] (EDGAR daily-index begins 1994 Q1; a future year has no published daily index) — refused before any fetch (path-segment bounds guard).`,
2101
+ );
2102
+ }
2103
+ if (!Number.isInteger(quarter) || quarter < 1 || quarter > 4) {
2104
+ dailyIndexInvalid(
2105
+ `edgar_daily_filing_index: quarter ${JSON.stringify(quarter)} must be an integer in {1,2,3,4} — refused before any fetch (path-segment bounds guard).`,
2106
+ );
2107
+ }
2108
+ if (!/^\d{8}$/.test(yyyymmdd) || yyyymmdd.slice(0, 4) !== String(year)) {
2109
+ dailyIndexInvalid(
2110
+ `edgar_daily_filing_index: yyyymmdd ${JSON.stringify(yyyymmdd)} must be 8 digits whose year prefix === ${year} — refused before any fetch (path-segment injection guard; ties the compact day to the validated year, admits no slash/dot/percent/'..').`,
2111
+ );
2112
+ }
2113
+ const built = `${DAILYINDEX_BASE}/${year}/QTR${quarter}/${DAILYINDEX_FILE(yyyymmdd)}`;
2114
+ let parsed: URL;
2115
+ try {
2116
+ parsed = new URL(built);
2117
+ } catch {
2118
+ dailyIndexInvalid(
2119
+ `edgar_daily_filing_index: could not construct a valid URL from year=${year} quarter=${quarter} yyyymmdd=${yyyymmdd} — refused before any fetch.`,
2120
+ );
2121
+ }
2122
+ if (parsed.protocol !== "https:" || parsed.hostname !== FULLINDEX_HOST) {
2123
+ dailyIndexInvalid(
2124
+ `edgar_daily_filing_index: constructed URL host/scheme is not https://${FULLINDEX_HOST} (${parsed.protocol}//${parsed.hostname}) — refused before any fetch (fixed-host assertion).`,
2125
+ );
2126
+ }
2127
+ return built;
2128
+ }
2129
+
2130
+ /** Build the quarter's index.json existence-oracle URL (same year/quarter guard). */
2131
+ function buildDailyOracleUrl(year: number, quarter: number): string {
2132
+ const maxYear = currentUtcYear();
2133
+ if (!Number.isInteger(year) || year < EDGAR_DAILY_START_YEAR || year > maxYear) {
2134
+ dailyIndexInvalid(
2135
+ `edgar_daily_filing_index: oracle year ${JSON.stringify(year)} must be an integer in [${EDGAR_DAILY_START_YEAR}, ${maxYear}] — refused before any fetch (path-segment bounds guard).`,
2136
+ );
2137
+ }
2138
+ if (!Number.isInteger(quarter) || quarter < 1 || quarter > 4) {
2139
+ dailyIndexInvalid(
2140
+ `edgar_daily_filing_index: oracle quarter ${JSON.stringify(quarter)} must be an integer in {1,2,3,4} — refused before any fetch (path-segment bounds guard).`,
2141
+ );
2142
+ }
2143
+ const built = `${DAILYINDEX_BASE}/${year}/QTR${quarter}/index.json`;
2144
+ let parsed: URL;
2145
+ try {
2146
+ parsed = new URL(built);
2147
+ } catch {
2148
+ dailyIndexInvalid(
2149
+ `edgar_daily_filing_index: could not construct a valid oracle URL from year=${year} quarter=${quarter} — refused before any fetch.`,
2150
+ );
2151
+ }
2152
+ if (parsed.protocol !== "https:" || parsed.hostname !== FULLINDEX_HOST) {
2153
+ dailyIndexInvalid(
2154
+ `edgar_daily_filing_index: constructed oracle URL host/scheme is not https://${FULLINDEX_HOST} (${parsed.protocol}//${parsed.hostname}) — refused before any fetch (fixed-host assertion).`,
2155
+ );
2156
+ }
2157
+ return built;
2158
+ }
2159
+
2160
+ /**
2161
+ * Parse the raw daily master.YYYYMMDD.idx body into `FilingIndexRow[]` (DEDICATED —
2162
+ * NOT parseFullIndex). DRIFT keys on the ABSENCE of the daily `CIK|Company Name|Form
2163
+ * Type|Date Filed|File Name` header (★ File Name WITH A SPACE) + the `----` dashes
2164
+ * boundary ONLY (M1/fact #2) — a non-index / error / format-changed body served with
2165
+ * HTTP 200 → THROW `driftError`. A body WITH the header+dashes but ZERO data rows is a
2166
+ * GENUINE-EMPTY day → returned (NOT thrown; near-unreachable for a real trading day but
2167
+ * honest). A body WITH the header+dashes whose EVERY data row fails the 5-field split →
2168
+ * THROW `driftError` (all-malformed = format drift). Date Filed is normalized to ISO.
2169
+ */
2170
+ export function parseDailyIndex(
2171
+ body: string,
2172
+ maxRows: number = MAX_INDEX_ROWS,
2173
+ ): ParsedFullIndex {
2174
+ const lines = body.split("\n");
2175
+ // Locate the DAILY header line immediately followed by the `----` dashes boundary.
2176
+ let dashesIdx = -1;
2177
+ for (let i = 0; i < lines.length - 1; i++) {
2178
+ if ((lines[i] ?? "").replace(/\r$/, "").trim() === DAILYINDEX_HEADER) {
2179
+ const next = (lines[i + 1] ?? "").replace(/\r$/, "");
2180
+ if (/^-{5,}\s*$/.test(next.trim())) {
2181
+ dashesIdx = i + 1;
2182
+ break;
2183
+ }
2184
+ }
2185
+ }
2186
+ if (dashesIdx === -1) {
2187
+ throw driftError(
2188
+ DAILYINDEX_LABEL,
2189
+ `edgar:daily-index body is missing the '${DAILYINDEX_HEADER}' header / '----' dashes boundary (a non-index / error / format-changed body served with HTTP 200 — note the daily header is 'File Name' WITH a space, NOT the full-index 'Filename') — refusing to report an empty result (schema drift, NOT a genuine-absent day).`,
2190
+ );
2191
+ }
2192
+ const rows: FilingIndexRow[] = [];
2193
+ let malformedRows = 0;
2194
+ let totalIsLowerBound = false;
2195
+ for (let i = dashesIdx + 1; i < lines.length; i++) {
2196
+ const line = (lines[i] ?? "").replace(/\r$/, "");
2197
+ if (line.trim() === "") continue; // blank line (e.g. the trailing newline) — skip
2198
+ if (rows.length + malformedRows >= maxRows) {
2199
+ totalIsLowerBound = true; // safety ceiling — never a SILENT truncation
2200
+ break;
2201
+ }
2202
+ const parts = splitOnFirstPipes(line, 4); // bounded → a `|`-in-company row stays 5 fields
2203
+ if (parts.length < 5) {
2204
+ malformedRows++;
2205
+ continue;
2206
+ }
2207
+ const cik = str(parts[0]);
2208
+ const filename = str(parts[4]);
2209
+ rows.push({
2210
+ cik,
2211
+ cikPadded: cik === null ? null : padCik(cik),
2212
+ companyName: str(parts[1]),
2213
+ formType: str(parts[2]),
2214
+ dateFiled: normDailyDate(parts[3]), // compact YYYYMMDD → ISO YYYY-MM-DD
2215
+ filename,
2216
+ filingUrl: filename === null ? null : FILING_ARCHIVE_BASE + filename,
2217
+ });
2218
+ }
2219
+ if (rows.length === 0 && malformedRows > 0) {
2220
+ throw driftError(
2221
+ DAILYINDEX_LABEL,
2222
+ `edgar:daily-index has the header/dashes boundary but ALL ${malformedRows} data row(s) failed the 5-field pipe split (format drift) — refusing to report an empty result.`,
2223
+ );
2224
+ }
2225
+ return { rows, malformedRows, totalIsLowerBound };
2226
+ }
2227
+
2228
+ /**
2229
+ * Fetch the whole day's master.YYYYMMDD.idx text (getEdgar VERBATIM + `.text()`),
2230
+ * through the bounded LRU. getEdgar sets the mandatory UA + gzip + the ≤10 req/s gate
2231
+ * + the 15s timeout; a 403 throws (rate_limited/403 — caught tool-locally by the
2232
+ * handler for the oracle disambiguation), so ONLY a 200 body is ever cached. A slow
2233
+ * body-read → abort → getEdgar's honest upstream_unavailable throw (never a fake-empty).
2234
+ */
2235
+ async function fetchDailyIndexText(built: string, yyyymmdd: string): Promise<string> {
2236
+ const key = yyyymmdd;
2237
+ const now = Date.now();
2238
+ const hit = dailyIndexCache.get(key);
2239
+ if (hit && hit.expiresAt > now) {
2240
+ dailyIndexCache.delete(key); // LRU touch → re-insert as the newest
2241
+ dailyIndexCache.set(key, hit);
2242
+ return hit.text;
2243
+ }
2244
+ if (hit) dailyIndexCache.delete(key); // expired
2245
+ const r = await getEdgar(built, DAILYINDEX_LABEL);
2246
+ const text = await r.text();
2247
+ const ttl = isTodayUtc(yyyymmdd) ? DAILYINDEX_TTL_TODAY_MS : DAILYINDEX_TTL_CLOSED_MS;
2248
+ dailyIndexCache.set(key, { text, expiresAt: now + ttl });
2249
+ while (dailyIndexCache.size > DAILYINDEX_CACHE_MAX) {
2250
+ const oldest = dailyIndexCache.keys().next().value as string | undefined;
2251
+ if (oldest === undefined) break;
2252
+ dailyIndexCache.delete(oldest);
2253
+ }
2254
+ return text;
2255
+ }
2256
+
2257
+ /** For tests: evict the daily-index bounded LRU (mirrors `_resetFullIndexCache`). */
2258
+ export function _resetDailyIndexCache(): void {
2259
+ dailyIndexCache.clear();
2260
+ }
2261
+
2262
+ /**
2263
+ * Consult the quarter's index.json existence oracle (getEdgar; ONLY on the 403 path).
2264
+ * Returns whether `master.<yyyymmdd>.idx` is listed AND `maxListed` = the newest
2265
+ * `YYYYMMDD` among items matching `/^master\.(\d{8})\.idx$/` (M1). Throws on a 403 /
2266
+ * unexpected shape — the caller converts that to the ambiguous both-causes
2267
+ * upstream_unavailable (the oracle itself is inconclusive).
2268
+ */
2269
+ async function fetchDailyOracle(
2270
+ url: string,
2271
+ yyyymmdd: string,
2272
+ ): Promise<{ listed: boolean; maxListed: string | null }> {
2273
+ const r = await getEdgar(url, DAILYINDEX_ORACLE_LABEL);
2274
+ const d = (await r.json()) as { directory?: { item?: Array<{ name?: unknown }> } };
2275
+ const items = d?.directory?.item;
2276
+ if (!Array.isArray(items)) {
2277
+ throw new ToolErrorCarrier({
2278
+ kind: "schema_drift",
2279
+ message: `edgar:daily-index index.json returned an unexpected shape (directory.item[] array missing) — the existence oracle is inconclusive.`,
2280
+ retryable: false,
2281
+ upstreamEndpoint: DAILYINDEX_ORACLE_LABEL,
2282
+ });
2283
+ }
2284
+ const wanted = DAILYINDEX_FILE(yyyymmdd);
2285
+ let listed = false;
2286
+ let maxListed: string | null = null;
2287
+ for (const it of items) {
2288
+ const name = typeof it?.name === "string" ? it.name : "";
2289
+ const mm = /^master\.(\d{8})\.idx$/.exec(name);
2290
+ if (!mm) continue;
2291
+ if (name === wanted) listed = true;
2292
+ const day = mm[1] as string;
2293
+ if (maxListed === null || day > maxListed) maxListed = day;
2294
+ }
2295
+ return { listed, maxListed };
2296
+ }
2297
+
2298
+ /** Honest found:false bundle for the daily-index absent/not-yet-disseminated states. */
2299
+ function dailyAbsentBundle(
2300
+ found: false,
2301
+ date: string,
2302
+ year: number,
2303
+ quarter: number,
2304
+ yyyymmdd: string,
2305
+ complete: boolean,
2306
+ note: string,
2307
+ ): MetaBundle {
2308
+ return withMeta(
2309
+ {
2310
+ found,
2311
+ date,
2312
+ year,
2313
+ quarter,
2314
+ indexFile: DAILYINDEX_FILE(yyyymmdd),
2315
+ returned: 0,
2316
+ totalAvailable: 0,
2317
+ filings: [] as FilingIndexRow[],
2318
+ },
2319
+ edgarMeta({
2320
+ returned: 0,
2321
+ totalAvailable: 0,
2322
+ complete,
2323
+ notes: [note, DAILY_SNAPSHOT_NOTE],
2324
+ }),
2325
+ );
2326
+ }
2327
+
2328
+ const DAILY_SNAPSHOT_NOTE =
2329
+ "The daily-index is a point-in-time snapshot of ONE dissemination DAY. For the LATEST or complete-history filings of a KNOWN filer use edgar_company_filings; for a whole quarter use edgar_filing_index; for a text query across 2001-present use edgar_full_text_search.";
2330
+
2331
+ /**
2332
+ * Read the SEC EDGAR daily-index for one calendar `date` and return the filings
2333
+ * matching the given CLIENT-SIDE filters (form / CIK / company substring),
2334
+ * offset-paginated, with the EXACT total match count for the day.
2335
+ *
2336
+ * HONESTY (ADR-0027 v1 + M1 + M2):
2337
+ * - ★M2 — an EXACT date round-trip (Date.UTC component re-extraction) rejects
2338
+ * Feb-30 / day-40 / non-leap-Feb-29 / a malformed / a FUTURE date PRE-fetch
2339
+ * (invalid_input, 0 GET). NOT the `!isNaN(Date.UTC(...))` shortcut (it rolls overflow).
2340
+ * - Fetch the .idx FIRST (happy path pays zero oracle cost). 200 + parseable ⇒
2341
+ * found:true; FULL-SCAN → EXACT totalAvailable (byte-cap forbidden).
2342
+ * - ★M1 — on a 403 (getEdgar mislabels the daily AccessDenied XML rate_limited/403;
2343
+ * caught TOOL-LOCAL), consult index.json; maxListedMasterDate makes it recency-aware:
2344
+ * requestedYyyymmdd > maxListed ⇒ NOT-YET-DISSEMINATED (found:false, complete:FALSE)
2345
+ * requestedYyyymmdd ≤ maxListed & unlisted ⇒ TRUE genuine-absent (found:false, complete:true)
2346
+ * listed but .idx 403'd ⇒ honest rate_limited (retryable ~600s)
2347
+ * oracle itself 403/bad-shape ⇒ ambiguous both-causes upstream_unavailable
2348
+ * - A REAL 429 (status 429, not 403) is NOT caught → stays honest rate_limited.
2349
+ * - CIK stays a STRING; every column via `str`; companyContains is a LITERAL
2350
+ * case-insensitive substring (C110 N/A — no token split).
2351
+ */
2352
+ export async function dailyFilingIndex(args: {
2353
+ date: string;
2354
+ formType?: string;
2355
+ cik?: string | number;
2356
+ companyContains?: string;
2357
+ limit?: number;
2358
+ offset?: number;
2359
+ }): Promise<MetaBundle> {
2360
+ // ★M2 — EXACT date round-trip PRE-fetch (before the URL builder), 0 GET on failure.
2361
+ const dm = /^(\d{4})-(\d{2})-(\d{2})$/.exec(args.date);
2362
+ if (!dm) {
2363
+ dailyIndexInvalid(
2364
+ `edgar_daily_filing_index: date ${JSON.stringify(args.date)} must be an ISO calendar day YYYY-MM-DD — refused before any fetch.`,
2365
+ );
2366
+ }
2367
+ const y = Number(dm[1]);
2368
+ const mo = Number(dm[2]);
2369
+ const d = Number(dm[3]);
2370
+ // Component re-extraction (NOT !isNaN(Date.UTC(...)) — JS silently rolls Feb-30 →
2371
+ // Mar-01, day-40, non-leap Feb-29, so the naive shortcut would FETCH a nonexistent day).
2372
+ const dt = new Date(Date.UTC(y, mo - 1, d));
2373
+ if (
2374
+ dt.getUTCFullYear() !== y ||
2375
+ dt.getUTCMonth() + 1 !== mo ||
2376
+ dt.getUTCDate() !== d
2377
+ ) {
2378
+ dailyIndexInvalid(
2379
+ `edgar_daily_filing_index: date ${JSON.stringify(args.date)} is not a real calendar day (it does not round-trip through Date.UTC — e.g. 2024-02-30, 2024-01-40, or a non-leap 2023-02-29) — refused before any fetch.`,
2380
+ );
2381
+ }
2382
+ const yyyymmdd = `${dm[1]}${dm[2]}${dm[3]}`;
2383
+ const todayYyyymmdd = todayUtcYyyymmdd();
2384
+ if (yyyymmdd > todayYyyymmdd) {
2385
+ // A future day has not happened — you cannot ask for its filings. TODAY is allowed
2386
+ // (reachable; the oracle handles the not-yet-posted case as not-yet-disseminated).
2387
+ dailyIndexInvalid(
2388
+ `edgar_daily_filing_index: date ${args.date} is in the FUTURE (> today ${todayYyyymmdd.slice(0, 4)}-${todayYyyymmdd.slice(4, 6)}-${todayYyyymmdd.slice(6, 8)} UTC) — no filings can exist for a day that has not happened. Refused before any fetch.`,
2389
+ );
2390
+ }
2391
+ const quarter = Math.floor((mo - 1) / 3) + 1;
2392
+
2393
+ // S1 — build + re-validate the path BEFORE any fetch (0 fetch on a bad year/quarter/day).
2394
+ const built = buildDailyIndexUrl(y, quarter, yyyymmdd);
2395
+
2396
+ // Fetch the whole day (getEdgar VERBATIM + `.text()`, bounded-LRU cached). A 403 is
2397
+ // caught TOOL-LOCAL and disambiguated via the index.json oracle (M1) — getEdgar is
2398
+ // UNTOUCHED (the other 6 edgar tools stay byte-identical).
2399
+ let body: string;
2400
+ try {
2401
+ body = await fetchDailyIndexText(built, yyyymmdd);
2402
+ } catch (e) {
2403
+ if (
2404
+ e instanceof ToolErrorCarrier &&
2405
+ e.toolError.kind === "rate_limited" &&
2406
+ e.toolError.upstreamStatus === 403
2407
+ ) {
2408
+ // ── ★M1 403-DISAMBIGUATION via the index.json existence oracle (recency-aware) ──
2409
+ const oracleUrl = buildDailyOracleUrl(y, quarter);
2410
+ let oracle: { listed: boolean; maxListed: string | null };
2411
+ try {
2412
+ oracle = await fetchDailyOracle(oracleUrl, yyyymmdd);
2413
+ } catch {
2414
+ // The oracle ITSELF 403'd / returned an unexpected shape ⇒ INCONCLUSIVE.
2415
+ throw new ToolErrorCarrier({
2416
+ kind: "upstream_unavailable",
2417
+ message: `SEC returned HTTP 403 for the daily-index ${DAILYINDEX_BASE}/${y}/QTR${quarter}/${DAILYINDEX_FILE(yyyymmdd)}, and the index.json existence oracle could not be consulted (it too 403'd / returned an unexpected shape). This is AMBIGUOUS: EITHER ${args.date} has no published daily index OR SEC is rate-limiting this IP at the 10 req/s ceiling (~10-minute block). NOT fabricated as empty — retry after ~10 minutes.`,
2418
+ retryable: true,
2419
+ retryAfterSeconds: 600,
2420
+ upstreamStatus: 403,
2421
+ upstreamEndpoint: DAILYINDEX_LABEL,
2422
+ });
2423
+ }
2424
+ if (oracle.maxListed === null || yyyymmdd > oracle.maxListed) {
2425
+ // ★M1 branch (a) — NEWER than anything the oracle has published yet (covers today
2426
+ // AND unlisted recent trading day(s) across weekends/holidays) ⇒ NOT-YET-DISSEMINATED.
2427
+ // NEVER complete:true — a day full of 8-Ks must not read as a confident empty.
2428
+ return dailyAbsentBundle(
2429
+ false,
2430
+ args.date,
2431
+ y,
2432
+ quarter,
2433
+ yyyymmdd,
2434
+ false, // complete:FALSE
2435
+ `${args.date} is NEWER than the newest daily index EDGAR has published for this quarter (${oracle.maxListed ? `${oracle.maxListed.slice(0, 4)}-${oracle.maxListed.slice(4, 6)}-${oracle.maxListed.slice(6, 8)}` : "none listed yet"}). EDGAR's daily index and its index.json listing LAG real filing activity and may not be posted for the most recent trading day(s), especially across weekends/holidays — this day may simply not be disseminated yet; retry later (EDGAR posts each day's index around 22:00 US-Eastern). This is NOT a confirmed empty day (complete:false).`,
2436
+ );
2437
+ }
2438
+ if (!oracle.listed) {
2439
+ // ★M1 branch (b) — requestedYyyymmdd ≤ maxListed AND still not listed ⇒ a TRUE
2440
+ // gap INSIDE the covered range (weekend / holiday / genuinely no-dissemination day).
2441
+ return dailyAbsentBundle(
2442
+ false,
2443
+ args.date,
2444
+ y,
2445
+ quarter,
2446
+ yyyymmdd,
2447
+ true, // complete:true — an HONEST genuine-absent, NOT an error/drift/fake-empty
2448
+ `master.${yyyymmdd}.idx is NOT published for ${args.date}: it is not listed in the quarter's index.json existence oracle, yet the oracle DOES list newer day(s) (newest ${oracle.maxListed.slice(0, 4)}-${oracle.maxListed.slice(4, 6)}-${oracle.maxListed.slice(6, 8)}) — so this is a genuine gap INSIDE the covered range (a weekend / holiday / no-dissemination day). This is an HONEST genuine-absent answer (found:false, complete:true), NOT an error, NOT a rate-block, NOT a fabricated empty.`,
2449
+ );
2450
+ }
2451
+ // listed === true but the .idx 403'd ⇒ the index EXISTS, SEC rate-blocked this IP.
2452
+ throw new ToolErrorCarrier({
2453
+ kind: "rate_limited",
2454
+ message: `The daily index master.${yyyymmdd}.idx for ${args.date} EXISTS (it IS listed in the quarter's index.json oracle) but the .idx fetch returned HTTP 403 — SEC is rate-limiting this IP at the 10 req/s ceiling (or a transient edge block). Slow down and retry after ~10 minutes. NOT a fake-empty, NOT a genuine-absent day.`,
2455
+ retryable: true,
2456
+ retryAfterSeconds: 600,
2457
+ upstreamStatus: 403,
2458
+ upstreamEndpoint: DAILYINDEX_LABEL,
2459
+ });
2460
+ }
2461
+ throw e; // schema_drift / invalid_input(UA) / upstream_unavailable / real 429 — loud
2462
+ }
2463
+
2464
+ // 200 path — FULL-SCAN → parse ALL rows past the preamble. Drift (header-absent /
2465
+ // all-malformed) THROWS; header+0-rows ⇒ genuine-empty (found:true, totalAvailable:0).
2466
+ const parsed = parseDailyIndex(body);
2467
+ const all = parsed.rows;
2468
+
2469
+ // CLIENT-SIDE filters — ZERO query string (none reach the URL). cik via padCik
2470
+ // both-sides; formType case-insensitive EXACT on col3 ('8-K' ≠ '8-K/A');
2471
+ // companyContains a LITERAL case-insensitive substring (no token split, C110 N/A).
2472
+ const formNeedle = args.formType?.trim() ? args.formType.trim().toLowerCase() : null;
2473
+ const cikFilter =
2474
+ args.cik != null && String(args.cik).trim() !== "" ? padCik(args.cik) : null;
2475
+ const companyNeedle = args.companyContains?.trim()
2476
+ ? args.companyContains.trim().toLowerCase()
2477
+ : null;
2478
+
2479
+ const matches = all.filter((row) => {
2480
+ if (formNeedle !== null) {
2481
+ if (row.formType === null || row.formType.toLowerCase() !== formNeedle) return false;
2482
+ }
2483
+ if (cikFilter !== null) {
2484
+ if (row.cikPadded !== cikFilter) return false;
2485
+ }
2486
+ if (companyNeedle !== null) {
2487
+ if (
2488
+ row.companyName === null ||
2489
+ !row.companyName.toLowerCase().includes(companyNeedle)
2490
+ )
2491
+ return false;
2492
+ }
2493
+ return true;
2494
+ });
2495
+
2496
+ const limit = args.limit ?? 100;
2497
+ const offset = args.offset ?? 0;
2498
+ const totalAvailable = matches.length; // EXACT (full-scan) — never a page length
2499
+ const page = matches.slice(offset, offset + limit);
2500
+ const returned = page.length;
2501
+ const hasMore = offset + returned < totalAvailable;
2502
+ const nextOffset = hasMore ? offset + returned : null;
2503
+
2504
+ const filtersApplied: string[] = [];
2505
+ if (formNeedle !== null) filtersApplied.push("formType");
2506
+ if (cikFilter !== null) filtersApplied.push("cik");
2507
+ if (companyNeedle !== null) filtersApplied.push("companyContains");
2508
+
2509
+ const notes: string[] = [];
2510
+ notes.push(DAILY_SNAPSHOT_NOTE);
2511
+ notes.push(
2512
+ `The whole ${args.date} daily index was downloaded and FULL-SCANNED; totalAvailable (${totalAvailable}) is the EXACT count of filings matching the filters across the ENTIRE day (not a page length, not a byte-capped subset — SEC ignores HTTP Range). This page contains ${returned} of ${totalAvailable}; page via _meta.pagination.nextOffset for the rest.`,
2513
+ );
2514
+ notes.push(
2515
+ `dateFiled is normalized to ISO YYYY-MM-DD from the index's compact YYYYMMDD column; every row in this file shares the requested day (${args.date}).`,
2516
+ );
2517
+ if (isTodayUtc(yyyymmdd)) {
2518
+ notes.push(
2519
+ `${args.date} is TODAY (UTC) — EDGAR posts each day's index around 22:00 US-Eastern and it GROWS as filings disseminate, so totalAvailable is EXACT AS-OF this (short-cached) snapshot, not exact-forever. A closed past day is immutable.`,
2520
+ );
2521
+ }
2522
+ if (companyNeedle !== null) {
2523
+ notes.push(
2524
+ `companyContains is a case-insensitive LITERAL substring match on the Company Name column — a multi-word value matches as ONE contiguous string, NOT AND/OR-tokenized.`,
2525
+ );
2526
+ }
2527
+ if (formNeedle !== null) {
2528
+ notes.push(
2529
+ `formType is a case-insensitive EXACT match on the Form Type column ("${args.formType?.trim()}" matches that form only — e.g. "8-K" does NOT match "8-K/A"). Pass each amendment variant separately.`,
2530
+ );
2531
+ }
2532
+ if (parsed.malformedRows > 0) {
2533
+ notes.push(
2534
+ `${parsed.malformedRows} row(s) did not split into 5 pipe fields and were skipped (tolerated as stray malformed rows; a body with ZERO valid rows would instead be refused as schema drift).`,
2535
+ );
2536
+ }
2537
+ if (parsed.totalIsLowerBound) {
2538
+ notes.push(
2539
+ `The scan hit the MAX_INDEX_ROWS safety ceiling (${MAX_INDEX_ROWS}); totals are a LOWER BOUND — the daily index is larger than expected (possible format drift). See totalIsLowerBound.`,
2540
+ );
2541
+ }
2542
+ if (totalAvailable === 0) {
2543
+ notes.push(
2544
+ filtersApplied.length
2545
+ ? `0 filings on ${args.date} matched the filters (${filtersApplied.join(", ")}). This is an EXACT ZERO over the full day index (found:true — the day IS published) — NOT a truncation, NOT an outage, NOT a genuine-absent day.`
2546
+ : `0 data rows in the ${args.date} daily index (found:true — the day IS published, but its index has no filing rows). This is an EXACT ZERO over the full day index — NOT a truncation, NOT an outage.`,
2547
+ );
2548
+ }
2549
+
2550
+ const meta: Partial<ResponseMeta> = {
2551
+ returned,
2552
+ totalAvailable,
2553
+ filtersApplied,
2554
+ pagination: { offset, limit, hasMore, nextOffset },
2555
+ notes,
2556
+ };
2557
+ if (parsed.totalIsLowerBound) meta.totalIsLowerBound = true;
2558
+
2559
+ return withMeta(
2560
+ {
2561
+ found: true,
2562
+ date: args.date,
2563
+ year: y,
2564
+ quarter,
2565
+ indexFile: DAILYINDEX_FILE(yyyymmdd),
2566
+ returned,
2567
+ totalAvailable,
2568
+ filings: page,
2569
+ },
2570
+ edgarMeta(meta),
2571
+ );
2572
+ }
2573
+
2574
+ // ─── Tool 8: edgar_company_concept (ADR-0041) ─────────────────────
2575
+ // ONE filer × ONE XBRL concept × the COMPLETE reported time-series (with the
2576
+ // amendment/restatement history + multi-unit disclosure) — the focused
2577
+ // financial-TREND / entity-vetting primitive that sits BETWEEN company_facts
2578
+ // (all curated concepts for one filer) and edgar_xbrl_frames (one concept across
2579
+ // ALL filers for one period). Reuses the C105/C106-hardened edgar.ts adapter
2580
+ // VERBATIM (getEdgar transport + self-throttle gate + UA + 403 disambiguation,
2581
+ // padCik + resolveCik, num/str null-never-0, edgarMeta, notFoundBundle, withMeta,
2582
+ // FRAMES_TAG_RE) — getEdgar UNTOUCHED, so the other 7 edgar tools stay byte-identical.
2583
+ //
2584
+ // LIVE-pinned (ADR-0041, 5 keyless GETs, SEC UA, data.sec.gov):
2585
+ // GET https://data.sec.gov/api/xbrl/companyconcept/CIK{cik10}/{taxonomy}/{Concept}.json
2586
+ // → { cik, taxonomy, tag, label, description, entityName, units:{ "USD":[…], "shares":[…] } }
2587
+ // Each unit key maps to its OWN row array; each row { start?, end, val, accn, fy, fp,
2588
+ // form, filed, (frame?) }. `unit` is a BODY key (filtered CLIENT-SIDE), NOT a path
2589
+ // segment. 404 (bad CIK/taxonomy/concept) → XML NoSuchKey → getEdgar throws not_found
2590
+ // BEFORE any JSON parse → notFoundBundle (never a fabricated val:0).
2591
+ //
2592
+ // ★M1 (period identity = the (start,end) PAIR, not `end` alone): every output row
2593
+ // carries `start` (null for INSTANT concepts, the ISO date for DURATION concepts).
2594
+ // The SAME `end` with a DIFFERENT `start` is a different-duration fact (a 3-month
2595
+ // quarter vs the 12-month year — LIVE: Apple NetIncomeLoss end=2009-09-26 carries
2596
+ // BOTH CY2009 (start 2008-09-28) + CY2009Q3 (start 2009-06-28), both frame-tagged
2597
+ // canonical), NOT a revision. A revision is ONLY multiple rows sharing the SAME
2598
+ // (start,end) with a differing accn/filed/val.
2599
+ // ★M2 (canonicalOnly dedup key = (unit,start,end)): partition by unit FIRST, then keep
2600
+ // ONE row per distinct (start,end) — never collapsing a whole unit's row.
2601
+ // ★S1 (unitsAvailable[].count = the RAW units[key].length, pre-filter).
2602
+ // ★S2 (CONCEPT_TAXONOMIES = {us-gaap, dei, ifrs-full} — all three live-confirmed; srt DROPPED).
2603
+
2604
+ /** The companyconcept endpoint base (same data.sec.gov the other edgar tools use). */
2605
+ const CONCEPT_BASE = "https://data.sec.gov/api/xbrl/companyconcept";
2606
+
2607
+ /**
2608
+ * The `taxonomy` path-segment enum — the SSRF guard for that segment (no free value
2609
+ * reaches the host). us-gaap + dei live-confirmed on Apple; ifrs-full live-confirmed on
2610
+ * Spotify (CIK0001639920 / ifrs-full / Assets → 200, unit EUR, frame CY2017Q4I). `srt`
2611
+ * is DROPPED (S2 — it was NOT probed to a resolving 200; a valid-but-unreported tuple is
2612
+ * an honest 404, so a slightly-broad enum can never fabricate — but ship only confirmed).
2613
+ */
2614
+ export const CONCEPT_TAXONOMIES = ["us-gaap", "dei", "ifrs-full"] as const;
2615
+
2616
+ /** A 10-digit CIK path segment (post-padCik; rejects an 11-digit overflow). */
2617
+ const CIK10_RE = /^\d{10}$/;
2618
+
2619
+ /** Throw the pre-fetch injection-guard error (invalid_input, 0 fetch). */
2620
+ function conceptInvalid(message: string): never {
2621
+ throw new ToolErrorCarrier({
2622
+ kind: "invalid_input",
2623
+ message,
2624
+ retryable: false,
2625
+ upstreamEndpoint: "edgar:companyconcept",
2626
+ });
2627
+ }
2628
+
2629
+ /**
2630
+ * Build the companyconcept URL from the THREE validated path segments (COPIES the
2631
+ * buildFramesUrl S1/S2 doctrine — with THREE segments, NOT four; `unit` is a BODY key,
2632
+ * never a path segment). BELT-AND-SUSPENDERS: re-run the cik regex + the taxonomy enum
2633
+ * + the concept regex and hard-throw invalid_input (0 fetch) on any mismatch — do NOT
2634
+ * trust that Zod already ran, and do NOT rely on the hostname assertion alone (it passes
2635
+ * a same-host `../` traversal). THEN assert the built URL is https on the fixed
2636
+ * data.sec.gov host (guards host-escape/downgrade).
2637
+ */
2638
+ function buildConceptUrl(cik10: string, taxonomy: string, tag: string): string {
2639
+ if (!CIK10_RE.test(cik10)) {
2640
+ conceptInvalid(
2641
+ `edgar_company_concept: CIK ${JSON.stringify(cik10)} must be exactly 10 digits after padding (an 11-digit overflow is rejected) — refused before any fetch (path-segment injection guard).`,
2642
+ );
2643
+ }
2644
+ if (!(CONCEPT_TAXONOMIES as readonly string[]).includes(taxonomy)) {
2645
+ conceptInvalid(
2646
+ `edgar_company_concept: taxonomy ${JSON.stringify(taxonomy)} is not one of {${CONCEPT_TAXONOMIES.join(", ")}} — refused before any fetch (path-segment injection guard).`,
2647
+ );
2648
+ }
2649
+ if (!FRAMES_TAG_RE.test(tag)) {
2650
+ conceptInvalid(
2651
+ `edgar_company_concept: concept ${JSON.stringify(tag)} must match ^[A-Za-z0-9]+$ (XBRL tags are alphanumeric CamelCase, e.g. NetIncomeLoss; slash/dot/percent/backslash/'..'/%2F/%2E/%00 are rejected) — refused before any fetch (path-segment injection guard).`,
2652
+ );
2653
+ }
2654
+ const built = `${CONCEPT_BASE}/CIK${cik10}/${taxonomy}/${tag}.json`;
2655
+ let parsed: URL;
2656
+ try {
2657
+ parsed = new URL(built);
2658
+ } catch {
2659
+ conceptInvalid(
2660
+ `edgar_company_concept: could not construct a valid URL from the segments — refused before any fetch.`,
2661
+ );
2662
+ }
2663
+ if (parsed.protocol !== "https:" || parsed.hostname !== "data.sec.gov") {
2664
+ conceptInvalid(
2665
+ `edgar_company_concept: constructed URL host/scheme is not https://data.sec.gov (${parsed.protocol}//${parsed.hostname}) — refused before any fetch (fixed-host assertion).`,
2666
+ );
2667
+ }
2668
+ return built;
2669
+ }
2670
+
2671
+ /** One output row: unit-tagged, (start,end)-keyed, canonical = frame present. */
2672
+ export type ConceptRow = {
2673
+ unit: string;
2674
+ start: string | null; // null for INSTANT concepts; the ISO period-start for DURATION concepts (M1)
2675
+ end: string | null;
2676
+ val: number | null; // num() null-never-0 (a real reported 0 survives)
2677
+ accn: string | null;
2678
+ fy: number | null;
2679
+ fp: string | null;
2680
+ form: string | null;
2681
+ filed: string | null;
2682
+ frame: string | null; // SEC's canonical-consolidation tag, verbatim (null when absent)
2683
+ canonical: boolean; // frame != null — SEC's OWN consolidated-value marker
2684
+ };
2685
+
2686
+ type ConceptRowRaw = {
2687
+ start?: unknown;
2688
+ end?: unknown;
2689
+ val?: unknown;
2690
+ accn?: unknown;
2691
+ fy?: unknown;
2692
+ fp?: unknown;
2693
+ form?: unknown;
2694
+ filed?: unknown;
2695
+ frame?: unknown;
2696
+ };
2697
+ type ConceptBody = {
2698
+ cik?: unknown;
2699
+ taxonomy?: unknown;
2700
+ tag?: unknown;
2701
+ label?: unknown;
2702
+ description?: unknown;
2703
+ entityName?: unknown;
2704
+ units?: unknown;
2705
+ };
2706
+
2707
+ /**
2708
+ * The all-rows default disclosure note (M1-corrected). Rides on EVERY found:true
2709
+ * response. The load-bearing correction vs v1: for flow/duration concepts the same
2710
+ * `end` with a different `start` is a DIFFERENT-DURATION fact, NOT a revision.
2711
+ */
2712
+ const AMENDMENT_DISCLOSURE_NOTE =
2713
+ "Rows are SEC's COMPLETE reported history for this concept, each tagged with its unit. Period identity is the (start,end) PAIR: `start` is null for INSTANT (balance-sheet) concepts and the ISO period-start date for DURATION (flow) concepts. For flow/duration concepts the same `end` with a different `start` is a different-duration fact (a 3-month quarter vs the 12-month year), NOT a revision; a revision is only multiple rows sharing the same (start,end) with different accn/filed/val — read the filed dates. `canonical:true` (a `frame` tag present) marks SEC's consolidated value for that (start,end) period; `canonical:false` rows are earlier/superseded/intra-year reports. Values are surfaced verbatim (val is null-never-0); none is recomputed.";
2714
+
2715
+ /** The (unit,start,end) group key (M1/M2). A space joiner keeps the three fields distinct. */
2716
+ function periodKey(r: ConceptRow): string {
2717
+ return `${r.unit} ${r.start ?? ""} ${r.end ?? ""}`;
2718
+ }
2719
+
2720
+ /**
2721
+ * Deterministic canonicalOnly tiebreak within a (unit,start,end) group: prefer a
2722
+ * frame-tagged row, then max `filed`, then max `accn`. ALWAYS a verbatim selection of
2723
+ * an existing row (never a merge/recompute).
2724
+ */
2725
+ function preferCanonical(a: ConceptRow, b: ConceptRow): ConceptRow {
2726
+ const af = a.frame != null;
2727
+ const bf = b.frame != null;
2728
+ if (af !== bf) return af ? a : b;
2729
+ const afiled = a.filed ?? "";
2730
+ const bfiled = b.filed ?? "";
2731
+ if (afiled !== bfiled) return afiled > bfiled ? a : b;
2732
+ const aaccn = a.accn ?? "";
2733
+ const baccn = b.accn ?? "";
2734
+ return aaccn >= baccn ? a : b;
2735
+ }
2736
+
2737
+ /**
2738
+ * One filer × one XBRL concept × the COMPLETE reported time-series. Reuses resolveCik
2739
+ * (ticker→CIK path EXISTS) + buildConceptUrl (the frames path-segment SSRF doctrine,
2740
+ * THREE segments) + getEdgar VERBATIM. `unit`/`form`/`fy` are CLIENT-SIDE filters;
2741
+ * `canonicalOnly` (default false) dedups to one canonical row per (unit,start,end),
2742
+ * FULLY DISCLOSED. limit/offset window the already-fully-fetched set.
2743
+ *
2744
+ * HONESTY (ADR-0041 v2):
2745
+ * - ★M1 — every row carries `start`; period identity is the (start,end) PAIR. A
2746
+ * same-`end` different-`start` pair is a different-duration fact, NOT a revision.
2747
+ * - ★M2 — canonicalOnly dedup key = (unit,start,end): partition by unit first, keep
2748
+ * one canonical row per distinct (start,end) — never dropping a whole unit's row.
2749
+ * - ★S1 — unitsAvailable[].count = the RAW units[key].length (pre-filter).
2750
+ * - val null-never-0 via num(); every row unit-tagged (no USD↔shares conflation).
2751
+ * - 404 (bad CIK/taxonomy/concept) → notFoundBundle (never a fabricated val:0);
2752
+ * 5xx/timeout/non-JSON/units-shape-drift → THROW; a bad `unit` filter → honest empty
2753
+ * + the available-units note (unit is CLIENT-SIDE, never a path segment).
2754
+ */
2755
+ export async function companyConcept(args: {
2756
+ cikOrTicker: string;
2757
+ concept: string;
2758
+ taxonomy?: string;
2759
+ unit?: string;
2760
+ form?: string;
2761
+ fy?: number;
2762
+ canonicalOnly?: boolean;
2763
+ limit?: number;
2764
+ offset?: number;
2765
+ }): Promise<MetaBundle> {
2766
+ const resolved = await resolveCik(args.cikOrTicker);
2767
+ if (!resolved) {
2768
+ return notFoundBundle(
2769
+ args.cikOrTicker,
2770
+ `Could not resolve "${args.cikOrTicker}" to a CIK (not a numeric CIK and no exact-ticker/title match in company_tickers.json).`,
2771
+ );
2772
+ }
2773
+ const cik = resolved.cik;
2774
+ const taxonomy = args.taxonomy ?? "us-gaap";
2775
+ const concept = args.concept;
2776
+
2777
+ // S1/S2 — build (and re-validate) the path BEFORE any fetch. Throws invalid_input
2778
+ // with 0 fetches on any bad segment or a non-fixed-host URL.
2779
+ const url = buildConceptUrl(cik, taxonomy, concept);
2780
+
2781
+ let body: ConceptBody;
2782
+ try {
2783
+ const r = await getEdgar(url, "edgar:companyconcept");
2784
+ // A 200 body that is NOT valid JSON (an HTML/XML outage slipping through with 200)
2785
+ // → SyntaxError → schema_drift THROW (ADR-0003 doctrine), never a fake empty.
2786
+ try {
2787
+ body = (await r.json()) as ConceptBody;
2788
+ } catch {
2789
+ throw new ToolErrorCarrier({
2790
+ kind: "schema_drift",
2791
+ message: `edgar:companyconcept returned an HTTP 200 body that is not valid JSON for CIK ${cik} / ${taxonomy} / ${concept} — an outage/error page masquerading as a 200. Refusing rather than fabricating an empty result.`,
2792
+ retryable: false,
2793
+ upstreamEndpoint: "edgar:companyconcept",
2794
+ });
2795
+ }
2796
+ } catch (e) {
2797
+ // 404 (bad CIK / bad taxonomy / a concept the filer never reported) ⇒ getEdgar throws
2798
+ // not_found BEFORE any .json() (the XML NoSuchKey body is never parsed) ⇒ notFoundBundle.
2799
+ if (e instanceof ToolErrorCarrier && e.toolError.kind === "not_found") {
2800
+ return notFoundBundle(
2801
+ `${cik}/${taxonomy}/${concept}`,
2802
+ `No XBRL companyconcept matched CIK ${cik} / ${taxonomy} / ${concept} (HTTP 404). The filer did NOT report this concept under this exact taxonomy — this is NOT a value of 0. Check the tag spelling/case (XBRL tags are CamelCase, e.g. NetIncomeLoss) and the taxonomy (us-gaap vs dei vs ifrs-full).`,
2803
+ );
2804
+ }
2805
+ throw e;
2806
+ }
2807
+
2808
+ // Drift guard: `units` MISSING / not an object / an array ⇒ schema_drift THROW (the
2809
+ // envelope changed) — never a fabricated empty.
2810
+ const units = body.units;
2811
+ if (units === null || typeof units !== "object" || Array.isArray(units)) {
2812
+ throw new ToolErrorCarrier({
2813
+ kind: "schema_drift",
2814
+ message: `edgar:companyconcept returned HTTP 200 without a units{} object for CIK ${cik} / ${taxonomy} / ${concept} — the companyconcept envelope changed.`,
2815
+ retryable: false,
2816
+ upstreamEndpoint: "edgar:companyconcept",
2817
+ });
2818
+ }
2819
+ const unitsObj = units as Record<string, unknown>;
2820
+ const unitKeys = Object.keys(unitsObj);
2821
+
2822
+ // ★S1 — unitsAvailable[].count = the RAW per-unit array length, computed BEFORE any
2823
+ // unit/form/fy/canonicalOnly/limit/offset filtering (a filtered count is a masquerade).
2824
+ const unitsAvailable = unitKeys.map((k) => ({
2825
+ unit: k,
2826
+ count: Array.isArray(unitsObj[k]) ? (unitsObj[k] as unknown[]).length : 0,
2827
+ }));
2828
+
2829
+ // Build ALL rows, TAGGED with unit (P3 — never conflate USD with shares). Iterate the
2830
+ // unit keys in a DETERMINISTIC order (ascending) for a stable window; each unit's rows
2831
+ // stay in SEC's returned order (no blind re-sort — the edgar_company_filings doctrine).
2832
+ const sortedUnitKeys = [...unitKeys].sort();
2833
+ const allRows: ConceptRow[] = [];
2834
+ for (const uKey of sortedUnitKeys) {
2835
+ const arr = unitsObj[uKey];
2836
+ if (!Array.isArray(arr)) continue; // defensive — a non-array unit contributes no rows
2837
+ for (const raw of arr as ConceptRowRaw[]) {
2838
+ const frame = str(raw.frame);
2839
+ allRows.push({
2840
+ unit: uKey,
2841
+ start: str(raw.start), // null for INSTANT concepts; ISO date for DURATION (M1)
2842
+ end: str(raw.end),
2843
+ val: num(raw.val), // null-never-0
2844
+ accn: str(raw.accn),
2845
+ fy: typeof raw.fy === "number" ? raw.fy : null,
2846
+ fp: str(raw.fp),
2847
+ form: str(raw.form),
2848
+ filed: str(raw.filed),
2849
+ frame,
2850
+ canonical: frame != null,
2851
+ });
2852
+ }
2853
+ }
2854
+
2855
+ // Defensive empty (Q5): a 200 whose units{} has ZERO keys or ALL-empty arrays ⇒ an
2856
+ // honest empty (found:false, complete:true), NOT a crash and NOT a fabricated row.
2857
+ if (allRows.length === 0) {
2858
+ return withMeta(
2859
+ {
2860
+ found: false,
2861
+ cik,
2862
+ entityName: str(body.entityName),
2863
+ taxonomy,
2864
+ concept,
2865
+ label: str(body.label),
2866
+ description: str(body.description),
2867
+ unitsAvailable,
2868
+ rows: [] as ConceptRow[],
2869
+ },
2870
+ edgarMeta({
2871
+ returned: 0,
2872
+ totalAvailable: 0,
2873
+ complete: true,
2874
+ filtersApplied: ["concept", "taxonomy"],
2875
+ notes: [
2876
+ `The companyconcept document for CIK ${cik} / ${taxonomy} / ${concept} has no reported data points (units{} is empty). This is an honest empty — NOT a value of 0 and NOT an outage.`,
2877
+ ],
2878
+ }),
2879
+ );
2880
+ }
2881
+
2882
+ // ── CLIENT-SIDE filters (unit/form/fy) — none reaches the URL. ──
2883
+ const unitFilter = args.unit?.trim() ? args.unit.trim() : null;
2884
+ const formFilter = args.form?.trim() ? args.form.trim().toLowerCase() : null;
2885
+ const fyFilter = typeof args.fy === "number" ? args.fy : null;
2886
+ const canonicalOnly = args.canonicalOnly ?? false;
2887
+
2888
+ let filtered = allRows;
2889
+ if (unitFilter !== null) filtered = filtered.filter((r) => r.unit === unitFilter);
2890
+ if (formFilter !== null)
2891
+ filtered = filtered.filter((r) => r.form != null && r.form.toLowerCase() === formFilter);
2892
+ if (fyFilter !== null) filtered = filtered.filter((r) => r.fy === fyFilter);
2893
+
2894
+ // Group the FILTERED rows by (unit,start,end) ONCE — used for BOTH revision detection
2895
+ // (over the pre-dedup set) and the canonicalOnly dedup (M1/M2).
2896
+ const groups = new Map<string, ConceptRow[]>();
2897
+ const groupOrder: string[] = [];
2898
+ for (const r of filtered) {
2899
+ const key = periodKey(r);
2900
+ let g = groups.get(key);
2901
+ if (!g) {
2902
+ g = [];
2903
+ groups.set(key, g);
2904
+ groupOrder.push(key);
2905
+ }
2906
+ g.push(r);
2907
+ }
2908
+
2909
+ // ★M1 revision detection: a GENUINE restatement is ≥2 rows sharing the SAME
2910
+ // (unit,start,end) with ≥2 DISTINCT non-null vals. A same-`end` different-`start`
2911
+ // pair lands in DIFFERENT groups (singletons) ⇒ NOT flagged as a revision.
2912
+ const revisedPeriods = groupOrder
2913
+ .map((k) => groups.get(k) as ConceptRow[])
2914
+ .filter((g) => {
2915
+ if (g.length < 2) return false;
2916
+ const vals = new Set(g.filter((r) => r.val !== null).map((r) => r.val));
2917
+ return vals.size >= 2;
2918
+ })
2919
+ .map((g) => {
2920
+ const first = g[0] as ConceptRow;
2921
+ return { unit: first.unit, start: first.start, end: first.end };
2922
+ });
2923
+
2924
+ // ★M2 canonicalOnly — partition by unit FIRST (already reflected in the (unit,start,end)
2925
+ // key), keep ONE row per distinct (start,end): frame-tagged, else the latest-filed
2926
+ // fallback (marked canonical:false). Deterministic tiebreak via preferCanonical.
2927
+ if (canonicalOnly) {
2928
+ filtered = groupOrder.map((k) => (groups.get(k) as ConceptRow[]).reduce(preferCanonical));
2929
+ }
2930
+
2931
+ // ── Pagination — client-side window over the (unit,start,end)-keyed filtered set. ──
2932
+ const totalAvailable = filtered.length;
2933
+ const limit = args.limit ?? 100;
2934
+ const offset = args.offset ?? 0;
2935
+ const page = filtered.slice(offset, offset + limit);
2936
+ const returned = page.length;
2937
+ const hasMore = offset + returned < totalAvailable;
2938
+ const nextOffset = hasMore ? offset + returned : null;
2939
+
2940
+ const filtersApplied: string[] = ["concept", "taxonomy"];
2941
+ if (unitFilter !== null) filtersApplied.push("unit");
2942
+ if (formFilter !== null) filtersApplied.push("form");
2943
+ if (fyFilter !== null) filtersApplied.push("fy");
2944
+ if (canonicalOnly) filtersApplied.push("canonicalOnly");
2945
+
2946
+ const notes: string[] = [AMENDMENT_DISCLOSURE_NOTE];
2947
+
2948
+ // Unit-filter disclosure (Q2-c) — NEVER hide the other units.
2949
+ if (unitFilter !== null) {
2950
+ if (unitKeys.includes(unitFilter)) {
2951
+ const others = unitKeys.filter((k) => k !== unitFilter);
2952
+ notes.push(
2953
+ others.length
2954
+ ? `unit filter '${unitFilter}' applied; this concept is ALSO reported in unit(s) ${others.join(", ")} for this filer (not shown). Re-request without the unit filter, or with a different unit, to see them.`
2955
+ : `unit filter '${unitFilter}' applied; it is the only unit this filer reports for this concept.`,
2956
+ );
2957
+ } else {
2958
+ notes.push(
2959
+ `unit filter '${unitFilter}' matched NONE of this filer's reported unit(s) for this concept (available: ${unitKeys.join(", ") || "none"}). Returning 0 rows for that filter — this is NOT a value of 0; re-request with one of the available units, or omit unit.`,
2960
+ );
2961
+ }
2962
+ }
2963
+
2964
+ // ★M1 revision note — a distinctive phrase so a caller (and the fault fixtures) can
2965
+ // tell a GENUINE restatement from ordinary duration-multiplicity.
2966
+ if (revisedPeriods.length) {
2967
+ const ex = revisedPeriods
2968
+ .slice(0, 3)
2969
+ .map(
2970
+ (p) =>
2971
+ `(unit ${p.unit}, ${p.start != null ? `start ${p.start}, ` : ""}end ${p.end})`,
2972
+ )
2973
+ .join("; ");
2974
+ notes.push(
2975
+ `Restatement/revision detected: ${revisedPeriods.length} (unit,start,end) period(s) carry MULTIPLE rows with a DIFFERING val — the figure WAS revised for ${ex}${revisedPeriods.length > 3 ? " (and more)" : ""}. Read the filed dates; the frame-tagged (canonical) row carries SEC's consolidated value.`,
2976
+ );
2977
+ }
2978
+
2979
+ // canonicalOnly disclosure (Q3) — the dedup is FULLY disclosed, never silent.
2980
+ if (canonicalOnly) {
2981
+ notes.push(
2982
+ "canonicalOnly=true: reduced to ONE row per distinct (unit,start,end) period — the frame-tagged canonical value, or (for a period SEC has not yet consolidated) the latest-filed row, marked canonical:false. SUPERSEDED/amendment rows sharing the same (start,end) were REMOVED; a value may have been revised — re-request with canonicalOnly=false to see the full amendment history. NOTE: the same `end` with a different `start` is a DIFFERENT period (kept separately), not a duplicate.",
2983
+ );
2984
+ }
2985
+
2986
+ // Raw-vs-filtered total disclosure (Q4) — fetch-once, exact filtered total.
2987
+ notes.push(
2988
+ `served ${allRows.length} row(s) across unit(s) ${unitKeys.join(", ")}; after filters (${filtersApplied.join(", ")}) → ${totalAvailable} pageable; totalAvailable reflects the FILTERED (unit,start,end)-keyed set. The full per-unit series was fetched in ONE request (SEC does not paginate companyconcept); limit/offset is a client-side window.`,
2989
+ );
2990
+
2991
+ if (resolved.title)
2992
+ notes.push(`Resolved "${args.cikOrTicker}" → ${resolved.title} (CIK ${cik}).`);
2993
+
2994
+ return withMeta(
2995
+ {
2996
+ found: true,
2997
+ cik,
2998
+ entityName: str(body.entityName),
2999
+ taxonomy,
3000
+ concept,
3001
+ label: str(body.label),
3002
+ description: str(body.description),
3003
+ unitsAvailable,
3004
+ rows: page,
3005
+ },
3006
+ edgarMeta({
3007
+ returned,
3008
+ totalAvailable,
3009
+ filtersApplied,
3010
+ pagination: { offset, limit, hasMore, nextOffset },
3011
+ notes,
3012
+ }),
3013
+ );
3014
+ }