@cliwant/mcp-sam-gov 0.3.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.ja.md +48 -16
- package/README.ko.md +48 -16
- package/README.md +226 -64
- package/dist/attachments.d.ts +63 -0
- package/dist/attachments.d.ts.map +1 -0
- package/dist/attachments.js +576 -0
- package/dist/attachments.js.map +1 -0
- package/dist/bls.d.ts +267 -0
- package/dist/bls.d.ts.map +1 -0
- package/dist/bls.js +1549 -0
- package/dist/bls.js.map +1 -0
- package/dist/cache.d.ts +2 -0
- package/dist/cache.d.ts.map +1 -1
- package/dist/cache.js +4 -0
- package/dist/cache.js.map +1 -1
- package/dist/census.d.ts +182 -0
- package/dist/census.d.ts.map +1 -0
- package/dist/census.js +576 -0
- package/dist/census.js.map +1 -0
- package/dist/ckan.d.ts +141 -0
- package/dist/ckan.d.ts.map +1 -0
- package/dist/ckan.js +393 -0
- package/dist/ckan.js.map +1 -0
- package/dist/clinicaltrials.d.ts +180 -0
- package/dist/clinicaltrials.d.ts.map +1 -0
- package/dist/clinicaltrials.js +730 -0
- package/dist/clinicaltrials.js.map +1 -0
- package/dist/cms.d.ts +140 -0
- package/dist/cms.d.ts.map +1 -0
- package/dist/cms.js +482 -0
- package/dist/cms.js.map +1 -0
- package/dist/coerce.d.ts +32 -0
- package/dist/coerce.d.ts.map +1 -0
- package/dist/coerce.js +50 -0
- package/dist/coerce.js.map +1 -0
- package/dist/datagov-catalog.d.ts +84 -0
- package/dist/datagov-catalog.d.ts.map +1 -0
- package/dist/datagov-catalog.js +233 -0
- package/dist/datagov-catalog.js.map +1 -0
- package/dist/datagov.d.ts +146 -0
- package/dist/datagov.d.ts.map +1 -0
- package/dist/datagov.js +689 -0
- package/dist/datagov.js.map +1 -0
- package/dist/datagovKey.d.ts +36 -0
- package/dist/datagovKey.d.ts.map +1 -0
- package/dist/datagovKey.js +60 -0
- package/dist/datagovKey.js.map +1 -0
- package/dist/datasource.d.ts +313 -0
- package/dist/datasource.d.ts.map +1 -0
- package/dist/datasource.js +551 -0
- package/dist/datasource.js.map +1 -0
- package/dist/disclosure.d.ts +55 -0
- package/dist/disclosure.d.ts.map +1 -0
- package/dist/disclosure.js +57 -0
- package/dist/disclosure.js.map +1 -0
- package/dist/ecfr.d.ts +4 -2
- package/dist/ecfr.d.ts.map +1 -1
- package/dist/ecfr.js +74 -1
- package/dist/ecfr.js.map +1 -1
- package/dist/echo.d.ts +143 -0
- package/dist/echo.d.ts.map +1 -0
- package/dist/echo.js +424 -0
- package/dist/echo.js.map +1 -0
- package/dist/edgar.d.ts +377 -0
- package/dist/edgar.d.ts.map +1 -0
- package/dist/edgar.js +2418 -0
- package/dist/edgar.js.map +1 -0
- package/dist/errors.d.ts +26 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +69 -1
- package/dist/errors.js.map +1 -1
- package/dist/fac.d.ts +180 -0
- package/dist/fac.d.ts.map +1 -0
- package/dist/fac.js +416 -0
- package/dist/fac.js.map +1 -0
- package/dist/far.d.ts +170 -0
- package/dist/far.d.ts.map +1 -0
- package/dist/far.js +804 -0
- package/dist/far.js.map +1 -0
- package/dist/fdic.d.ts +599 -0
- package/dist/fdic.d.ts.map +1 -0
- package/dist/fdic.js +1624 -0
- package/dist/fdic.js.map +1 -0
- package/dist/federal-register.d.ts +139 -2
- package/dist/federal-register.d.ts.map +1 -1
- package/dist/federal-register.js +419 -3
- package/dist/federal-register.js.map +1 -1
- package/dist/fema.d.ts +181 -0
- package/dist/fema.d.ts.map +1 -0
- package/dist/fema.js +436 -0
- package/dist/fema.js.map +1 -0
- package/dist/fpds.d.ts +108 -0
- package/dist/fpds.d.ts.map +1 -0
- package/dist/fpds.js +519 -0
- package/dist/fpds.js.map +1 -0
- package/dist/gao.d.ts +64 -0
- package/dist/gao.d.ts.map +1 -0
- package/dist/gao.js +640 -0
- package/dist/gao.js.map +1 -0
- package/dist/govinfo.d.ts +111 -0
- package/dist/govinfo.d.ts.map +1 -0
- package/dist/govinfo.js +422 -0
- package/dist/govinfo.js.map +1 -0
- package/dist/grants.d.ts +27 -4
- package/dist/grants.d.ts.map +1 -1
- package/dist/grants.js +111 -6
- package/dist/grants.js.map +1 -1
- package/dist/gsa-csv.d.ts +249 -0
- package/dist/gsa-csv.d.ts.map +1 -0
- package/dist/gsa-csv.js +784 -0
- package/dist/gsa-csv.js.map +1 -0
- package/dist/integrity.d.ts +212 -0
- package/dist/integrity.d.ts.map +1 -0
- package/dist/integrity.js +707 -0
- package/dist/integrity.js.map +1 -0
- package/dist/meta.d.ts +165 -0
- package/dist/meta.d.ts.map +1 -0
- package/dist/meta.js +162 -0
- package/dist/meta.js.map +1 -0
- package/dist/nih.d.ts +117 -0
- package/dist/nih.d.ts.map +1 -0
- package/dist/nih.js +291 -0
- package/dist/nih.js.map +1 -0
- package/dist/nppes.d.ts +157 -0
- package/dist/nppes.d.ts.map +1 -0
- package/dist/nppes.js +648 -0
- package/dist/nppes.js.map +1 -0
- package/dist/nsf.d.ts +176 -0
- package/dist/nsf.d.ts.map +1 -0
- package/dist/nsf.js +554 -0
- package/dist/nsf.js.map +1 -0
- package/dist/nvd.d.ts +176 -0
- package/dist/nvd.d.ts.map +1 -0
- package/dist/nvd.js +912 -0
- package/dist/nvd.js.map +1 -0
- package/dist/ofac.d.ts +205 -0
- package/dist/ofac.d.ts.map +1 -0
- package/dist/ofac.js +919 -0
- package/dist/ofac.js.map +1 -0
- package/dist/pricing.d.ts +110 -0
- package/dist/pricing.d.ts.map +1 -0
- package/dist/pricing.js +843 -0
- package/dist/pricing.js.map +1 -0
- package/dist/sam-gov/client.d.ts +60 -2
- package/dist/sam-gov/client.d.ts.map +1 -1
- package/dist/sam-gov/client.js +320 -54
- package/dist/sam-gov/client.js.map +1 -1
- package/dist/sam-gov/index.d.ts +1 -1
- package/dist/sam-gov/index.d.ts.map +1 -1
- package/dist/sam-gov/index.js +1 -1
- package/dist/sam-gov/index.js.map +1 -1
- package/dist/sam-gov/types.d.ts +24 -0
- package/dist/sam-gov/types.d.ts.map +1 -1
- package/dist/sba.d.ts +72 -0
- package/dist/sba.d.ts.map +1 -0
- package/dist/sba.js +281 -0
- package/dist/sba.js.map +1 -0
- package/dist/server.d.ts +14 -2
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +3889 -296
- package/dist/server.js.map +1 -1
- package/dist/snapshot.d.ts +98 -0
- package/dist/snapshot.d.ts.map +1 -0
- package/dist/snapshot.js +146 -0
- package/dist/snapshot.js.map +1 -0
- package/dist/socrata.d.ts +157 -0
- package/dist/socrata.d.ts.map +1 -0
- package/dist/socrata.js +448 -0
- package/dist/socrata.js.map +1 -0
- package/dist/treasury.d.ts +143 -0
- package/dist/treasury.d.ts.map +1 -0
- package/dist/treasury.js +436 -0
- package/dist/treasury.js.map +1 -0
- package/dist/usaspending.d.ts +260 -65
- package/dist/usaspending.d.ts.map +1 -1
- package/dist/usaspending.js +1629 -201
- package/dist/usaspending.js.map +1 -1
- package/dist/usitc.d.ts +142 -0
- package/dist/usitc.d.ts.map +1 -0
- package/dist/usitc.js +339 -0
- package/dist/usitc.js.map +1 -0
- package/package.json +24 -2
- package/src/attachments.ts +652 -0
- package/src/bls.ts +1943 -0
- package/src/cache.ts +5 -0
- package/src/census.ts +735 -0
- package/src/ckan.ts +495 -0
- package/src/clinicaltrials.ts +923 -0
- package/src/cms.ts +634 -0
- package/src/coerce.ts +47 -0
- package/src/datagov-catalog.ts +296 -0
- package/src/datagov.ts +907 -0
- package/src/datagovKey.ts +68 -0
- package/src/datasource.ts +721 -0
- package/src/disclosure.ts +61 -0
- package/src/ecfr.ts +98 -1
- package/src/echo.ts +496 -0
- package/src/edgar.ts +3014 -0
- package/src/errors.ts +84 -1
- package/src/fac.ts +529 -0
- package/src/far.ts +1007 -0
- package/src/fdic.ts +2052 -0
- package/src/federal-register.ts +512 -3
- package/src/fema.ts +541 -0
- package/src/fpds.ts +620 -0
- package/src/gao.ts +744 -0
- package/src/govinfo.ts +497 -0
- package/src/grants.ts +139 -7
- package/src/gsa-csv.ts +992 -0
- package/src/integrity.ts +928 -0
- package/src/meta.ts +292 -0
- package/src/nih.ts +375 -0
- package/src/nppes.ts +834 -0
- package/src/nsf.ts +706 -0
- package/src/nvd.ts +1124 -0
- package/src/ofac.ts +1166 -0
- package/src/pricing.ts +1075 -0
- package/src/sam-gov/client.ts +345 -63
- package/src/sam-gov/index.ts +5 -1
- package/src/sam-gov/types.ts +22 -0
- package/src/sba.ts +357 -0
- package/src/server.ts +4551 -328
- package/src/snapshot.ts +192 -0
- package/src/socrata.ts +532 -0
- package/src/treasury.ts +575 -0
- package/src/usaspending.ts +1946 -214
- package/src/usitc.ts +420 -0
package/src/edgar.ts
ADDED
|
@@ -0,0 +1,3014 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SEC EDGAR — company filings, XBRL financial facts, ticker→CIK, and full-text
|
|
3
|
+
* search (keyless). First capital-markets source (ADR-0003); 2nd consumer of the
|
|
4
|
+
* fetch/map/meta shape after treasury.ts.
|
|
5
|
+
*
|
|
6
|
+
* Fully PUBLIC, KEYLESS. Hosts: data.sec.gov, efts.sec.gov, www.sec.gov/files
|
|
7
|
+
* (+ www.sec.gov/Archives for constructed links). No API key; instead SEC
|
|
8
|
+
* requires a descriptive `User-Agent` on EVERY request and self-throttling to
|
|
9
|
+
* ≤10 req/s (a breach → ~10-minute IP block).
|
|
10
|
+
*
|
|
11
|
+
* Three layers (mirror treasury.ts):
|
|
12
|
+
* fetch — `getEdgar(url, label)`: sets the mandatory UA (+ gzip) on the init,
|
|
13
|
+
* serializes every EDGAR fetch through a per-process min-interval gate
|
|
14
|
+
* (~110ms → ≤~9 req/s), and reuses errors.ts retry/timeout/taxonomy.
|
|
15
|
+
* map — PURE columnar/curated mappers (zipRecent, companyfacts extraction,
|
|
16
|
+
* FTS hit map). `num(x)` → number|null (NEVER 0 for absent). `padCik`.
|
|
17
|
+
* meta — `withMeta(...)`: hands totalAvailable/returned/pagination/notes to
|
|
18
|
+
* meta.ts's buildMeta, which DERIVES complete/truncated.
|
|
19
|
+
*
|
|
20
|
+
* HONESTY / REVIEW FIXES (ADR-0003 "Review outcome (v2)", live-verified 2026-07-10):
|
|
21
|
+
* F1 — the committed default UA is an ORG contact on the project domain
|
|
22
|
+
* (`cliwant-mcp-sam-gov/1.0 (contact: opendata@cliwant.com)`), NOT a
|
|
23
|
+
* github.com URL (SEC 403s any UA containing `github.com`) and NOT a
|
|
24
|
+
* personal email. Override via EDGAR_USER_AGENT.
|
|
25
|
+
* F2 — `edgar_full_text_search` has NO `size` param: efts ignores it (5/20/100/
|
|
26
|
+
* 200 all return 100), so page size is a fixed 100. Pagination is by `from`.
|
|
27
|
+
* F3 — FTS window overflow: `from > 9900` (from+100 > 10000) is rejected as
|
|
28
|
+
* invalid_input BEFORE the fetch; and after r.json(), a missing `hits.hits`
|
|
29
|
+
* (SEC returns HTTP 200 + `{message:"Internal server error"}` past the
|
|
30
|
+
* window) is thrown as schema_drift, never crashed on.
|
|
31
|
+
* F4 — default curated concepts DO NOT include EarningsPerShareBasic (its unit
|
|
32
|
+
* is `USD/shares`, so the default `unit="USD"` would silently return
|
|
33
|
+
* nothing). A requested concept present only in another unit → a note,
|
|
34
|
+
* never a silent/fabricated 0.
|
|
35
|
+
* F5 — FTS `hits.total.relation === "gte"` → `totalIsLowerBound:true` passed to
|
|
36
|
+
* withMeta (machine-readable, not only a note). See meta.ts.
|
|
37
|
+
* F6 — getEdgar disambiguates a 403 by reading the body: "automated"/
|
|
38
|
+
* "Undeclared" ⇒ invalid_input (bad UA, don't retry); else ⇒ rate_limited,
|
|
39
|
+
* retryable, retryAfterSeconds 600 (the ~10-min block).
|
|
40
|
+
* F7 — FTS `_source` has NO primary-document filename; the output `filingIndexUrl`
|
|
41
|
+
* is the filing's ARCHIVE INDEX directory built from `adsh` (no fabricated
|
|
42
|
+
* doc URL). `edgar_company_filings` DOES use the real `primaryDocument`.
|
|
43
|
+
*
|
|
44
|
+
* CIK↔UEI JOIN CAVEAT (load-bearing, in every tool's `_meta.notes` +
|
|
45
|
+
* fieldsUnavailable): EDGAR keys on 10-digit SEC CIK, NOT SAM UEI/DUNS or a
|
|
46
|
+
* USAspending recipient id. No reliable programmatic CIK↔UEI join exists;
|
|
47
|
+
* bridging an EDGAR filer to a federal contracting entity is name/ticker-only
|
|
48
|
+
* (fuzzy) and MUST NOT be asserted as authoritative.
|
|
49
|
+
*/
|
|
50
|
+
|
|
51
|
+
import { fetchWithRetry, ToolErrorCarrier, errorFromResponse } from "./errors.js";
|
|
52
|
+
import { throughGate, driftError } from "./datasource.js";
|
|
53
|
+
import { memoize } from "./cache.js";
|
|
54
|
+
import { withMeta, type MetaBundle, type ResponseMeta } from "./meta.js";
|
|
55
|
+
|
|
56
|
+
// ─── Hosts + the mandatory User-Agent (F1) ────────────────────────
|
|
57
|
+
const SUBMISSIONS_BASE = "https://data.sec.gov/submissions";
|
|
58
|
+
const XBRL_FACTS_BASE = "https://data.sec.gov/api/xbrl/companyfacts";
|
|
59
|
+
const TICKERS_URL = "https://www.sec.gov/files/company_tickers.json";
|
|
60
|
+
const FTS_URL = "https://efts.sec.gov/LATEST/search-index";
|
|
61
|
+
const ARCHIVES_BASE = "https://www.sec.gov/Archives/edgar/data";
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* The descriptive User-Agent SEC mandates on every request (Policy①). Default is
|
|
65
|
+
* an ORG contact on the project domain — live-verified 200 (F1). It must NOT
|
|
66
|
+
* contain `github.com` (SEC 403s that on all 3 hosts) and must NOT be a personal
|
|
67
|
+
* email. Operators override with the `EDGAR_USER_AGENT` env var (SEC's suggested
|
|
68
|
+
* form: "Sample Company AdminContact@example.com").
|
|
69
|
+
*/
|
|
70
|
+
const EDGAR_UA =
|
|
71
|
+
process.env.EDGAR_USER_AGENT ??
|
|
72
|
+
"cliwant-mcp-sam-gov/1.0 (contact: opendata@cliwant.com)";
|
|
73
|
+
|
|
74
|
+
const EDGAR_HEADERS = {
|
|
75
|
+
"User-Agent": EDGAR_UA,
|
|
76
|
+
"Accept-Encoding": "gzip",
|
|
77
|
+
} as const;
|
|
78
|
+
|
|
79
|
+
const EDGAR_SOURCE = "data.sec.gov / efts.sec.gov (keyless)";
|
|
80
|
+
const EDGAR_FIELDS_UNAVAILABLE = ["uei", "duns", "sam_recipient_id"];
|
|
81
|
+
const CIK_UEI_CAVEAT =
|
|
82
|
+
"EDGAR identifies filers by 10-digit SEC CIK, NOT SAM UEI/DUNS or a USAspending recipient id. There is no reliable programmatic CIK↔UEI join — bridging an EDGAR filer to a federal contracting entity is name/ticker-only (fuzzy) and must NOT be treated as authoritative.";
|
|
83
|
+
|
|
84
|
+
// ─── Min-interval gate (self-throttle ≤10 req/s) ──────────────────
|
|
85
|
+
// A tool call can fan out (lookup→submissions), and the process may run many
|
|
86
|
+
// tools; serialize EVERY EDGAR fetch through one promise chain with a ≥110ms
|
|
87
|
+
// spacing (~9 req/s < the 10 req/s ceiling → no ~10-min block). The chain +
|
|
88
|
+
// spacing math now live in the shared `throughGate("edgar", 110, fn)` primitive
|
|
89
|
+
// (datasource.ts, ADR-0011 R2 deferred slice) — same single-chain, same
|
|
90
|
+
// lastAt-stamped-before-fn, same bare-`setTimeout` global (so the fault tests'
|
|
91
|
+
// timer-neutralizing patch keeps it instant offline). Behavior is byte-identical
|
|
92
|
+
// to the former module singleton this replaced.
|
|
93
|
+
const EDGAR_MIN_INTERVAL_MS = 110;
|
|
94
|
+
|
|
95
|
+
// ─── fetch layer ──────────────────────────────────────────────────
|
|
96
|
+
/**
|
|
97
|
+
* GET one EDGAR resource with the mandatory UA (+ gzip) and a 15s timeout,
|
|
98
|
+
* serialized through the min-interval gate. Reuses errors.ts for the retry/
|
|
99
|
+
* backoff on transient 429/5xx/network faults.
|
|
100
|
+
*
|
|
101
|
+
* F6 — a 403 from SEC is ambiguous (a missing/undeclared UA vs a rate-block).
|
|
102
|
+
* Read the body to disambiguate BEFORE the generic errorFromResponse mislabels
|
|
103
|
+
* it as a plain invalid_input:
|
|
104
|
+
* body ~ /automated|undeclared/i ⇒ invalid_input, non-retryable (fix the UA)
|
|
105
|
+
* otherwise ⇒ rate_limited, retryable, retryAfter 600s
|
|
106
|
+
* Common non-retryable statuses (404/400) throw immediately; 404 in particular
|
|
107
|
+
* is caught by the caller and turned into an honest `found:false`.
|
|
108
|
+
*/
|
|
109
|
+
async function getEdgar(url: string, label: string): Promise<Response> {
|
|
110
|
+
const init: RequestInit = {
|
|
111
|
+
headers: { ...EDGAR_HEADERS },
|
|
112
|
+
signal: AbortSignal.timeout(15_000),
|
|
113
|
+
};
|
|
114
|
+
return throughGate("edgar", EDGAR_MIN_INTERVAL_MS, async () => {
|
|
115
|
+
let r: Response;
|
|
116
|
+
try {
|
|
117
|
+
r = await fetch(url, init);
|
|
118
|
+
} catch {
|
|
119
|
+
// Network-level fault (DNS/reset/timeout). Reuse the shared retry/backoff
|
|
120
|
+
// with a fresh AbortSignal per attempt.
|
|
121
|
+
return fetchWithRetry(
|
|
122
|
+
url,
|
|
123
|
+
{ headers: { ...EDGAR_HEADERS }, signal: AbortSignal.timeout(15_000) },
|
|
124
|
+
label,
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
if (r.ok) return r;
|
|
128
|
+
if (r.status === 403) {
|
|
129
|
+
const body = await r.text().catch(() => "");
|
|
130
|
+
if (/automated|undeclared/i.test(body)) {
|
|
131
|
+
throw new ToolErrorCarrier({
|
|
132
|
+
kind: "invalid_input",
|
|
133
|
+
message: `SEC EDGAR rejected the request as automated/undeclared (HTTP 403) at ${label}. SEC requires a descriptive, non-github User-Agent — set EDGAR_USER_AGENT (e.g. "Sample Company AdminContact@example.com"). Upstream note: ${body.slice(0, 160)}`,
|
|
134
|
+
retryable: false,
|
|
135
|
+
upstreamStatus: 403,
|
|
136
|
+
upstreamEndpoint: label,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
throw new ToolErrorCarrier({
|
|
140
|
+
kind: "rate_limited",
|
|
141
|
+
message: `SEC EDGAR returned HTTP 403 at ${label} — likely the 10 req/s limit / a ~10-minute IP block. Slow down and retry after ~10 minutes.`,
|
|
142
|
+
retryable: true,
|
|
143
|
+
retryAfterSeconds: 600,
|
|
144
|
+
upstreamStatus: 403,
|
|
145
|
+
upstreamEndpoint: label,
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
const err = errorFromResponse(r, label);
|
|
149
|
+
if (err.retryable) {
|
|
150
|
+
// 429 / 5xx — reuse the shared retry/backoff machinery.
|
|
151
|
+
return fetchWithRetry(
|
|
152
|
+
url,
|
|
153
|
+
{ headers: { ...EDGAR_HEADERS }, signal: AbortSignal.timeout(15_000) },
|
|
154
|
+
label,
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
// 404 / 400 (non-retryable) — throw now; callers translate 404 → found:false.
|
|
158
|
+
throw new ToolErrorCarrier(err);
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// ─── shared coercions ─────────────────────────────────────────────
|
|
163
|
+
/**
|
|
164
|
+
* A CIK (int or string, padded or not) → the canonical 10-digit zero-padded
|
|
165
|
+
* form EDGAR's data.sec.gov paths require. Strips non-digits first (accepts
|
|
166
|
+
* "CIK320193", "320193", 320193). e.g. 320193 → "0000320193".
|
|
167
|
+
*/
|
|
168
|
+
export function padCik(x: string | number): string {
|
|
169
|
+
const digits = String(x).replace(/\D/g, "");
|
|
170
|
+
return digits.padStart(10, "0");
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** The un-padded CIK for Archives URLs (SEC drops leading zeros there). */
|
|
174
|
+
function unpadCik(cik10: string): string {
|
|
175
|
+
return cik10.replace(/^0+/, "") || "0";
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Coerce an XBRL value to number|null. Returns **null (never 0)** for absent —
|
|
180
|
+
* a missing fact is an honest "unknown", never a fabricated zero. XBRL `val` is
|
|
181
|
+
* already a JSON number, but guard strings/nullish/non-finite defensively.
|
|
182
|
+
*/
|
|
183
|
+
export function num(x: unknown): number | null {
|
|
184
|
+
if (x === null || x === undefined) return null;
|
|
185
|
+
if (typeof x === "number") return Number.isFinite(x) ? x : null;
|
|
186
|
+
if (typeof x === "string") {
|
|
187
|
+
const s = x.trim();
|
|
188
|
+
if (s === "" || s.toLowerCase() === "null") return null;
|
|
189
|
+
const n = Number(s);
|
|
190
|
+
return Number.isFinite(n) ? n : null;
|
|
191
|
+
}
|
|
192
|
+
return null;
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/** null for absent (null/undefined/""), else the trimmed string. */
|
|
196
|
+
function str(x: unknown): string | null {
|
|
197
|
+
if (x === null || x === undefined) return null;
|
|
198
|
+
const s = String(x).trim();
|
|
199
|
+
return s === "" ? null : s;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// ─── ticker → CIK map (memoized ~6h) ──────────────────────────────
|
|
203
|
+
export type TickerEntry = { cik: string; ticker: string; title: string };
|
|
204
|
+
|
|
205
|
+
async function fetchTickerMap(): Promise<TickerEntry[]> {
|
|
206
|
+
const r = await getEdgar(TICKERS_URL, "edgar:tickers");
|
|
207
|
+
const d = (await r.json()) as unknown;
|
|
208
|
+
if (!d || typeof d !== "object") {
|
|
209
|
+
throw new ToolErrorCarrier({
|
|
210
|
+
kind: "schema_drift",
|
|
211
|
+
message:
|
|
212
|
+
"edgar:tickers returned an unexpected shape (company_tickers.json should be a keyed dict of {cik_str,ticker,title}).",
|
|
213
|
+
retryable: false,
|
|
214
|
+
upstreamEndpoint: "edgar:tickers",
|
|
215
|
+
});
|
|
216
|
+
}
|
|
217
|
+
const out: TickerEntry[] = [];
|
|
218
|
+
for (const v of Object.values(d as Record<string, unknown>)) {
|
|
219
|
+
const e = v as { cik_str?: unknown; ticker?: unknown; title?: unknown };
|
|
220
|
+
if (e && e.cik_str != null && e.ticker != null) {
|
|
221
|
+
out.push({
|
|
222
|
+
cik: padCik(e.cik_str as string | number),
|
|
223
|
+
ticker: String(e.ticker),
|
|
224
|
+
title: e.title == null ? "" : String(e.title),
|
|
225
|
+
});
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
return out;
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** The full ticker→CIK table, memoized ~6h (slow-changing reference read). */
|
|
232
|
+
async function tickerMap(): Promise<TickerEntry[]> {
|
|
233
|
+
return memoize("edgar:tickers", fetchTickerMap, 6 * 60 * 60 * 1000);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* Resolve a `cikOrTicker` argument to a 10-digit CIK. Pure digits (or "CIK…")
|
|
238
|
+
* → padCik directly (no network). Otherwise look up the memoized ticker map:
|
|
239
|
+
* exact ticker match first, then a case-insensitive title substring. null when
|
|
240
|
+
* nothing matches (caller → honest found:false).
|
|
241
|
+
*/
|
|
242
|
+
async function resolveCik(
|
|
243
|
+
cikOrTicker: string,
|
|
244
|
+
): Promise<{ cik: string; ticker: string | null; title: string | null } | null> {
|
|
245
|
+
const raw = cikOrTicker.trim();
|
|
246
|
+
if (/^(cik)?\s*\d+$/i.test(raw)) {
|
|
247
|
+
return { cik: padCik(raw), ticker: null, title: null };
|
|
248
|
+
}
|
|
249
|
+
const map = await tickerMap();
|
|
250
|
+
const upper = raw.toUpperCase();
|
|
251
|
+
const exact = map.find((e) => e.ticker.toUpperCase() === upper);
|
|
252
|
+
if (exact) return { cik: exact.cik, ticker: exact.ticker, title: exact.title };
|
|
253
|
+
const byName = map.find((e) => e.title.toUpperCase().includes(upper));
|
|
254
|
+
if (byName) return { cik: byName.cik, ticker: byName.ticker, title: byName.title };
|
|
255
|
+
return null;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// ─── meta helper ──────────────────────────────────────────────────
|
|
259
|
+
/**
|
|
260
|
+
* Build a partial `_meta` with the EDGAR source + the mandatory CIK↔UEI caveat
|
|
261
|
+
* appended to every tool's notes and `fieldsUnavailable` set. Tool-specific
|
|
262
|
+
* signals (returned/totalAvailable/pagination/totalIsLowerBound/complete) are
|
|
263
|
+
* merged in via `extra`; buildMeta DERIVES complete/truncated from them.
|
|
264
|
+
*/
|
|
265
|
+
function edgarMeta(extra: Partial<ResponseMeta>): Partial<ResponseMeta> {
|
|
266
|
+
return {
|
|
267
|
+
source: EDGAR_SOURCE,
|
|
268
|
+
keylessMode: true,
|
|
269
|
+
filtersApplied: extra.filtersApplied ?? [],
|
|
270
|
+
filtersDropped: [],
|
|
271
|
+
fieldsUnavailable: EDGAR_FIELDS_UNAVAILABLE,
|
|
272
|
+
...extra,
|
|
273
|
+
notes: [...(extra.notes ?? []), CIK_UEI_CAVEAT],
|
|
274
|
+
};
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/** Honest empty for an unresolved/unknown filer: a definitive answer (complete). */
|
|
278
|
+
function notFoundBundle(identifier: string, note: string): MetaBundle {
|
|
279
|
+
return withMeta(
|
|
280
|
+
{ found: false, identifier },
|
|
281
|
+
edgarMeta({
|
|
282
|
+
returned: 0,
|
|
283
|
+
totalAvailable: 0,
|
|
284
|
+
complete: true,
|
|
285
|
+
notes: [note],
|
|
286
|
+
}),
|
|
287
|
+
);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// ─── Tool 1: edgar_lookup_cik ─────────────────────────────────────
|
|
291
|
+
/**
|
|
292
|
+
* Map a company query (exact ticker, else title substring) to its 10-digit CIK
|
|
293
|
+
* via the memoized company_tickers.json. Returns up to 50 matches; `found:false`
|
|
294
|
+
* on none. The CIK is the join key the other three tools take.
|
|
295
|
+
*/
|
|
296
|
+
export async function lookupCik(args: { query: string }): Promise<MetaBundle> {
|
|
297
|
+
const q = args.query.trim();
|
|
298
|
+
const map = await tickerMap();
|
|
299
|
+
const upper = q.toUpperCase();
|
|
300
|
+
const exact = map.filter((e) => e.ticker.toUpperCase() === upper);
|
|
301
|
+
const matched = exact.length
|
|
302
|
+
? exact
|
|
303
|
+
: map.filter((e) => e.title.toUpperCase().includes(upper));
|
|
304
|
+
const LIMIT = 50;
|
|
305
|
+
const results = matched
|
|
306
|
+
.slice(0, LIMIT)
|
|
307
|
+
.map((e) => ({ cik: e.cik, ticker: e.ticker, title: e.title }));
|
|
308
|
+
const notes: string[] = [];
|
|
309
|
+
if (matched.length === 0) {
|
|
310
|
+
notes.push(
|
|
311
|
+
`No company in SEC's ticker registry matched "${q}" (exact ticker or title substring). Not every SEC filer has a ticker; try edgar_full_text_search by name.`,
|
|
312
|
+
);
|
|
313
|
+
} else {
|
|
314
|
+
notes.push(
|
|
315
|
+
exact.length
|
|
316
|
+
? `Matched by exact ticker "${q.toUpperCase()}".`
|
|
317
|
+
: `Matched by title substring "${q}" (case-insensitive).`,
|
|
318
|
+
);
|
|
319
|
+
if (matched.length > LIMIT) {
|
|
320
|
+
notes.push(
|
|
321
|
+
`${matched.length} companies matched; returning the first ${LIMIT}. Refine the query for a specific filer.`,
|
|
322
|
+
);
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
return withMeta(
|
|
326
|
+
{ found: results.length > 0, query: q, results },
|
|
327
|
+
edgarMeta({
|
|
328
|
+
returned: results.length,
|
|
329
|
+
totalAvailable: matched.length,
|
|
330
|
+
filtersApplied: ["query"],
|
|
331
|
+
notes,
|
|
332
|
+
}),
|
|
333
|
+
);
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// ─── Tool 2: edgar_company_filings ────────────────────────────────
|
|
337
|
+
export type Filing = {
|
|
338
|
+
accession: string | null;
|
|
339
|
+
form: string | null;
|
|
340
|
+
filingDate: string | null;
|
|
341
|
+
reportDate: string | null;
|
|
342
|
+
primaryDocument: string | null;
|
|
343
|
+
primaryDocDescription: string | null;
|
|
344
|
+
primaryDocUrl: string | null;
|
|
345
|
+
isXBRL: boolean;
|
|
346
|
+
};
|
|
347
|
+
|
|
348
|
+
type SubmissionsRecent = Record<string, unknown[]>;
|
|
349
|
+
type SubmissionsShard = {
|
|
350
|
+
name?: string;
|
|
351
|
+
filingCount?: number;
|
|
352
|
+
filingFrom?: string;
|
|
353
|
+
filingTo?: string;
|
|
354
|
+
};
|
|
355
|
+
|
|
356
|
+
/**
|
|
357
|
+
* The fixed grammar for a `filings.files[].name` older-submissions shard (ADR-0019).
|
|
358
|
+
* Live-confirmed 2026-07-12: `CIK0000320193-submissions-001.json`,
|
|
359
|
+
* `CIK0000019617-submissions-068.json`. This is the FIRST-pass SSRF grammar guard
|
|
360
|
+
* (SEC-format check); `fetchShard` ALSO builds a stronger CIK-BOUND regex from the
|
|
361
|
+
* resolved parent CIK (M3) so a name for ANY OTHER CIK is refused before any fetch.
|
|
362
|
+
* The `\d{1,4}` shard-number quantifier is bounded (a giant number just 404s); the
|
|
363
|
+
* `%`/`/`/`.`/`\`/`..` traversal characters are simply not in the allowed classes.
|
|
364
|
+
*/
|
|
365
|
+
const SHARD_NAME_RE = /^CIK\d{10}-submissions-\d{1,4}\.json$/;
|
|
366
|
+
|
|
367
|
+
/**
|
|
368
|
+
* Unroll the COLUMNAR `filings.recent` (parallel arrays; index i = one filing)
|
|
369
|
+
* into `Filing[]`, constructing the primary-document ARCHIVE URL from the real
|
|
370
|
+
* accession + primaryDocument. Alignment is strictly by index i.
|
|
371
|
+
*/
|
|
372
|
+
function zipRecent(recent: SubmissionsRecent, cik10: string): Filing[] {
|
|
373
|
+
const accs = (recent.accessionNumber as string[]) ?? [];
|
|
374
|
+
const cikUnpadded = unpadCik(cik10);
|
|
375
|
+
const out: Filing[] = [];
|
|
376
|
+
for (let i = 0; i < accs.length; i++) {
|
|
377
|
+
const accession = str(accs[i]);
|
|
378
|
+
const primaryDocument = str(recent.primaryDocument?.[i]);
|
|
379
|
+
let primaryDocUrl: string | null = null;
|
|
380
|
+
if (accession && primaryDocument) {
|
|
381
|
+
const accNoDash = accession.replace(/-/g, "");
|
|
382
|
+
primaryDocUrl = `${ARCHIVES_BASE}/${cikUnpadded}/${accNoDash}/${primaryDocument}`;
|
|
383
|
+
}
|
|
384
|
+
out.push({
|
|
385
|
+
accession,
|
|
386
|
+
form: str(recent.form?.[i]),
|
|
387
|
+
filingDate: str(recent.filingDate?.[i]),
|
|
388
|
+
reportDate: str(recent.reportDate?.[i]),
|
|
389
|
+
primaryDocument,
|
|
390
|
+
primaryDocDescription: str(recent.primaryDocDescription?.[i]),
|
|
391
|
+
primaryDocUrl,
|
|
392
|
+
isXBRL: recent.isXBRL?.[i] === 1 || recent.isXBRL?.[i] === true,
|
|
393
|
+
});
|
|
394
|
+
}
|
|
395
|
+
return out;
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
async function fetchSubmissions(cik10: string): Promise<{
|
|
399
|
+
name: string | null;
|
|
400
|
+
recent: SubmissionsRecent;
|
|
401
|
+
files: SubmissionsShard[];
|
|
402
|
+
recentCount: number;
|
|
403
|
+
}> {
|
|
404
|
+
const url = `${SUBMISSIONS_BASE}/CIK${cik10}.json`;
|
|
405
|
+
const r = await getEdgar(url, "edgar:submissions");
|
|
406
|
+
const d = (await r.json()) as {
|
|
407
|
+
name?: string;
|
|
408
|
+
filings?: { recent?: SubmissionsRecent; files?: SubmissionsShard[] };
|
|
409
|
+
};
|
|
410
|
+
const recent = d?.filings?.recent;
|
|
411
|
+
if (!recent || !Array.isArray(recent.accessionNumber)) {
|
|
412
|
+
throw new ToolErrorCarrier({
|
|
413
|
+
kind: "schema_drift",
|
|
414
|
+
message: `edgar:submissions returned an unexpected shape for CIK ${cik10} (filings.recent.accessionNumber array missing).`,
|
|
415
|
+
retryable: false,
|
|
416
|
+
upstreamEndpoint: "edgar:submissions",
|
|
417
|
+
});
|
|
418
|
+
}
|
|
419
|
+
const files = Array.isArray(d.filings?.files) ? d.filings!.files! : [];
|
|
420
|
+
return {
|
|
421
|
+
name: str(d.name),
|
|
422
|
+
recent,
|
|
423
|
+
files,
|
|
424
|
+
recentCount: recent.accessionNumber.length,
|
|
425
|
+
};
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
/**
|
|
429
|
+
* The 7 parallel-array columns `zipRecent` reads by index i. The INTRA-shard
|
|
430
|
+
* shape guard (M1 — the REAL alignment guard, the `pts===data.length` analogue):
|
|
431
|
+
* a shard 200 body whose `accessionNumber` is missing/non-array, or any of these
|
|
432
|
+
* 7 arrays is not `=== accessionNumber.length`, would let `zipRecent` borrow a
|
|
433
|
+
* value from a shorter/absent column at index i → a fabricated (form,date,accession)
|
|
434
|
+
* tuple. So a non-columnar/ragged shard THROWS schema_drift (never emits rows).
|
|
435
|
+
*/
|
|
436
|
+
const SHARD_ZIP_COLUMNS = [
|
|
437
|
+
"accessionNumber",
|
|
438
|
+
"form",
|
|
439
|
+
"filingDate",
|
|
440
|
+
"reportDate",
|
|
441
|
+
"primaryDocument",
|
|
442
|
+
"primaryDocDescription",
|
|
443
|
+
"isXBRL",
|
|
444
|
+
] as const;
|
|
445
|
+
|
|
446
|
+
/**
|
|
447
|
+
* Fetch ONE older-submissions shard and return its `Filing[]` (ADR-0019). The
|
|
448
|
+
* shard `name` is SERVER-PROVIDED (from the parent `filings.files[].name`); the
|
|
449
|
+
* caller never constructs it from user input. Belt-and-suspenders, mirroring
|
|
450
|
+
* `buildFramesUrl` (S1/S2):
|
|
451
|
+
* - **[M3] CIK-BIND + grammar validate BEFORE any URL is built.** The name must
|
|
452
|
+
* match BOTH the fixed SEC grammar (`SHARD_NAME_RE`) AND a regex bound to THIS
|
|
453
|
+
* filer's resolved `cik10` — so a drifted/hostile/MITM'd parent whose name
|
|
454
|
+
* embeds a DIFFERENT CIK (`CIK9999999999-…`) or a traversal (`../`, `%2F`) is
|
|
455
|
+
* NEVER fetched (thrown as `not_found` → the caller skips + discloses PARTIAL).
|
|
456
|
+
* - Fixed host: build `${SUBMISSIONS_BASE}/${name}` and assert the parsed URL is
|
|
457
|
+
* `https://data.sec.gov` (host-escape/downgrade guard).
|
|
458
|
+
* - `getEdgar` → the shard body; **[M1]** assert the columnar shape (SHARD_ZIP_COLUMNS
|
|
459
|
+
* all equal-length) or THROW schema_drift; then `zipRecent`.
|
|
460
|
+
* Throw taxonomy the caller relies on: `not_found` (bad/cross-CIK/host name OR a
|
|
461
|
+
* genuine 404) → skip+PARTIAL; `schema_drift`/`rate_limited`/other → re-throw loud.
|
|
462
|
+
*/
|
|
463
|
+
async function fetchShard(name: string, cik10: string): Promise<Filing[]> {
|
|
464
|
+
// [M3] CIK-bound grammar — ONLY a shard whose embedded CIK === this filer's CIK
|
|
465
|
+
// is eligible. cik10 is 10 zero-padded DIGITS (padCik strips non-digits), so it
|
|
466
|
+
// is safe to interpolate into a RegExp source (no metacharacters).
|
|
467
|
+
const cikBoundRe = new RegExp(`^CIK${cik10}-submissions-\\d{1,4}\\.json$`);
|
|
468
|
+
if (!SHARD_NAME_RE.test(name) || !cikBoundRe.test(name)) {
|
|
469
|
+
throw new ToolErrorCarrier({
|
|
470
|
+
kind: "not_found",
|
|
471
|
+
message: `edgar:submissions shard name ${JSON.stringify(name)} did not match the CIK-bound grammar ^CIK${cik10}-submissions-\\d{1,4}\\.json$ (SEC format change, or a cross-entity/hostile name) — refused before any fetch, counted as a skipped shard.`,
|
|
472
|
+
retryable: false,
|
|
473
|
+
upstreamEndpoint: "edgar:submissions",
|
|
474
|
+
});
|
|
475
|
+
}
|
|
476
|
+
const built = `${SUBMISSIONS_BASE}/${name}`;
|
|
477
|
+
let parsed: URL;
|
|
478
|
+
try {
|
|
479
|
+
parsed = new URL(built);
|
|
480
|
+
} catch {
|
|
481
|
+
throw new ToolErrorCarrier({
|
|
482
|
+
kind: "not_found",
|
|
483
|
+
message: `edgar:submissions shard URL could not be constructed from ${JSON.stringify(name)} — refused before any fetch.`,
|
|
484
|
+
retryable: false,
|
|
485
|
+
upstreamEndpoint: "edgar:submissions",
|
|
486
|
+
});
|
|
487
|
+
}
|
|
488
|
+
if (parsed.protocol !== "https:" || parsed.hostname !== "data.sec.gov") {
|
|
489
|
+
throw new ToolErrorCarrier({
|
|
490
|
+
kind: "not_found",
|
|
491
|
+
message: `edgar:submissions shard URL host/scheme is not https://data.sec.gov (${parsed.protocol}//${parsed.hostname}) — refused before any fetch (fixed-host assertion).`,
|
|
492
|
+
retryable: false,
|
|
493
|
+
upstreamEndpoint: "edgar:submissions",
|
|
494
|
+
});
|
|
495
|
+
}
|
|
496
|
+
const r = await getEdgar(built, "edgar:submissions");
|
|
497
|
+
const shard = (await r.json()) as SubmissionsRecent;
|
|
498
|
+
// [M1] Intra-shard columnar-shape guard (the real alignment guard).
|
|
499
|
+
const acc = shard?.accessionNumber;
|
|
500
|
+
if (!Array.isArray(acc)) {
|
|
501
|
+
throw new ToolErrorCarrier({
|
|
502
|
+
kind: "schema_drift",
|
|
503
|
+
message: `edgar:submissions shard ${name} returned HTTP 200 without a columnar accessionNumber[] array — the submissions shard envelope changed.`,
|
|
504
|
+
retryable: false,
|
|
505
|
+
upstreamEndpoint: "edgar:submissions",
|
|
506
|
+
});
|
|
507
|
+
}
|
|
508
|
+
for (const col of SHARD_ZIP_COLUMNS) {
|
|
509
|
+
const arr = shard[col];
|
|
510
|
+
if (!Array.isArray(arr) || arr.length !== acc.length) {
|
|
511
|
+
throw new ToolErrorCarrier({
|
|
512
|
+
kind: "schema_drift",
|
|
513
|
+
message: `edgar:submissions shard ${name} is not columnar: '${col}' is ${Array.isArray(arr) ? `length ${arr.length}` : "missing/non-array"} ≠ accessionNumber length ${acc.length}. Emitting index-aligned rows would fabricate (form,date,accession) tuples — refusing.`,
|
|
514
|
+
retryable: false,
|
|
515
|
+
upstreamEndpoint: "edgar:submissions",
|
|
516
|
+
});
|
|
517
|
+
}
|
|
518
|
+
}
|
|
519
|
+
return zipRecent(shard, cik10);
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
/**
|
|
523
|
+
* A company's SEC filings. By default (`fullHistory` off) returns the recent
|
|
524
|
+
* window (from `filings.recent` — up to 1 year OR 1000 filings, whichever is
|
|
525
|
+
* more), optionally narrowed to specific `forms`, with offset pagination. HONESTY:
|
|
526
|
+
* the response is COMPLETE only when `filings.files[]` (older shards) is empty;
|
|
527
|
+
* when shards exist, `totalAvailable` is the grand total (recent + Σ shard counts),
|
|
528
|
+
* `hasMore:true`, and a note discloses that only the recent window was searched.
|
|
529
|
+
* With `fullHistory:true`, the older `files[]` shards are fetched (newest-first up
|
|
530
|
+
* to `maxShards`, default 10) and assembled (recent ++ shard001..N, descending
|
|
531
|
+
* preserved, NO re-sort) into the COMPLETE history — a capped/failed fan-out is
|
|
532
|
+
* disclosed as PARTIAL (never a capped set claimed complete). `totalAvailable`
|
|
533
|
+
* stays the grand total regardless of the cap (buildMeta forces complete:false
|
|
534
|
+
* when returned < total).
|
|
535
|
+
*/
|
|
536
|
+
export async function companyFilings(args: {
|
|
537
|
+
cikOrTicker: string;
|
|
538
|
+
forms?: string[];
|
|
539
|
+
limit?: number;
|
|
540
|
+
offset?: number;
|
|
541
|
+
fullHistory?: boolean;
|
|
542
|
+
maxShards?: number;
|
|
543
|
+
}): Promise<MetaBundle> {
|
|
544
|
+
const resolved = await resolveCik(args.cikOrTicker);
|
|
545
|
+
if (!resolved) {
|
|
546
|
+
return notFoundBundle(
|
|
547
|
+
args.cikOrTicker,
|
|
548
|
+
`Could not resolve "${args.cikOrTicker}" to a CIK (not a numeric CIK and no exact-ticker/title match in company_tickers.json).`,
|
|
549
|
+
);
|
|
550
|
+
}
|
|
551
|
+
const cik = resolved.cik;
|
|
552
|
+
let subm;
|
|
553
|
+
try {
|
|
554
|
+
subm = await fetchSubmissions(cik);
|
|
555
|
+
} catch (e) {
|
|
556
|
+
if (e instanceof ToolErrorCarrier && e.toolError.kind === "not_found") {
|
|
557
|
+
return notFoundBundle(
|
|
558
|
+
cik,
|
|
559
|
+
`No SEC submissions found for CIK ${cik} (HTTP 404) — the CIK does not exist or has no filings. Not fabricated.`,
|
|
560
|
+
);
|
|
561
|
+
}
|
|
562
|
+
throw e;
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
const fullHistory = args.fullHistory ?? false;
|
|
566
|
+
const maxShards = args.maxShards ?? 10;
|
|
567
|
+
|
|
568
|
+
const hasShards = subm.files.length > 0;
|
|
569
|
+
const totalShards = subm.files.length;
|
|
570
|
+
// Σ ALL files[].filingCount — the grand shard total, INCLUDING un-fetched shards.
|
|
571
|
+
// Authoritative from the PARENT payload alone; never recomputed down (M1).
|
|
572
|
+
const shardCount = subm.files.reduce((s, f) => s + (f.filingCount ?? 0), 0);
|
|
573
|
+
|
|
574
|
+
// ── Shard fan-out (gated STRICTLY on fullHistory && there are shards) ──
|
|
575
|
+
// Assemble `recent ++ shard001 ++ … ++ shardNNN` — descending date order is
|
|
576
|
+
// preserved by construction (recent newest, then each shard newest-first), so
|
|
577
|
+
// there is NO re-sort (a blind re-sort would risk breaking SEC's own tie order).
|
|
578
|
+
let assembled = zipRecent(subm.recent, cik);
|
|
579
|
+
let fetchedShards = 0;
|
|
580
|
+
let failedShards = 0; // shards ATTEMPTED (within the cap) that 404'd / were refused.
|
|
581
|
+
let failedFilings = 0; // Σ declared filingCount of those FAILED shards.
|
|
582
|
+
let shortfallFilings = 0; // Σ (declared − actual) for SHORT (but fetched) shards.
|
|
583
|
+
const shardNotes: string[] = [];
|
|
584
|
+
const fanoutRan = fullHistory && hasShards;
|
|
585
|
+
if (fanoutRan) {
|
|
586
|
+
// newest-first up to maxShards (files[] is ordered shard-001 newest → shard-NNN oldest).
|
|
587
|
+
const toFetch = subm.files.slice(0, maxShards);
|
|
588
|
+
for (const shardEntry of toFetch) {
|
|
589
|
+
const name = shardEntry.name ?? "";
|
|
590
|
+
const declared = shardEntry.filingCount ?? 0;
|
|
591
|
+
try {
|
|
592
|
+
const rows = await fetchShard(name, cik);
|
|
593
|
+
fetchedShards++;
|
|
594
|
+
assembled = assembled.concat(rows);
|
|
595
|
+
// [M1] filingCount !== length is INTER-document (parent vs a later shard
|
|
596
|
+
// fetch) and can be legit (recent→shard roll / upstream lag). Do NOT throw;
|
|
597
|
+
// use the ACTUAL fetched count and disclose the shortfall as partial.
|
|
598
|
+
if (rows.length < declared) {
|
|
599
|
+
shortfallFilings += declared - rows.length;
|
|
600
|
+
shardNotes.push(
|
|
601
|
+
`shard ${name} returned ${rows.length} of ${declared} declared filings (recent→shard roll or upstream lag); not fabricated.`,
|
|
602
|
+
);
|
|
603
|
+
}
|
|
604
|
+
} catch (e) {
|
|
605
|
+
const kind = e instanceof ToolErrorCarrier ? e.toolError.kind : "unknown";
|
|
606
|
+
// not_found = a genuine 404 OR a refused bad/cross-CIK/host name (0 fetch).
|
|
607
|
+
// Degrade to PARTIAL and continue; the declared filings are missing. This is
|
|
608
|
+
// a FAILURE of an ATTEMPTED shard — NOT the maxShards cap (see cappedShards).
|
|
609
|
+
if (kind === "not_found") {
|
|
610
|
+
failedShards++;
|
|
611
|
+
failedFilings += declared;
|
|
612
|
+
continue;
|
|
613
|
+
}
|
|
614
|
+
// schema_drift / rate_limited / invalid_input / upstream_unavailable →
|
|
615
|
+
// fail LOUD (a shape break or a systemic UA/rate/outage fault is not a
|
|
616
|
+
// per-shard degrade — never a fake-partial).
|
|
617
|
+
throw e;
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
// CAP remainder — the older shards the maxShards cap NEVER ATTEMPTED (totalShards >
|
|
622
|
+
// maxShards). These are the ONLY shards reachable by RAISING maxShards; a shard that
|
|
623
|
+
// was attempted-and-404'd is a FAILURE (failedShards), NOT a cap remainder, and
|
|
624
|
+
// raising the cap will NOT recover it. Keeping the two causes separate is the fix.
|
|
625
|
+
const cappedShards = fanoutRan && totalShards > maxShards ? totalShards - maxShards : 0;
|
|
626
|
+
const cappedFilings =
|
|
627
|
+
cappedShards > 0
|
|
628
|
+
? subm.files.slice(maxShards).reduce((s, f) => s + (f.filingCount ?? 0), 0)
|
|
629
|
+
: 0;
|
|
630
|
+
|
|
631
|
+
const forms = args.forms?.map((f) => f.trim().toUpperCase()).filter(Boolean);
|
|
632
|
+
const filtered =
|
|
633
|
+
forms && forms.length
|
|
634
|
+
? assembled.filter((f) => f.form != null && forms.includes(f.form.toUpperCase()))
|
|
635
|
+
: assembled;
|
|
636
|
+
|
|
637
|
+
const limit = args.limit ?? 20;
|
|
638
|
+
const offset = args.offset ?? 0;
|
|
639
|
+
const page = filtered.slice(offset, offset + limit);
|
|
640
|
+
const returned = page.length;
|
|
641
|
+
|
|
642
|
+
const moreInWindow = offset + returned < filtered.length;
|
|
643
|
+
// [M2/M4] Two "more" axes. When NOT fanning out, the beyond-window axis is
|
|
644
|
+
// today's "shards exist but were not fetched" (hasShards). When fanning out, it
|
|
645
|
+
// is the CAP remainder OR any FAILED shard (cappedShards>0 || failedShards>0 —
|
|
646
|
+
// equivalent to fetchedShards<totalShards but attributed to its actual cause),
|
|
647
|
+
// NOT the plain hasShards boolean. A mutation that sets hasMore from moreInWindow
|
|
648
|
+
// alone → RED.
|
|
649
|
+
const beyondWindow = fanoutRan
|
|
650
|
+
? cappedShards > 0 || failedShards > 0
|
|
651
|
+
: hasShards;
|
|
652
|
+
const hasMore = moreInWindow || beyondWindow;
|
|
653
|
+
// nextOffset walks ONLY the assembled+fetched window; the older un-fetched shards
|
|
654
|
+
// are UNREACHABLE via paging (mirror the FTS beyond-window pattern: hasMore true
|
|
655
|
+
// while nextOffset is null).
|
|
656
|
+
const nextOffset = moreInWindow ? offset + returned : null;
|
|
657
|
+
// [M1] totalAvailable UNCHANGED — grand total when shards exist (incl un-fetched),
|
|
658
|
+
// else the filtered recent length. Never recomputed down to fetched rows.
|
|
659
|
+
const totalAvailable = hasShards ? subm.recentCount + shardCount : filtered.length;
|
|
660
|
+
|
|
661
|
+
const filtersApplied: string[] = [];
|
|
662
|
+
if (forms && forms.length) filtersApplied.push("forms");
|
|
663
|
+
|
|
664
|
+
const notes: string[] = [];
|
|
665
|
+
if (fanoutRan) {
|
|
666
|
+
// The THREE independent PARTIAL causes, attributed + disclosed SEPARATELY (each
|
|
667
|
+
// only when it actually applies): (1) the maxShards CAP skipped older shards
|
|
668
|
+
// (cappedShards) → RAISE maxShards; (2) an ATTEMPTED shard FAILED/404'd
|
|
669
|
+
// (failedShards) → retry may recover, raising maxShards will NOT; (3) a fetched
|
|
670
|
+
// shard was SHORT (shortfallFilings) → benign roll/lag. "complete" needs all three
|
|
671
|
+
// absent. Misattributing a failure to the cap (and advising "raise maxShards") is
|
|
672
|
+
// the defect this split fixes.
|
|
673
|
+
const complete = cappedShards === 0 && failedShards === 0 && shortfallFilings === 0;
|
|
674
|
+
if (complete) {
|
|
675
|
+
notes.push(
|
|
676
|
+
`COMPLETE filing history: fetched all ${totalShards} older shard(s) plus the recent window (${subm.recentCount}) = ${totalAvailable} filings.`,
|
|
677
|
+
);
|
|
678
|
+
} else {
|
|
679
|
+
if (cappedShards > 0) {
|
|
680
|
+
// PARTIAL-BY-CAP — ONLY the un-attempted older shards the cap skipped; the
|
|
681
|
+
// "RAISE maxShards" advice appears ONLY here (it recovers cap remainder, not
|
|
682
|
+
// a 404). The counts are the CAPPED counts, never a failed-shard count.
|
|
683
|
+
notes.push(
|
|
684
|
+
`PARTIAL history (maxShards cap ${maxShards}): the cap limited the fan-out to the newest ${maxShards} of ${totalShards} shard(s); ${cappedShards} older shard(s) (${cappedFilings} filings) were NOT attempted. Paginate via nextOffset for MORE OF THIS fetched window (the fetched shard(s) + the recent ${subm.recentCount}); pagination will NOT reach the un-attempted older shard(s) — RAISE maxShards (max 100) to fetch them.`,
|
|
685
|
+
);
|
|
686
|
+
}
|
|
687
|
+
if (failedShards > 0) {
|
|
688
|
+
// PARTIAL-BY-FAILURE — shards that WERE attempted (within the cap) but 404'd /
|
|
689
|
+
// errored. NOT a cap issue → do NOT advise raising maxShards (a re-fetch hits
|
|
690
|
+
// the same 404); a retry may recover a transient failure.
|
|
691
|
+
notes.push(
|
|
692
|
+
`PARTIAL history: ${failedShards} attempted shard(s) could not be fetched (HTTP 404 / bad-or-cross-CIK name / transient); ${failedFilings} filing(s) are missing from this history. Not fabricated — a retry may recover a transient failure (raising maxShards will NOT recover a shard that 404'd).`,
|
|
693
|
+
);
|
|
694
|
+
}
|
|
695
|
+
if (shortfallFilings > 0) {
|
|
696
|
+
// A fetched shard returned fewer rows than its parent-declared filingCount
|
|
697
|
+
// (recent→shard roll / upstream lag) — a benign shortfall, per-shard below.
|
|
698
|
+
notes.push(
|
|
699
|
+
`${shortfallFilings} declared filing(s) were absent from otherwise-fetched shard body(ies) (recent→shard roll or upstream lag); disclosed per-shard below and NOT fabricated.`,
|
|
700
|
+
);
|
|
701
|
+
}
|
|
702
|
+
}
|
|
703
|
+
if (forms && forms.length) {
|
|
704
|
+
notes.push(
|
|
705
|
+
complete
|
|
706
|
+
? "The form filter now spans the COMPLETE fetched history (recent + all shards); totalAvailable stays the grand UNFILTERED total, so complete may read false — an under-claim, NOT missing older matches."
|
|
707
|
+
: "Form filtering was applied across the fetched window (recent + fetched shards) — older matching filings may exist in the un-attempted/failed shards.",
|
|
708
|
+
);
|
|
709
|
+
}
|
|
710
|
+
// [M4] Shared-throttle contention: the fan-out serializes N shard GETs through
|
|
711
|
+
// the module-level edgar gate SHARED across all edgar tools.
|
|
712
|
+
notes.push(
|
|
713
|
+
`fullHistory serialized ${fetchedShards + failedShards} shard GET(s) through the shared EDGAR throttle gate (~${EDGAR_MIN_INTERVAL_MS}ms spacing, shared across ALL edgar tools) — a large fan-out adds latency to concurrent edgar tool calls.`,
|
|
714
|
+
);
|
|
715
|
+
notes.push(...shardNotes);
|
|
716
|
+
} else if (hasShards) {
|
|
717
|
+
// fullHistory OFF (default) with shards → today's INCOMPLETE note VERBATIM
|
|
718
|
+
// (backward-compat: byte-identical output when the flag is absent).
|
|
719
|
+
notes.push(
|
|
720
|
+
`INCOMPLETE HISTORY: only the ${subm.recentCount} most-recent filings (filings.recent) were fetched. ${subm.files.length} older shard(s) (~${shardCount} filings, filings.files[]) were NOT fetched, so this is NOT the full filing history.` +
|
|
721
|
+
(forms && forms.length
|
|
722
|
+
? " Form filtering was applied to the recent window only — older matching filings may exist in the un-fetched shards."
|
|
723
|
+
: ""),
|
|
724
|
+
);
|
|
725
|
+
} else {
|
|
726
|
+
notes.push(
|
|
727
|
+
`Complete filing history: filings.files[] is empty, so filings.recent (${subm.recentCount}) is the full set.` +
|
|
728
|
+
(moreInWindow ? " This page is a subset; paginate via nextOffset for the rest." : ""),
|
|
729
|
+
);
|
|
730
|
+
}
|
|
731
|
+
if (resolved.title) notes.push(`Resolved "${args.cikOrTicker}" → ${resolved.title} (CIK ${cik}).`);
|
|
732
|
+
|
|
733
|
+
return withMeta(
|
|
734
|
+
{
|
|
735
|
+
cik,
|
|
736
|
+
entityName: subm.name,
|
|
737
|
+
filings: page,
|
|
738
|
+
},
|
|
739
|
+
edgarMeta({
|
|
740
|
+
returned,
|
|
741
|
+
totalAvailable,
|
|
742
|
+
filtersApplied,
|
|
743
|
+
pagination: {
|
|
744
|
+
offset,
|
|
745
|
+
limit,
|
|
746
|
+
hasMore,
|
|
747
|
+
nextOffset,
|
|
748
|
+
},
|
|
749
|
+
notes,
|
|
750
|
+
}),
|
|
751
|
+
);
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
// ─── Tool 3: edgar_company_facts ──────────────────────────────────
|
|
755
|
+
/**
|
|
756
|
+
* The default curated us-gaap concepts (F4 — NO EarningsPerShareBasic; its unit
|
|
757
|
+
* is USD/shares, so the default unit="USD" would silently return nothing). The
|
|
758
|
+
* two revenue tags cover the same logical concept — filers report under one or
|
|
759
|
+
* the other — so this is the 6 curated USD concepts. Extracting only these
|
|
760
|
+
* avoids returning the full ~500-concept companyfacts payload.
|
|
761
|
+
*/
|
|
762
|
+
export const DEFAULT_FACT_CONCEPTS = [
|
|
763
|
+
"Revenues",
|
|
764
|
+
"RevenueFromContractWithCustomerExcludingAssessedTax",
|
|
765
|
+
"Assets",
|
|
766
|
+
"Liabilities",
|
|
767
|
+
"StockholdersEquity",
|
|
768
|
+
"NetIncomeLoss",
|
|
769
|
+
"CashAndCashEquivalentsAtCarryingValue",
|
|
770
|
+
];
|
|
771
|
+
|
|
772
|
+
type FactPoint = {
|
|
773
|
+
start?: string; // present (ISO date) for DURATION concepts; absent for INSTANT ones
|
|
774
|
+
end: string;
|
|
775
|
+
val: number;
|
|
776
|
+
accn?: string;
|
|
777
|
+
fy?: number;
|
|
778
|
+
fp?: string;
|
|
779
|
+
form?: string;
|
|
780
|
+
filed?: string;
|
|
781
|
+
};
|
|
782
|
+
type FactNode = {
|
|
783
|
+
label?: string;
|
|
784
|
+
description?: string;
|
|
785
|
+
units?: Record<string, FactPoint[]>;
|
|
786
|
+
};
|
|
787
|
+
type FactsDoc = {
|
|
788
|
+
cik?: number;
|
|
789
|
+
entityName?: string;
|
|
790
|
+
facts?: Record<string, Record<string, FactNode>>;
|
|
791
|
+
};
|
|
792
|
+
|
|
793
|
+
async function fetchFacts(cik10: string): Promise<FactsDoc> {
|
|
794
|
+
return memoize(
|
|
795
|
+
`edgar:facts:${cik10}`,
|
|
796
|
+
async () => {
|
|
797
|
+
const url = `${XBRL_FACTS_BASE}/CIK${cik10}.json`;
|
|
798
|
+
const r = await getEdgar(url, "edgar:companyfacts");
|
|
799
|
+
const d = (await r.json()) as FactsDoc;
|
|
800
|
+
if (!d || typeof d.facts !== "object" || d.facts === null) {
|
|
801
|
+
throw new ToolErrorCarrier({
|
|
802
|
+
kind: "schema_drift",
|
|
803
|
+
message: `edgar:companyfacts returned an unexpected shape for CIK ${cik10} (facts object missing).`,
|
|
804
|
+
retryable: false,
|
|
805
|
+
upstreamEndpoint: "edgar:companyfacts",
|
|
806
|
+
});
|
|
807
|
+
}
|
|
808
|
+
return d;
|
|
809
|
+
},
|
|
810
|
+
60 * 60 * 1000,
|
|
811
|
+
);
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
/**
|
|
815
|
+
* Curated XBRL financial facts for a filer. Extracts only the requested (or the
|
|
816
|
+
* 6 default) concepts in the requested `unit` (default USD), from the memoized
|
|
817
|
+
* companyfacts doc. HONESTY: a concept ABSENT for this filer is OMITTED and
|
|
818
|
+
* listed in a note (NEVER surfaced as 0); a concept present only in a DIFFERENT
|
|
819
|
+
* unit (e.g. EPS in USD/shares) is listed under `wrongUnit` with a note, never a
|
|
820
|
+
* silent 0. `latest` reduces each concept to its single most-recent data point.
|
|
821
|
+
*/
|
|
822
|
+
export async function companyFacts(args: {
|
|
823
|
+
cikOrTicker: string;
|
|
824
|
+
concepts?: string[];
|
|
825
|
+
unit?: string;
|
|
826
|
+
latest?: boolean;
|
|
827
|
+
}): Promise<MetaBundle> {
|
|
828
|
+
const resolved = await resolveCik(args.cikOrTicker);
|
|
829
|
+
if (!resolved) {
|
|
830
|
+
return notFoundBundle(
|
|
831
|
+
args.cikOrTicker,
|
|
832
|
+
`Could not resolve "${args.cikOrTicker}" to a CIK (not a numeric CIK and no exact-ticker/title match in company_tickers.json).`,
|
|
833
|
+
);
|
|
834
|
+
}
|
|
835
|
+
const cik = resolved.cik;
|
|
836
|
+
let doc: FactsDoc;
|
|
837
|
+
try {
|
|
838
|
+
doc = await fetchFacts(cik);
|
|
839
|
+
} catch (e) {
|
|
840
|
+
if (e instanceof ToolErrorCarrier && e.toolError.kind === "not_found") {
|
|
841
|
+
return notFoundBundle(
|
|
842
|
+
cik,
|
|
843
|
+
`No XBRL company facts for CIK ${cik} (HTTP 404) — the CIK does not exist or has filed no XBRL financial data. Not fabricated.`,
|
|
844
|
+
);
|
|
845
|
+
}
|
|
846
|
+
throw e;
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
const unit = (args.unit ?? "USD").trim();
|
|
850
|
+
const latest = args.latest ?? false;
|
|
851
|
+
const requested =
|
|
852
|
+
args.concepts && args.concepts.length ? args.concepts : DEFAULT_FACT_CONCEPTS;
|
|
853
|
+
|
|
854
|
+
const usGaap = doc.facts?.["us-gaap"] ?? {};
|
|
855
|
+
const dei = doc.facts?.["dei"] ?? {};
|
|
856
|
+
|
|
857
|
+
const concepts: Array<{
|
|
858
|
+
concept: string;
|
|
859
|
+
label: string | null;
|
|
860
|
+
unit: string;
|
|
861
|
+
points: Array<{
|
|
862
|
+
start: string | null; // null for INSTANT concepts; the ISO period-start for DURATION concepts (mirrors companyConcept M1)
|
|
863
|
+
end: string | null;
|
|
864
|
+
val: number | null;
|
|
865
|
+
accn: string | null;
|
|
866
|
+
fy: number | null;
|
|
867
|
+
fp: string | null;
|
|
868
|
+
form: string | null;
|
|
869
|
+
filed: string | null;
|
|
870
|
+
}>;
|
|
871
|
+
}> = [];
|
|
872
|
+
const absent: string[] = [];
|
|
873
|
+
const wrongUnit: Array<{ concept: string; availableUnits: string[] }> = [];
|
|
874
|
+
// Concepts where ≥2 points share an `end` with DIFFERING `start` (a quarter AND a
|
|
875
|
+
// YTD/annual figure coexist under one `end`) — surfaced in a note so a consumer
|
|
876
|
+
// never conflates a Q4 figure with a full-year one.
|
|
877
|
+
const sameEndMultiDuration: string[] = [];
|
|
878
|
+
|
|
879
|
+
for (const concept of requested) {
|
|
880
|
+
const node: FactNode | undefined = usGaap[concept] ?? dei[concept];
|
|
881
|
+
if (!node) {
|
|
882
|
+
absent.push(concept);
|
|
883
|
+
continue;
|
|
884
|
+
}
|
|
885
|
+
const units = node.units ?? {};
|
|
886
|
+
const series = units[unit];
|
|
887
|
+
if (!series || !Array.isArray(series)) {
|
|
888
|
+
wrongUnit.push({ concept, availableUnits: Object.keys(units) });
|
|
889
|
+
continue;
|
|
890
|
+
}
|
|
891
|
+
let points = series.map((p) => ({
|
|
892
|
+
start: str(p.start), // null for INSTANT concepts; ISO period-start for DURATION (M1)
|
|
893
|
+
end: str(p.end),
|
|
894
|
+
val: num(p.val),
|
|
895
|
+
accn: str(p.accn),
|
|
896
|
+
fy: typeof p.fy === "number" ? p.fy : null,
|
|
897
|
+
fp: str(p.fp),
|
|
898
|
+
form: str(p.form),
|
|
899
|
+
filed: str(p.filed),
|
|
900
|
+
}));
|
|
901
|
+
// Detect same-`end` different-`start` coexistence on the FULL series (before any
|
|
902
|
+
// `latest` reduction): group by `end`, flag when one `end` carries ≥2 distinct
|
|
903
|
+
// `start` values (a shorter period AND a longer one under the same end date).
|
|
904
|
+
const startsByEnd = new Map<string, Set<string>>();
|
|
905
|
+
for (const p of points) {
|
|
906
|
+
const endKey = p.end ?? "";
|
|
907
|
+
let set = startsByEnd.get(endKey);
|
|
908
|
+
if (!set) {
|
|
909
|
+
set = new Set<string>();
|
|
910
|
+
startsByEnd.set(endKey, set);
|
|
911
|
+
}
|
|
912
|
+
set.add(p.start ?? "");
|
|
913
|
+
}
|
|
914
|
+
if ([...startsByEnd.values()].some((set) => set.size >= 2)) {
|
|
915
|
+
sameEndMultiDuration.push(concept);
|
|
916
|
+
}
|
|
917
|
+
if (latest && points.length > 0) {
|
|
918
|
+
// Most-recent by period end (fallback: filed date), keep a single point.
|
|
919
|
+
points = [
|
|
920
|
+
points.reduce((best, p) =>
|
|
921
|
+
(p.end ?? "") > (best.end ?? "") ||
|
|
922
|
+
((p.end ?? "") === (best.end ?? "") && (p.filed ?? "") > (best.filed ?? ""))
|
|
923
|
+
? p
|
|
924
|
+
: best,
|
|
925
|
+
),
|
|
926
|
+
];
|
|
927
|
+
}
|
|
928
|
+
concepts.push({ concept, label: str(node.label), unit, points });
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
const notes: string[] = [];
|
|
932
|
+
if (absent.length) {
|
|
933
|
+
notes.push(
|
|
934
|
+
`Concept(s) not reported by this filer (OMITTED, not zero): ${absent.join(", ")}.`,
|
|
935
|
+
);
|
|
936
|
+
}
|
|
937
|
+
if (wrongUnit.length) {
|
|
938
|
+
for (const w of wrongUnit) {
|
|
939
|
+
notes.push(
|
|
940
|
+
`Concept "${w.concept}" exists but not in unit "${unit}" (available: ${w.availableUnits.join(", ") || "none"}). Re-request with the correct unit — NOT reported as 0.`,
|
|
941
|
+
);
|
|
942
|
+
}
|
|
943
|
+
}
|
|
944
|
+
notes.push(
|
|
945
|
+
"Each point carries `start`: null for INSTANT (balance-sheet) concepts and the ISO period-start for DURATION (flow) concepts. Period identity is the (start,end) PAIR — a same-`end` different-`start` pair is a different-duration fact (a 3-month quarter vs the 12-month year), NOT a revision.",
|
|
946
|
+
);
|
|
947
|
+
if (sameEndMultiDuration.length) {
|
|
948
|
+
notes.push(
|
|
949
|
+
`Concept(s) with MULTIPLE durations sharing the same period-end (a shorter quarter AND a longer YTD/annual figure coexist under one \`end\`): ${sameEndMultiDuration.join(", ")}. Each point's \`start\` distinguishes them — do NOT compare or sum values across differing \`start\` (e.g. never read a Q4 figure as the full year) without checking \`start\`.`,
|
|
950
|
+
);
|
|
951
|
+
}
|
|
952
|
+
notes.push(
|
|
953
|
+
latest
|
|
954
|
+
? "latest=true: each concept reduced to its single most-recent data point (by period end). The selected point retains its `start`, so a duration value is never presented unqualified — a same-`end` shorter/longer period may have been dropped (see the (start,end) note)."
|
|
955
|
+
: "Full reported time series per concept (curated to the requested concepts only; companyfacts holds ~hundreds of concepts).",
|
|
956
|
+
);
|
|
957
|
+
|
|
958
|
+
// A definitive curated extraction of the requested concepts (no upstream
|
|
959
|
+
// pagination) — complete for the concepts that ARE present in this unit.
|
|
960
|
+
return withMeta(
|
|
961
|
+
{ cik, entityName: str(doc.entityName), unit, concepts, absent, wrongUnit },
|
|
962
|
+
edgarMeta({
|
|
963
|
+
returned: concepts.length,
|
|
964
|
+
totalAvailable: concepts.length,
|
|
965
|
+
complete: true,
|
|
966
|
+
filtersApplied: latest ? ["concepts", "unit", "latest"] : ["concepts", "unit"],
|
|
967
|
+
notes,
|
|
968
|
+
}),
|
|
969
|
+
);
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
// ─── Tool 4: edgar_full_text_search ───────────────────────────────
|
|
973
|
+
export type FtsResult = {
|
|
974
|
+
accession: string | null;
|
|
975
|
+
form: string | null;
|
|
976
|
+
filingDate: string | null;
|
|
977
|
+
entityNames: string[];
|
|
978
|
+
ciks: string[];
|
|
979
|
+
filingIndexUrl: string | null;
|
|
980
|
+
};
|
|
981
|
+
|
|
982
|
+
/**
|
|
983
|
+
* Map one efts hit → a stable output row. F7: `_source` has NO primary-document
|
|
984
|
+
* filename, so `filingIndexUrl` is the filing's ARCHIVE INDEX DIRECTORY built
|
|
985
|
+
* from `adsh` (accession) + the first CIK — a real, resolvable URL, with NO
|
|
986
|
+
* fabricated document filename appended.
|
|
987
|
+
*/
|
|
988
|
+
function mapFtsHit(hit: {
|
|
989
|
+
_id?: string;
|
|
990
|
+
_source?: {
|
|
991
|
+
ciks?: string[];
|
|
992
|
+
display_names?: string[];
|
|
993
|
+
form?: string;
|
|
994
|
+
file_date?: string;
|
|
995
|
+
adsh?: string;
|
|
996
|
+
};
|
|
997
|
+
}): FtsResult {
|
|
998
|
+
const src = hit._source ?? {};
|
|
999
|
+
const ciks = Array.isArray(src.ciks) ? src.ciks.map(String) : [];
|
|
1000
|
+
const adsh =
|
|
1001
|
+
str(src.adsh) ??
|
|
1002
|
+
(typeof hit._id === "string" ? str(hit._id.split(":")[0]) : null);
|
|
1003
|
+
let filingIndexUrl: string | null = null;
|
|
1004
|
+
if (adsh && ciks[0]) {
|
|
1005
|
+
filingIndexUrl = `${ARCHIVES_BASE}/${unpadCik(padCik(ciks[0]))}/${adsh.replace(/-/g, "")}/`;
|
|
1006
|
+
}
|
|
1007
|
+
return {
|
|
1008
|
+
accession: adsh,
|
|
1009
|
+
form: str(src.form),
|
|
1010
|
+
filingDate: str(src.file_date),
|
|
1011
|
+
entityNames: Array.isArray(src.display_names) ? src.display_names.map(String) : [],
|
|
1012
|
+
ciks,
|
|
1013
|
+
filingIndexUrl,
|
|
1014
|
+
};
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
const FTS_PAGE_SIZE = 100; // F2 — efts ignores `size`; it is fixed at 100/page.
|
|
1018
|
+
const FTS_WINDOW = 10_000; // upstream hard cap: from + 100 must be ≤ 10000.
|
|
1019
|
+
const FTS_MAX_FROM = FTS_WINDOW - FTS_PAGE_SIZE; // 9900 → from > 9900 is invalid (9900 is the valid final page).
|
|
1020
|
+
|
|
1021
|
+
/**
|
|
1022
|
+
* Full-text search across EDGAR filings (2001-present). F2 — NO `size` param
|
|
1023
|
+
* (efts always returns 100/page); pagination is by `from`. F3 — `from > 9900`
|
|
1024
|
+
* is rejected as invalid_input BEFORE the fetch (from+100 would exceed the 10000
|
|
1025
|
+
* window, which efts answers with HTTP 200 + an error body; from=9900 itself is a
|
|
1026
|
+
* VALID final page), and a response missing `hits.hits` is thrown as schema_drift
|
|
1027
|
+
* (never crashed on). F5 — `hits.total.relation === "gte"` (true total unknown,
|
|
1028
|
+
* ≥10000) surfaces as `totalIsLowerBound:true`. ADR-0018 — optional `ciks` (pin
|
|
1029
|
+
* filings BY entities, exact 10-digit CIK) + `entityName` (fuzzy filer-name)
|
|
1030
|
+
* narrowing filters; a no-digit/CIK-0 `ciks` entry is rejected pre-fetch (M1).
|
|
1031
|
+
*/
|
|
1032
|
+
export async function fullTextSearch(args: {
|
|
1033
|
+
q: string;
|
|
1034
|
+
forms?: string[];
|
|
1035
|
+
startdt?: string;
|
|
1036
|
+
enddt?: string;
|
|
1037
|
+
ciks?: string[];
|
|
1038
|
+
entityName?: string;
|
|
1039
|
+
from?: number;
|
|
1040
|
+
}): Promise<MetaBundle> {
|
|
1041
|
+
const from = args.from ?? 0;
|
|
1042
|
+
// Off-by-one fix (ADR-0018): `from=9900` is a VALID final page (from+100=10000,
|
|
1043
|
+
// the exact ES window boundary — live-confirmed). Only `from > 9900` overflows.
|
|
1044
|
+
if (from > FTS_MAX_FROM) {
|
|
1045
|
+
throw new ToolErrorCarrier({
|
|
1046
|
+
kind: "invalid_input",
|
|
1047
|
+
message: `EDGAR full-text search 'from' (${from}) is out of range: the paging window is from+100 ≤ ${FTS_WINDOW}, so 'from' must be <= ${FTS_MAX_FROM}. Narrow the query with forms/startdt/enddt instead of paging past ${FTS_WINDOW} results.`,
|
|
1048
|
+
retryable: false,
|
|
1049
|
+
upstreamEndpoint: "edgar:fts",
|
|
1050
|
+
});
|
|
1051
|
+
}
|
|
1052
|
+
|
|
1053
|
+
// ── Entity filters (ADR-0018) — normalize BEFORE any fetch. ──
|
|
1054
|
+
// M1 (fake-empty landmine): each `ciks` entry is reduced to digits; an entry
|
|
1055
|
+
// that yields NO digits (a ticker/garbage like "AAPL"/"../") OR an all-zeros
|
|
1056
|
+
// value ("0"/"0000000000") pads to the NON-EXISTENT CIK 0000000000, which efts
|
|
1057
|
+
// answers with a LIVE 0/eq fake-empty. Reject BOTH explicitly here (0 fetch) —
|
|
1058
|
+
// NEVER `.map(padCik).filter(Boolean)` (padCik("AAPL")="0000000000" is truthy).
|
|
1059
|
+
// Whitespace-only/empty entries are silently dropped (probe 10 — an empty CIK
|
|
1060
|
+
// is ignored upstream anyway).
|
|
1061
|
+
const ciks: string[] = [];
|
|
1062
|
+
for (const raw of args.ciks ?? []) {
|
|
1063
|
+
const s = raw == null ? "" : String(raw).trim();
|
|
1064
|
+
if (s === "") continue; // drop empty/whitespace entries
|
|
1065
|
+
const digits = s.replace(/\D/g, "");
|
|
1066
|
+
if (digits.replace(/0/g, "") === "") {
|
|
1067
|
+
throw new ToolErrorCarrier({
|
|
1068
|
+
kind: "invalid_input",
|
|
1069
|
+
message: `EDGAR full-text search 'ciks' entry ${JSON.stringify(raw)} is not a valid SEC CIK: 'ciks' takes NUMERIC CIKs only (each is zero-padded to 10 digits; CIK 0 / a ticker / a company name is rejected — it would silently return 0 matches). For a ticker or company name, use 'entityName' or resolve the CIK first with edgar_lookup_cik (by name/ticker).`,
|
|
1070
|
+
retryable: false,
|
|
1071
|
+
upstreamEndpoint: "edgar:fts",
|
|
1072
|
+
});
|
|
1073
|
+
}
|
|
1074
|
+
ciks.push(digits.padStart(10, "0"));
|
|
1075
|
+
}
|
|
1076
|
+
const entityName = args.entityName?.trim() ?? "";
|
|
1077
|
+
|
|
1078
|
+
const params = new URLSearchParams();
|
|
1079
|
+
params.set("q", args.q);
|
|
1080
|
+
const forms = args.forms?.map((f) => f.trim()).filter(Boolean);
|
|
1081
|
+
if (forms && forms.length) params.set("forms", forms.join(","));
|
|
1082
|
+
if (args.startdt || args.enddt) {
|
|
1083
|
+
params.set("dateRange", "custom");
|
|
1084
|
+
if (args.startdt) params.set("startdt", args.startdt);
|
|
1085
|
+
if (args.enddt) params.set("enddt", args.enddt);
|
|
1086
|
+
}
|
|
1087
|
+
// Whitelisted entity filters — set (and echo in filtersApplied) ONLY when a
|
|
1088
|
+
// live-honored value was actually sent (silent-drop guard; probe 7).
|
|
1089
|
+
if (ciks.length) params.set("ciks", ciks.join(","));
|
|
1090
|
+
if (entityName) params.set("entityName", entityName);
|
|
1091
|
+
if (from > 0) params.set("from", String(from));
|
|
1092
|
+
|
|
1093
|
+
const r = await getEdgar(`${FTS_URL}?${params.toString()}`, "edgar:fts");
|
|
1094
|
+
const d = (await r.json()) as {
|
|
1095
|
+
hits?: { total?: { value?: number; relation?: string }; hits?: unknown[] };
|
|
1096
|
+
};
|
|
1097
|
+
// F3 — window overflow / any error body arrives as HTTP 200 with no hits.hits.
|
|
1098
|
+
if (!d || !d.hits || !Array.isArray(d.hits.hits)) {
|
|
1099
|
+
throw new ToolErrorCarrier({
|
|
1100
|
+
kind: "schema_drift",
|
|
1101
|
+
message:
|
|
1102
|
+
"edgar:fts returned HTTP 200 without hits.hits — SEC answers a paging-window overflow (or a malformed query) with 200 + an error body. Narrow the query or reduce 'from'.",
|
|
1103
|
+
retryable: false,
|
|
1104
|
+
upstreamEndpoint: "edgar:fts",
|
|
1105
|
+
});
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
const total = d.hits.total ?? {};
|
|
1109
|
+
const totalAvailable = typeof total.value === "number" ? total.value : 0;
|
|
1110
|
+
const isLowerBound = total.relation === "gte";
|
|
1111
|
+
const results = (d.hits.hits as Parameters<typeof mapFtsHit>[0][]).map(mapFtsHit);
|
|
1112
|
+
const returned = results.length;
|
|
1113
|
+
|
|
1114
|
+
const nextFrom = from + FTS_PAGE_SIZE;
|
|
1115
|
+
// More results exist AND the next page is still inside the 10000 window.
|
|
1116
|
+
const hasMore = from + returned < totalAvailable && nextFrom < FTS_WINDOW;
|
|
1117
|
+
// The next page is only reachable if it does not trip the `from > 9900` guard.
|
|
1118
|
+
// Off-by-one fix (ADR-0018): `<=` so pagination can reach the final `from=9900`
|
|
1119
|
+
// page (nextFrom=9900) instead of stopping one page short.
|
|
1120
|
+
const nextOffset = hasMore && nextFrom <= FTS_MAX_FROM ? nextFrom : null;
|
|
1121
|
+
|
|
1122
|
+
const filtersApplied: string[] = ["q"];
|
|
1123
|
+
if (forms && forms.length) filtersApplied.push("forms");
|
|
1124
|
+
if (args.startdt || args.enddt) filtersApplied.push("dateRange");
|
|
1125
|
+
if (ciks.length) filtersApplied.push("ciks");
|
|
1126
|
+
if (entityName) filtersApplied.push("entityName");
|
|
1127
|
+
|
|
1128
|
+
const notes: string[] = [
|
|
1129
|
+
"EDGAR full-text search covers 2001-present only (earlier filings are not indexed).",
|
|
1130
|
+
"Page size is fixed at 100 (the efts `size` param is ignored); paginate via `from`.",
|
|
1131
|
+
];
|
|
1132
|
+
// Entity-filter semantics (ADR-0018) — emitted only when the filter was applied.
|
|
1133
|
+
if (ciks.length) {
|
|
1134
|
+
notes.push(
|
|
1135
|
+
"Results are pinned to the supplied CIK(s) (exact 10-digit match). A 0-result set may mean the CIK is wrong OR the entity has no matching filings in the query/form/date window — it is NOT proof of absence; verify the CIK via edgar_lookup_cik (by name/ticker) or on SEC EDGAR directly.",
|
|
1136
|
+
);
|
|
1137
|
+
}
|
|
1138
|
+
if (entityName) {
|
|
1139
|
+
notes.push(
|
|
1140
|
+
"entityName is a FUZZY filer-name filter (it can match related entities, e.g. multiple 'Apple*' filers) — it is NOT a CIK-exact pin; combine with ciks for an exact-entity result.",
|
|
1141
|
+
);
|
|
1142
|
+
}
|
|
1143
|
+
if (returned === 0 && (ciks.length || entityName)) {
|
|
1144
|
+
notes.push(
|
|
1145
|
+
"0 results with ciks/entityName applied is NOT proof of absence — a wrong CIK, a too-fuzzy/mismatched entityName, or a genuine no-match are indistinguishable here; verify the identifier via edgar_lookup_cik (by name/ticker) or on SEC EDGAR directly.",
|
|
1146
|
+
);
|
|
1147
|
+
}
|
|
1148
|
+
if (isLowerBound) {
|
|
1149
|
+
notes.push(
|
|
1150
|
+
`totalAvailable is a LOWER BOUND: SEC reported hits.total.relation="gte" with the value pinned at ${totalAvailable}; the true match count is UNKNOWN and ≥ ${totalAvailable}. See totalIsLowerBound. Narrow the query for an exact count.`,
|
|
1151
|
+
);
|
|
1152
|
+
}
|
|
1153
|
+
if (hasMore && nextOffset === null) {
|
|
1154
|
+
notes.push(
|
|
1155
|
+
`More matches exist but are beyond EDGAR's ${FTS_WINDOW}-result full-text window and are UNREACHABLE via paging — narrow the query with forms/date filters to retrieve them.`,
|
|
1156
|
+
);
|
|
1157
|
+
}
|
|
1158
|
+
const meta: Partial<ResponseMeta> = {
|
|
1159
|
+
returned,
|
|
1160
|
+
totalAvailable,
|
|
1161
|
+
filtersApplied,
|
|
1162
|
+
pagination: { offset: from, limit: FTS_PAGE_SIZE, hasMore, nextOffset },
|
|
1163
|
+
notes,
|
|
1164
|
+
};
|
|
1165
|
+
if (isLowerBound) meta.totalIsLowerBound = true;
|
|
1166
|
+
|
|
1167
|
+
return withMeta({ query: args.q, results }, edgarMeta(meta));
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
// ─── Tool 5: edgar_xbrl_frames ────────────────────────────────────
|
|
1171
|
+
// ADR-0017 (v1 + v2 AUTHORITATIVE). A COMPLETE keyless cross-filer cross-section:
|
|
1172
|
+
// every XBRL filer's reported value for ONE concept in ONE calendar period, in a
|
|
1173
|
+
// single call — the peer-benchmarking / distribution primitive. Reuses getEdgar
|
|
1174
|
+
// VERBATIM (no new host/gate/UA); the frames path puts caller-supplied segments
|
|
1175
|
+
// (taxonomy/tag/unit/period) as RAW PATH SEGMENTS, so the load-bearing control is
|
|
1176
|
+
// the pre-fetch enum/regex validation + fixed-host assertion (§SSRF, S1/S2).
|
|
1177
|
+
|
|
1178
|
+
/** The whitelisted frames host + base (same data.sec.gov the other edgar tools use). */
|
|
1179
|
+
const FRAMES_BASE = "https://data.sec.gov/api/xbrl/frames";
|
|
1180
|
+
|
|
1181
|
+
/**
|
|
1182
|
+
* The `taxonomy` path-segment enum — the SSRF guard for that segment (no free
|
|
1183
|
+
* value reaches the host). Only members LIVE-CONFIRMED to resolve a real frame
|
|
1184
|
+
* (per-segment live-verify discipline, ADR-0017 Open-Q5) are shipped. Maker
|
|
1185
|
+
* probes 2026-07-12: us-gaap (Assets/Revenues/NetIncomeLoss/EPS) + dei
|
|
1186
|
+
* (EntityCommonStockSharesOutstanding/EntityPublicFloat) → 200; the guessed
|
|
1187
|
+
* srt/invest/us-ins tags → 404, so they are DROPPED (conservative floor).
|
|
1188
|
+
*/
|
|
1189
|
+
export const FRAMES_TAXONOMIES = ["us-gaap", "dei"] as const;
|
|
1190
|
+
|
|
1191
|
+
// Segment grammars (belt-and-suspenders re-check in the builder — S1). These are
|
|
1192
|
+
// the SAME regexes/enum the server Zod schema applies; re-running them here does
|
|
1193
|
+
// NOT rely on Zod (a direct call could bypass it) nor on the hostname assertion
|
|
1194
|
+
// alone (a same-host `../` traversal normalizes to host=data.sec.gov and PASSES a
|
|
1195
|
+
// hostname check). They are allowlists: `%`, `/`, `.`, `\`, `..`, `%2F`, `%2E`,
|
|
1196
|
+
// `%00` all fail (S2 — those characters are simply not in the allowed classes).
|
|
1197
|
+
const FRAMES_TAG_RE = /^[A-Za-z0-9]+$/;
|
|
1198
|
+
const FRAMES_UNIT_RE = /^[A-Za-z0-9-]+$/;
|
|
1199
|
+
const FRAMES_PERIOD_RE = /^CY\d{4}(Q[1-4]I?)?$/;
|
|
1200
|
+
|
|
1201
|
+
/** Throw the pre-fetch injection-guard error (invalid_input, 0 fetch). */
|
|
1202
|
+
function framesInvalid(message: string): never {
|
|
1203
|
+
throw new ToolErrorCarrier({
|
|
1204
|
+
kind: "invalid_input",
|
|
1205
|
+
message,
|
|
1206
|
+
retryable: false,
|
|
1207
|
+
upstreamEndpoint: "edgar:frames",
|
|
1208
|
+
});
|
|
1209
|
+
}
|
|
1210
|
+
|
|
1211
|
+
/**
|
|
1212
|
+
* Build the frames URL from validated path segments (S1/S2). BELT-AND-SUSPENDERS:
|
|
1213
|
+
* re-run the enum + the three regexes on each segment and hard-throw invalid_input
|
|
1214
|
+
* (0 fetch) on any mismatch — do NOT trust that Zod already ran, and do NOT rely on
|
|
1215
|
+
* the hostname assertion alone (it passes same-host traversal). THEN assert the
|
|
1216
|
+
* built URL is https on the fixed data.sec.gov host (guards host-escape/downgrade).
|
|
1217
|
+
*/
|
|
1218
|
+
function buildFramesUrl(
|
|
1219
|
+
taxonomy: string,
|
|
1220
|
+
tag: string,
|
|
1221
|
+
unit: string,
|
|
1222
|
+
period: string,
|
|
1223
|
+
): string {
|
|
1224
|
+
if (!(FRAMES_TAXONOMIES as readonly string[]).includes(taxonomy)) {
|
|
1225
|
+
framesInvalid(
|
|
1226
|
+
`edgar_xbrl_frames: taxonomy ${JSON.stringify(taxonomy)} is not one of {${FRAMES_TAXONOMIES.join(", ")}} — refused before any fetch (path-segment injection guard).`,
|
|
1227
|
+
);
|
|
1228
|
+
}
|
|
1229
|
+
if (!FRAMES_TAG_RE.test(tag)) {
|
|
1230
|
+
framesInvalid(
|
|
1231
|
+
`edgar_xbrl_frames: tag ${JSON.stringify(tag)} must match ^[A-Za-z0-9]+$ (XBRL tags are alphanumeric; slash/dot/percent/backslash/'..' are rejected) — refused before any fetch (path-segment injection guard).`,
|
|
1232
|
+
);
|
|
1233
|
+
}
|
|
1234
|
+
if (!FRAMES_UNIT_RE.test(unit)) {
|
|
1235
|
+
framesInvalid(
|
|
1236
|
+
`edgar_xbrl_frames: unit ${JSON.stringify(unit)} must match ^[A-Za-z0-9-]+$ (hyphen allowed, e.g. USD-per-shares; slash/dot/percent forbidden — never the 'USD/shares' companyfacts key form) — refused before any fetch (path-segment injection guard).`,
|
|
1237
|
+
);
|
|
1238
|
+
}
|
|
1239
|
+
if (!FRAMES_PERIOD_RE.test(period)) {
|
|
1240
|
+
framesInvalid(
|
|
1241
|
+
`edgar_xbrl_frames: period ${JSON.stringify(period)} must match ^CY\\d{4}(Q[1-4]I?)?$ (e.g. CY2023, CY2023Q1, CY2023Q4I) — refused before any fetch (path-segment injection guard).`,
|
|
1242
|
+
);
|
|
1243
|
+
}
|
|
1244
|
+
const built = `${FRAMES_BASE}/${taxonomy}/${tag}/${unit}/${period}.json`;
|
|
1245
|
+
let parsed: URL;
|
|
1246
|
+
try {
|
|
1247
|
+
parsed = new URL(built);
|
|
1248
|
+
} catch {
|
|
1249
|
+
framesInvalid(
|
|
1250
|
+
`edgar_xbrl_frames: could not construct a valid URL from the segments — refused before any fetch.`,
|
|
1251
|
+
);
|
|
1252
|
+
}
|
|
1253
|
+
if (parsed.protocol !== "https:" || parsed.hostname !== "data.sec.gov") {
|
|
1254
|
+
framesInvalid(
|
|
1255
|
+
`edgar_xbrl_frames: constructed URL host/scheme is not https://data.sec.gov (${parsed.protocol}//${parsed.hostname}) — refused before any fetch (fixed-host assertion).`,
|
|
1256
|
+
);
|
|
1257
|
+
}
|
|
1258
|
+
return built;
|
|
1259
|
+
}
|
|
1260
|
+
|
|
1261
|
+
/** One filer's row in a frame. `start` is present ONLY for duration concepts. */
|
|
1262
|
+
export type FrameRow = {
|
|
1263
|
+
accn: string | null;
|
|
1264
|
+
cik: string | null;
|
|
1265
|
+
entityName: string | null;
|
|
1266
|
+
loc: string | null;
|
|
1267
|
+
end: string | null;
|
|
1268
|
+
val: number | null;
|
|
1269
|
+
start?: string | null;
|
|
1270
|
+
};
|
|
1271
|
+
|
|
1272
|
+
/**
|
|
1273
|
+
* Summary distribution over the FULL cross-section (M2/M3). Computed over the
|
|
1274
|
+
* FINITE `val`s only (num() → null for absent/blank/"null"/non-finite, which are
|
|
1275
|
+
* excluded and counted in `nonFiniteExcluded`; a real 0 survives). Percentiles use
|
|
1276
|
+
* LINEAR INTERPOLATION on the ascending-sorted finite vals. `count===0` (no finite
|
|
1277
|
+
* vals) ⇒ EVERY stat field null (never 0/NaN/Infinity — the null-never-0 row rule
|
|
1278
|
+
* lifted onto the aggregate).
|
|
1279
|
+
*/
|
|
1280
|
+
export type FrameStats = {
|
|
1281
|
+
count: number;
|
|
1282
|
+
min: number | null;
|
|
1283
|
+
max: number | null;
|
|
1284
|
+
sum: number | null;
|
|
1285
|
+
mean: number | null;
|
|
1286
|
+
median: number | null;
|
|
1287
|
+
p25: number | null;
|
|
1288
|
+
p75: number | null;
|
|
1289
|
+
nonFiniteExcluded: number;
|
|
1290
|
+
};
|
|
1291
|
+
|
|
1292
|
+
/** Compute FrameStats over the full data[] (all rows, before any client slice). */
|
|
1293
|
+
function framesStats(data: Array<{ val?: unknown }>): FrameStats {
|
|
1294
|
+
const finite: number[] = [];
|
|
1295
|
+
let nonFiniteExcluded = 0;
|
|
1296
|
+
for (const d of data) {
|
|
1297
|
+
const v = num(d.val); // null for absent/""/"null"/non-finite; real 0 survives.
|
|
1298
|
+
if (v === null) nonFiniteExcluded++;
|
|
1299
|
+
else finite.push(v);
|
|
1300
|
+
}
|
|
1301
|
+
const count = finite.length;
|
|
1302
|
+
// M3 — no finite values ⇒ no distribution computable ⇒ every field null.
|
|
1303
|
+
if (count === 0) {
|
|
1304
|
+
return {
|
|
1305
|
+
count: 0,
|
|
1306
|
+
min: null,
|
|
1307
|
+
max: null,
|
|
1308
|
+
sum: null,
|
|
1309
|
+
mean: null,
|
|
1310
|
+
median: null,
|
|
1311
|
+
p25: null,
|
|
1312
|
+
p75: null,
|
|
1313
|
+
nonFiniteExcluded,
|
|
1314
|
+
};
|
|
1315
|
+
}
|
|
1316
|
+
finite.sort((a, b) => a - b);
|
|
1317
|
+
const sum = finite.reduce((s, v) => s + v, 0);
|
|
1318
|
+
// M2 — linear interpolation: pos=q*(n-1), interpolate between the bracketing
|
|
1319
|
+
// sorted values. median = q=0.5 (mean of the two middle values for even n).
|
|
1320
|
+
const quantile = (q: number): number => {
|
|
1321
|
+
const pos = q * (count - 1);
|
|
1322
|
+
const lo = Math.floor(pos);
|
|
1323
|
+
const hi = Math.ceil(pos);
|
|
1324
|
+
const vLo = finite[lo] as number;
|
|
1325
|
+
const vHi = finite[hi] as number;
|
|
1326
|
+
return pos === lo ? vLo : vLo + (pos - lo) * (vHi - vLo);
|
|
1327
|
+
};
|
|
1328
|
+
return {
|
|
1329
|
+
count,
|
|
1330
|
+
min: finite[0] as number,
|
|
1331
|
+
max: finite[count - 1] as number,
|
|
1332
|
+
sum,
|
|
1333
|
+
mean: sum / count,
|
|
1334
|
+
median: quantile(0.5),
|
|
1335
|
+
p25: quantile(0.25),
|
|
1336
|
+
p75: quantile(0.75),
|
|
1337
|
+
nonFiniteExcluded,
|
|
1338
|
+
};
|
|
1339
|
+
}
|
|
1340
|
+
|
|
1341
|
+
/**
|
|
1342
|
+
* Keyless cross-filer XBRL cross-section. In ONE call, return every filer's
|
|
1343
|
+
* reported value for a single us-gaap/dei concept in a single calendar period —
|
|
1344
|
+
* the complete cross-section — for peer benchmarking + distribution stats.
|
|
1345
|
+
*
|
|
1346
|
+
* HONESTY (ADR-0017 v2):
|
|
1347
|
+
* - `totalAvailable` = SEC's own `pts` (NEVER a page length). Drift guards THROW
|
|
1348
|
+
* schema_drift on a non-frames shape (data not array / pts non-numeric) or a
|
|
1349
|
+
* `pts !== data.length` mismatch (a truncation frames has no way to page past,
|
|
1350
|
+
* so refusing is the honest move — Open-Q2 resolved: THROW is the DEFAULT).
|
|
1351
|
+
* - The upstream frame is fetched in FULL; `limit`/`offset` is a CLIENT-SIDE
|
|
1352
|
+
* window disclosed as such (M1 — the completeness note never calls a subset
|
|
1353
|
+
* page "complete"; buildMeta derives complete/truncated from returned/total/
|
|
1354
|
+
* hasMore, mirroring edgar_company_filings — NO forced complete:true).
|
|
1355
|
+
* - A 404 (tag/unit/period/taxonomy quadruple did not match) ⇒ honest found:false
|
|
1356
|
+
* with the semantic note (absence ≠ 0). NEVER a fabricated val:0.
|
|
1357
|
+
* - Row `val` is num()-coerced (null-never-0). `start` appears only on duration
|
|
1358
|
+
* rows. `uom` echoes SEC's OWN unit (e.g. requested 'USD-per-shares' ⇒ 'USD/shares').
|
|
1359
|
+
* - `includeStats` computes over the FULL data[] (all rows, before the slice).
|
|
1360
|
+
*/
|
|
1361
|
+
export async function xbrlFrames(args: {
|
|
1362
|
+
tag: string;
|
|
1363
|
+
period: string;
|
|
1364
|
+
taxonomy?: string;
|
|
1365
|
+
unit?: string;
|
|
1366
|
+
limit?: number;
|
|
1367
|
+
offset?: number;
|
|
1368
|
+
includeStats?: boolean;
|
|
1369
|
+
}): Promise<MetaBundle> {
|
|
1370
|
+
const taxonomy = args.taxonomy ?? "us-gaap";
|
|
1371
|
+
const unit = args.unit ?? "USD";
|
|
1372
|
+
const limit = args.limit ?? 100;
|
|
1373
|
+
const offset = args.offset ?? 0;
|
|
1374
|
+
const includeStats = args.includeStats ?? false;
|
|
1375
|
+
|
|
1376
|
+
// S1/S2 — build (and re-validate) the path BEFORE any fetch. Throws
|
|
1377
|
+
// invalid_input with 0 fetches on any bad segment or a non-fixed-host URL.
|
|
1378
|
+
const url = buildFramesUrl(taxonomy, args.tag, unit, args.period);
|
|
1379
|
+
|
|
1380
|
+
type FramesBody = {
|
|
1381
|
+
tag?: unknown;
|
|
1382
|
+
uom?: unknown;
|
|
1383
|
+
label?: unknown;
|
|
1384
|
+
description?: unknown;
|
|
1385
|
+
pts?: unknown;
|
|
1386
|
+
data?: unknown;
|
|
1387
|
+
};
|
|
1388
|
+
let body: FramesBody;
|
|
1389
|
+
try {
|
|
1390
|
+
const r = await getEdgar(url, "edgar:frames");
|
|
1391
|
+
body = (await r.json()) as FramesBody;
|
|
1392
|
+
} catch (e) {
|
|
1393
|
+
// 404 ⇒ the quadruple did not match a frame ⇒ honest found:false (NEVER 0).
|
|
1394
|
+
if (e instanceof ToolErrorCarrier && e.toolError.kind === "not_found") {
|
|
1395
|
+
return notFoundBundle(
|
|
1396
|
+
`${taxonomy}/${args.tag}/${unit}/${args.period}`,
|
|
1397
|
+
`No XBRL frame matched taxonomy=${taxonomy} tag=${args.tag} unit=${unit} period=${args.period} (HTTP 404). The concept was NOT reported under that exact tag/unit/calendar-frame — this is NOT a value of 0. Check: instant (balance-sheet) concepts need the trailing 'I' (e.g. CY2023Q4I); EPS uses unit 'USD-per-shares' (hyphen), never 'USD/shares'.`,
|
|
1398
|
+
);
|
|
1399
|
+
}
|
|
1400
|
+
throw e;
|
|
1401
|
+
}
|
|
1402
|
+
|
|
1403
|
+
// Drift guards (schema_drift THROW — never a fabricated empty). Order matters:
|
|
1404
|
+
// (1) data[] must be an array, (2) pts must be a finite number, (3) the
|
|
1405
|
+
// load-bearing invariant pts === data.length (a mismatch ⇒ truncation/shape
|
|
1406
|
+
// change; frames has NO page param to recover the rest, so we refuse).
|
|
1407
|
+
const data = body.data;
|
|
1408
|
+
if (!Array.isArray(data)) {
|
|
1409
|
+
throw new ToolErrorCarrier({
|
|
1410
|
+
kind: "schema_drift",
|
|
1411
|
+
message: `edgar:frames returned HTTP 200 without a data[] array (taxonomy=${taxonomy} tag=${args.tag} unit=${unit} period=${args.period}) — the frames envelope changed.`,
|
|
1412
|
+
retryable: false,
|
|
1413
|
+
upstreamEndpoint: "edgar:frames",
|
|
1414
|
+
});
|
|
1415
|
+
}
|
|
1416
|
+
if (typeof body.pts !== "number" || !Number.isFinite(body.pts)) {
|
|
1417
|
+
throw new ToolErrorCarrier({
|
|
1418
|
+
kind: "schema_drift",
|
|
1419
|
+
message: `edgar:frames returned a non-numeric 'pts' (${JSON.stringify(body.pts)}) — SEC's own data-point count is missing; the envelope changed.`,
|
|
1420
|
+
retryable: false,
|
|
1421
|
+
upstreamEndpoint: "edgar:frames",
|
|
1422
|
+
});
|
|
1423
|
+
}
|
|
1424
|
+
if (body.pts !== data.length) {
|
|
1425
|
+
throw new ToolErrorCarrier({
|
|
1426
|
+
kind: "schema_drift",
|
|
1427
|
+
message: `SEC frames pts (${body.pts}) ≠ data.length (${data.length}) — the cross-section may be truncated or the envelope changed. Frames has no pagination to recover the rest, so refusing rather than presenting a partial set as complete.`,
|
|
1428
|
+
retryable: false,
|
|
1429
|
+
upstreamEndpoint: "edgar:frames",
|
|
1430
|
+
});
|
|
1431
|
+
}
|
|
1432
|
+
// SEC's own count; validated === data.length. NEVER a page length (mutation a).
|
|
1433
|
+
const totalAvailable = body.pts;
|
|
1434
|
+
|
|
1435
|
+
// Stats over the FULL data[] (ALL rows, BEFORE the client-side slice) — M2/M3.
|
|
1436
|
+
const stats = includeStats
|
|
1437
|
+
? framesStats(data as Array<{ val?: unknown }>)
|
|
1438
|
+
: undefined;
|
|
1439
|
+
|
|
1440
|
+
// Map every row: null-never-0 via num() on val; start only on duration rows.
|
|
1441
|
+
const allRows: FrameRow[] = (data as Array<Record<string, unknown>>).map((d) => {
|
|
1442
|
+
const row: FrameRow = {
|
|
1443
|
+
accn: str(d.accn),
|
|
1444
|
+
cik: d.cik == null ? null : padCik(d.cik as string | number),
|
|
1445
|
+
entityName: str(d.entityName),
|
|
1446
|
+
loc: str(d.loc),
|
|
1447
|
+
end: str(d.end),
|
|
1448
|
+
val: num(d.val),
|
|
1449
|
+
};
|
|
1450
|
+
if ("start" in d) row.start = str(d.start);
|
|
1451
|
+
return row;
|
|
1452
|
+
});
|
|
1453
|
+
|
|
1454
|
+
// Client-side window over the already-fully-fetched cross-section (M1).
|
|
1455
|
+
const page = allRows.slice(offset, offset + limit);
|
|
1456
|
+
const returned = page.length;
|
|
1457
|
+
const hasMore = offset + returned < totalAvailable;
|
|
1458
|
+
const uom = str(body.uom);
|
|
1459
|
+
const found = data.length > 0;
|
|
1460
|
+
|
|
1461
|
+
const notes: string[] = [];
|
|
1462
|
+
// SEMANTIC (the CIK-caveat analogue) — absence ≠ 0.
|
|
1463
|
+
notes.push(
|
|
1464
|
+
"A frame contains ONLY filers who reported this EXACT tag for this EXACT calendar period, with SEC selecting one best-fit fact per entity. Absence from the frame ≠ 0 and ≠ 'the company has none' — it means the filer did not report under that tag/calendar-frame.",
|
|
1465
|
+
);
|
|
1466
|
+
// COMPLETENESS — M1 two-clause split (the word "complete" never describes a
|
|
1467
|
+
// subset page): the upstream-frame-fetched-in-full sense and the this-page-is-a-
|
|
1468
|
+
// subset sense are stated distinctly.
|
|
1469
|
+
notes.push(
|
|
1470
|
+
`The upstream frame was fetched in FULL server-side (pts=${totalAvailable} = the entire cross-section for this taxonomy/tag/unit/period, uncapped and unpaginated by SEC). THIS response page contains ${returned} of ${totalAvailable} rows; when hasMore is true, page via nextOffset to retrieve the remaining filers.`,
|
|
1471
|
+
);
|
|
1472
|
+
// DISCOVERABILITY — period grammar + hyphen unit + 404-not-zero.
|
|
1473
|
+
notes.push(
|
|
1474
|
+
"Period grammar: CY2023 (annual flow) · CY2023Q1 (quarterly flow, no I) · CY2023Q4I (instant, trailing I). EPS uses unit 'USD-per-shares' (hyphen), never 'USD/shares'. A tag/unit/period mismatch returns found:false (a 404), NOT an empty zero.",
|
|
1475
|
+
);
|
|
1476
|
+
// val null-never-0.
|
|
1477
|
+
notes.push(
|
|
1478
|
+
"Each row 'val' is coerced to number-or-null: a real reported 0 survives as 0, but an absent/blank/non-finite value is null — never a fabricated 0.",
|
|
1479
|
+
);
|
|
1480
|
+
// uom echo (minor) — path-form vs response-form.
|
|
1481
|
+
if (uom && uom !== unit) {
|
|
1482
|
+
notes.push(
|
|
1483
|
+
`SEC's response uom is '${uom}', which differs from the requested unit path-segment '${unit}' (path-form vs response-form — e.g. 'USD-per-shares' is reported as 'USD/shares'). The 'uom' field echoes SEC's own value.`,
|
|
1484
|
+
);
|
|
1485
|
+
}
|
|
1486
|
+
// Stats disclosure — M2 method / M3 no-finite.
|
|
1487
|
+
if (stats) {
|
|
1488
|
+
notes.push(
|
|
1489
|
+
stats.count === 0
|
|
1490
|
+
? `stats: no finite values across the full cross-section (nonFiniteExcluded=${stats.nonFiniteExcluded}) — no distribution computable, so every stat field (min/max/sum/mean/median/p25/p75) is null, never 0/NaN/Infinity.`
|
|
1491
|
+
: `stats: min/max/sum/mean and linear-interpolated p25/median/p75 computed over the ${stats.count} FINITE vals (nulls/non-finite excluded; nonFiniteExcluded=${stats.nonFiniteExcluded}) across the FULL cross-section of ${totalAvailable} rows, sorted ascending — not a robust estimator for small frames.`,
|
|
1492
|
+
);
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1495
|
+
const outData: Record<string, unknown> = {
|
|
1496
|
+
found,
|
|
1497
|
+
taxonomy,
|
|
1498
|
+
tag: str(body.tag) ?? args.tag,
|
|
1499
|
+
unit,
|
|
1500
|
+
uom,
|
|
1501
|
+
period: args.period,
|
|
1502
|
+
label: str(body.label),
|
|
1503
|
+
description: str(body.description),
|
|
1504
|
+
rows: page,
|
|
1505
|
+
};
|
|
1506
|
+
if (stats) outData.stats = stats;
|
|
1507
|
+
|
|
1508
|
+
return withMeta(
|
|
1509
|
+
outData,
|
|
1510
|
+
edgarMeta({
|
|
1511
|
+
returned,
|
|
1512
|
+
totalAvailable,
|
|
1513
|
+
filtersApplied: ["taxonomy", "tag", "unit", "period"],
|
|
1514
|
+
pagination: {
|
|
1515
|
+
offset,
|
|
1516
|
+
limit,
|
|
1517
|
+
hasMore,
|
|
1518
|
+
nextOffset: hasMore ? offset + returned : null,
|
|
1519
|
+
},
|
|
1520
|
+
notes,
|
|
1521
|
+
}),
|
|
1522
|
+
);
|
|
1523
|
+
}
|
|
1524
|
+
|
|
1525
|
+
// ─── Tool 6: edgar_filing_index (ADR-0026) ────────────────────────
|
|
1526
|
+
// A KEYLESS BULK cross-filer capability on the EXISTING edgar source: read the
|
|
1527
|
+
// SEC EDGAR quarterly FULL-INDEX (www.sec.gov/Archives/edgar/full-index/<year>/
|
|
1528
|
+
// QTR<n>/master.idx — a COMPLETE cross-filer index of EVERY filer's EVERY filing
|
|
1529
|
+
// in a quarter, pipe-delimited, ~33MB / ~370K rows), FULL-SCAN it, apply
|
|
1530
|
+
// CLIENT-SIDE filters, and return offset-paginated filings with an EXACT total.
|
|
1531
|
+
// The per-filer edgar tools require you ALREADY hold a CIK; this is the
|
|
1532
|
+
// bulk-enumeration primitive ("every 8-K in 2024 Q1", "every filing by CIK X in
|
|
1533
|
+
// Q1", "every filer named '…'"). Reuses getEdgar VERBATIM (UA + Accept-Encoding:
|
|
1534
|
+
// gzip + the ≤10 req/s throughGate — UNTOUCHED, so the other 5 edgar tools stay
|
|
1535
|
+
// byte-identical), `.text()`, padCik, the module-local `str`, `driftError`,
|
|
1536
|
+
// `buildMeta`/`withMeta`. Copies (does NOT import) the frames buildFramesUrl
|
|
1537
|
+
// S1/S2 path-segment idiom.
|
|
1538
|
+
//
|
|
1539
|
+
// LIVE-VERIFIED 2026-07-12 (org UA `cliwant-mcp-sam-gov/1.0`):
|
|
1540
|
+
// - 2024/QTR1/master.idx → 200, 33,206,408 bytes, 370,304 data rows (all 5-field);
|
|
1541
|
+
// exact 8-K count 16,997, exact 10-K count 4,980.
|
|
1542
|
+
// - `Range: bytes=0-1000` → 200 FULL body (Range IGNORED, no 206) → a byte-cap
|
|
1543
|
+
// would truncate the CIK-sorted tail → under-count; FULL-SCAN → EXACT
|
|
1544
|
+
// totalAvailable is the ONLY honest model (byte-cap FORBIDDEN).
|
|
1545
|
+
// - 2026/QTR4 (current-YEAR future quarter) → 200, header+dashes, 0 data rows →
|
|
1546
|
+
// GENUINE-EMPTY (M1: NOT drift). 2026/QTR3 (current quarter) → 200, ~33K rows,
|
|
1547
|
+
// GROWING daily (short cache TTL + a point-in-time-snapshot note).
|
|
1548
|
+
// - 2024/QTR5 → 403 `AccessDenied` XML (does NOT match getEdgar F6's /automated|
|
|
1549
|
+
// undeclared/ → getEdgar mislabels it `rate_limited`) → TOOL-LOCAL reclassify.
|
|
1550
|
+
// - 2027/QTR1 (future YEAR) → 403, but the year bound blocks it PRE-FETCH
|
|
1551
|
+
// (invalid_input, 0 fetch — a future year has no published quarter index).
|
|
1552
|
+
|
|
1553
|
+
const FULLINDEX_HOST = "www.sec.gov";
|
|
1554
|
+
const FULLINDEX_BASE = "https://www.sec.gov/Archives/edgar/full-index";
|
|
1555
|
+
const FULLINDEX_FILE = "master.idx"; // the single backing file (pipe-delimited)
|
|
1556
|
+
const FULLINDEX_LABEL = "edgar:full-index"; // host+path only; keyless ⇒ no token can appear
|
|
1557
|
+
const FULLINDEX_HEADER = "CIK|Company Name|Form Type|Date Filed|Filename";
|
|
1558
|
+
const EDGAR_FULLINDEX_START_YEAR = 1993; // EDGAR full-index begins 1993 Q1
|
|
1559
|
+
const FILING_ARCHIVE_BASE = "https://www.sec.gov/Archives/"; // prefix for row.filename → a resolvable URL
|
|
1560
|
+
// Safety ceiling ABOVE the live ~370K rows (2024Q1 = 370,304). If a body EXCEEDS
|
|
1561
|
+
// it (drift/hostile/giant), the scan stops + `totalIsLowerBound` is set + a note —
|
|
1562
|
+
// NEVER a silent truncation. The normal path never reaches it, so totals stay EXACT.
|
|
1563
|
+
// Exported (with the `maxRows` param on parseFullIndex) so the fault suite can drive
|
|
1564
|
+
// the ceiling with a COMPACT fixture (A2) instead of a 500K-row body.
|
|
1565
|
+
export const MAX_INDEX_ROWS = 500_000;
|
|
1566
|
+
|
|
1567
|
+
/**
|
|
1568
|
+
* Fresh UTC year at CALL time (M2/S1) — NOT a module-load constant, so the upper
|
|
1569
|
+
* year bound survives a year rollover in a long-running process.
|
|
1570
|
+
*/
|
|
1571
|
+
function currentUtcYear(): number {
|
|
1572
|
+
return new Date().getUTCFullYear();
|
|
1573
|
+
}
|
|
1574
|
+
|
|
1575
|
+
/**
|
|
1576
|
+
* True iff (year,quarter) is the CURRENT calendar quarter (UTC). The current
|
|
1577
|
+
* quarter GROWS daily (short cache TTL + a point-in-time-snapshot note); a closed
|
|
1578
|
+
* past quarter is immutable (long TTL). NOTE: this is a freshness/cache signal
|
|
1579
|
+
* ONLY — it is deliberately NOT a pre-fetch guard (M2: a same-year future quarter
|
|
1580
|
+
* returns a well-formed empty 200, so it must be reachable, not refused).
|
|
1581
|
+
*/
|
|
1582
|
+
function isCurrentQuarter(year: number, quarter: number): boolean {
|
|
1583
|
+
const now = new Date();
|
|
1584
|
+
return (
|
|
1585
|
+
year === now.getUTCFullYear() &&
|
|
1586
|
+
quarter === Math.floor(now.getUTCMonth() / 3) + 1
|
|
1587
|
+
);
|
|
1588
|
+
}
|
|
1589
|
+
|
|
1590
|
+
/** Throw the pre-fetch bounds/injection guard error (invalid_input, 0 fetch). */
|
|
1591
|
+
function fullIndexInvalid(message: string): never {
|
|
1592
|
+
throw new ToolErrorCarrier({
|
|
1593
|
+
kind: "invalid_input",
|
|
1594
|
+
message,
|
|
1595
|
+
retryable: false,
|
|
1596
|
+
upstreamEndpoint: FULLINDEX_LABEL,
|
|
1597
|
+
});
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
/**
|
|
1601
|
+
* Build the full-index URL from the ONLY two caller-influenced path segments
|
|
1602
|
+
* (year, quarter) — the frames `buildFramesUrl` S1/S2 idiom, with the segments as
|
|
1603
|
+
* BOUNDED INTEGERS (a range check, not a regex: `%`/`/`/`.`/`\`/`..`/`%2F` cannot
|
|
1604
|
+
* appear in an integer, so there is no path-segment-injection surface). This is
|
|
1605
|
+
* BELT-AND-SUSPENDERS behind Zod (a direct handler call could bypass Zod):
|
|
1606
|
+
* - year ∈ [1993, currentUtcYear()] else invalid_input (0 fetch) — the upper
|
|
1607
|
+
* bound is CALL-TIME fresh (M2); a future YEAR has no published quarter index.
|
|
1608
|
+
* - quarter ∈ {1,2,3,4} else invalid_input (0 fetch).
|
|
1609
|
+
* There is NO `≤currentCalendarQuarter` guard (M2): a same-year FUTURE quarter
|
|
1610
|
+
* returns a well-formed EMPTY 200 (2026/QTR4 live), so refusing it would be a false
|
|
1611
|
+
* "not available"; the two honest terminal paths (200+header+0rows → genuine-empty;
|
|
1612
|
+
* a bounds-valid 403 → the ambiguous reclassify) handle everything else.
|
|
1613
|
+
* THEN assert the built URL is https on the fixed www.sec.gov host.
|
|
1614
|
+
*/
|
|
1615
|
+
function buildFullIndexUrl(year: number, quarter: number): string {
|
|
1616
|
+
const maxYear = currentUtcYear();
|
|
1617
|
+
if (
|
|
1618
|
+
!Number.isInteger(year) ||
|
|
1619
|
+
year < EDGAR_FULLINDEX_START_YEAR ||
|
|
1620
|
+
year > maxYear
|
|
1621
|
+
) {
|
|
1622
|
+
fullIndexInvalid(
|
|
1623
|
+
`edgar_filing_index: year ${JSON.stringify(year)} must be an integer in [${EDGAR_FULLINDEX_START_YEAR}, ${maxYear}] (EDGAR full-index begins 1993 Q1; a future year has no published quarter index) — refused before any fetch (path-segment bounds guard).`,
|
|
1624
|
+
);
|
|
1625
|
+
}
|
|
1626
|
+
if (!Number.isInteger(quarter) || quarter < 1 || quarter > 4) {
|
|
1627
|
+
fullIndexInvalid(
|
|
1628
|
+
`edgar_filing_index: quarter ${JSON.stringify(quarter)} must be an integer in {1,2,3,4} — refused before any fetch (path-segment bounds guard).`,
|
|
1629
|
+
);
|
|
1630
|
+
}
|
|
1631
|
+
const built = `${FULLINDEX_BASE}/${year}/QTR${quarter}/${FULLINDEX_FILE}`;
|
|
1632
|
+
let parsed: URL;
|
|
1633
|
+
try {
|
|
1634
|
+
parsed = new URL(built);
|
|
1635
|
+
} catch {
|
|
1636
|
+
fullIndexInvalid(
|
|
1637
|
+
`edgar_filing_index: could not construct a valid URL from year=${year} quarter=${quarter} — refused before any fetch.`,
|
|
1638
|
+
);
|
|
1639
|
+
}
|
|
1640
|
+
if (parsed.protocol !== "https:" || parsed.hostname !== FULLINDEX_HOST) {
|
|
1641
|
+
fullIndexInvalid(
|
|
1642
|
+
`edgar_filing_index: constructed URL host/scheme is not https://${FULLINDEX_HOST} (${parsed.protocol}//${parsed.hostname}) — refused before any fetch (fixed-host assertion).`,
|
|
1643
|
+
);
|
|
1644
|
+
}
|
|
1645
|
+
return built;
|
|
1646
|
+
}
|
|
1647
|
+
|
|
1648
|
+
/** One filing row from master.idx. CIK stays a STRING (never num-coerced). */
|
|
1649
|
+
export type FilingIndexRow = {
|
|
1650
|
+
cik: string | null; // raw CIK from the row (e.g. "1000045")
|
|
1651
|
+
cikPadded: string | null; // padCik(cik) canonical 10-digit — the join key to the per-filer edgar tools
|
|
1652
|
+
companyName: string | null;
|
|
1653
|
+
formType: string | null;
|
|
1654
|
+
dateFiled: string | null; // "YYYY-MM-DD"
|
|
1655
|
+
filename: string | null; // raw archive path "edgar/data/<cik>/<accession>.txt"
|
|
1656
|
+
filingUrl: string | null; // FILING_ARCHIVE_BASE + filename → a resolvable URL (null if filename null)
|
|
1657
|
+
};
|
|
1658
|
+
|
|
1659
|
+
type ParsedFullIndex = {
|
|
1660
|
+
rows: FilingIndexRow[];
|
|
1661
|
+
malformedRows: number; // data rows that did NOT split into 5 pipe fields (skipped + counted)
|
|
1662
|
+
totalIsLowerBound: boolean; // true iff the MAX_INDEX_ROWS safety ceiling was hit
|
|
1663
|
+
};
|
|
1664
|
+
|
|
1665
|
+
/**
|
|
1666
|
+
* Split `line` on its FIRST `n` occurrences of `|`; the remainder (which may
|
|
1667
|
+
* itself contain `|`) is the FINAL element (the BOUNDED pipe-split — minor fix). For
|
|
1668
|
+
* n=4 a well-formed row yields exactly 5 fields, and a row carrying an EXTRA `|`
|
|
1669
|
+
* still yields 5 fields (the extra pipe stays inside the rejoined tail) rather than
|
|
1670
|
+
* being dropped as "malformed" — so `totalAvailable` is never SILENTLY under-counted
|
|
1671
|
+
* by an over-split. A row with FEWER than 4 pipes returns <5 elements → malformed.
|
|
1672
|
+
*/
|
|
1673
|
+
function splitOnFirstPipes(line: string, n: number): string[] {
|
|
1674
|
+
const out: string[] = [];
|
|
1675
|
+
let start = 0;
|
|
1676
|
+
for (let k = 0; k < n; k++) {
|
|
1677
|
+
const idx = line.indexOf("|", start);
|
|
1678
|
+
if (idx === -1) break;
|
|
1679
|
+
out.push(line.slice(start, idx));
|
|
1680
|
+
start = idx + 1;
|
|
1681
|
+
}
|
|
1682
|
+
out.push(line.slice(start));
|
|
1683
|
+
return out;
|
|
1684
|
+
}
|
|
1685
|
+
|
|
1686
|
+
/**
|
|
1687
|
+
* Parse the raw master.idx body into `FilingIndexRow[]`.
|
|
1688
|
+
*
|
|
1689
|
+
* DRIFT keys on the ABSENCE of the `CIK|Company Name|Form Type|Date Filed|Filename`
|
|
1690
|
+
* header + the `----` dashes boundary ONLY (M1) — a non-index / error / format-changed
|
|
1691
|
+
* body served with HTTP 200 (e.g. an S3 error HTML page) → THROW `driftError`. A
|
|
1692
|
+
* body WITH the header+dashes but ZERO data rows is a GENUINE-EMPTY quarter (2026/QTR4
|
|
1693
|
+
* live) → returned to the caller (NOT thrown). A body WITH the header+dashes whose
|
|
1694
|
+
* EVERY data row fails the 5-field split → THROW `driftError` (all-malformed = format
|
|
1695
|
+
* drift). The fixed preamble (`Description:` …) BEFORE the header is skipped.
|
|
1696
|
+
*/
|
|
1697
|
+
export function parseFullIndex(
|
|
1698
|
+
body: string,
|
|
1699
|
+
maxRows: number = MAX_INDEX_ROWS,
|
|
1700
|
+
): ParsedFullIndex {
|
|
1701
|
+
const lines = body.split("\n");
|
|
1702
|
+
// Locate the header line immediately followed by the `----` dashes boundary.
|
|
1703
|
+
let dashesIdx = -1;
|
|
1704
|
+
for (let i = 0; i < lines.length - 1; i++) {
|
|
1705
|
+
if ((lines[i] ?? "").replace(/\r$/, "").trim() === FULLINDEX_HEADER) {
|
|
1706
|
+
const next = (lines[i + 1] ?? "").replace(/\r$/, "");
|
|
1707
|
+
if (/^-{5,}\s*$/.test(next.trim())) {
|
|
1708
|
+
dashesIdx = i + 1;
|
|
1709
|
+
break;
|
|
1710
|
+
}
|
|
1711
|
+
}
|
|
1712
|
+
}
|
|
1713
|
+
if (dashesIdx === -1) {
|
|
1714
|
+
throw driftError(
|
|
1715
|
+
FULLINDEX_LABEL,
|
|
1716
|
+
`edgar:full-index body is missing the '${FULLINDEX_HEADER}' header / '----' dashes boundary (a non-index / error / format-changed body served with HTTP 200) — refusing to report an empty result (schema drift, NOT a genuine-empty quarter).`,
|
|
1717
|
+
);
|
|
1718
|
+
}
|
|
1719
|
+
const rows: FilingIndexRow[] = [];
|
|
1720
|
+
let malformedRows = 0;
|
|
1721
|
+
let totalIsLowerBound = false;
|
|
1722
|
+
for (let i = dashesIdx + 1; i < lines.length; i++) {
|
|
1723
|
+
const line = (lines[i] ?? "").replace(/\r$/, "");
|
|
1724
|
+
if (line.trim() === "") continue; // blank line (e.g. the trailing newline) — skip
|
|
1725
|
+
// Safety ceiling (never a SILENT truncation): once the scanned data-row count
|
|
1726
|
+
// reaches MAX_INDEX_ROWS, stop + set totalIsLowerBound. The live ~370K is well
|
|
1727
|
+
// under 500K, so the normal path never trips it and totals stay EXACT.
|
|
1728
|
+
if (rows.length + malformedRows >= maxRows) {
|
|
1729
|
+
totalIsLowerBound = true;
|
|
1730
|
+
break;
|
|
1731
|
+
}
|
|
1732
|
+
const parts = splitOnFirstPipes(line, 4);
|
|
1733
|
+
if (parts.length < 5) {
|
|
1734
|
+
malformedRows++;
|
|
1735
|
+
continue;
|
|
1736
|
+
}
|
|
1737
|
+
const cik = str(parts[0]);
|
|
1738
|
+
const filename = str(parts[4]);
|
|
1739
|
+
rows.push({
|
|
1740
|
+
cik,
|
|
1741
|
+
cikPadded: cik === null ? null : padCik(cik),
|
|
1742
|
+
companyName: str(parts[1]),
|
|
1743
|
+
formType: str(parts[2]),
|
|
1744
|
+
dateFiled: str(parts[3]),
|
|
1745
|
+
filename,
|
|
1746
|
+
filingUrl: filename === null ? null : FILING_ARCHIVE_BASE + filename,
|
|
1747
|
+
});
|
|
1748
|
+
}
|
|
1749
|
+
// M1 — header+dashes PRESENT but EVERY data row failed the split ⇒ drift (a
|
|
1750
|
+
// format change). 0 data rows total (rows.length===0 && malformedRows===0) ⇒
|
|
1751
|
+
// GENUINE-EMPTY (returned by the caller, NOT thrown).
|
|
1752
|
+
if (rows.length === 0 && malformedRows > 0) {
|
|
1753
|
+
throw driftError(
|
|
1754
|
+
FULLINDEX_LABEL,
|
|
1755
|
+
`edgar:full-index has the header/dashes boundary but ALL ${malformedRows} data row(s) failed the 5-field pipe split (format drift) — refusing to report an empty result.`,
|
|
1756
|
+
);
|
|
1757
|
+
}
|
|
1758
|
+
return { rows, malformedRows, totalIsLowerBound };
|
|
1759
|
+
}
|
|
1760
|
+
|
|
1761
|
+
// A1 — a BOUNDED LRU for the full-index RAW TEXT. DELIBERATELY NOT the shared,
|
|
1762
|
+
// UNBOUNDED `memoize` (cache.ts): as the review flagged, memoizing ~370K parsed
|
|
1763
|
+
// row-objects per (year,quarter) with no eviction would OOM when a client sweeps
|
|
1764
|
+
// many quarters. This caps at FULLINDEX_CACHE_MAX distinct quarters (a HARD size
|
|
1765
|
+
// bound), caches the RAW TEXT (leaner than parsed rows) re-parsed per call (cheap),
|
|
1766
|
+
// and TTL-splits: the CURRENT quarter grows daily → short TTL; a CLOSED quarter is
|
|
1767
|
+
// immutable → long TTL. Map insertion order = LRU recency; the oldest quarter is
|
|
1768
|
+
// evicted past the cap. A repeat call for the same quarter (different filters/
|
|
1769
|
+
// offset) re-uses the cached text — NO second ~33MB download.
|
|
1770
|
+
const FULLINDEX_CACHE_MAX = 3;
|
|
1771
|
+
const FULLINDEX_TTL_CURRENT_MS = 5 * 60 * 1000; // current quarter: 5 min (grows daily)
|
|
1772
|
+
const FULLINDEX_TTL_CLOSED_MS = 6 * 60 * 60 * 1000; // closed quarter: 6h (immutable)
|
|
1773
|
+
const fullIndexCache = new Map<string, { text: string; expiresAt: number }>();
|
|
1774
|
+
|
|
1775
|
+
/** Fetch the whole quarter's master.idx text (getEdgar VERBATIM + `.text()`),
|
|
1776
|
+
* through the bounded LRU (A1). getEdgar sets the mandatory UA + gzip + the ≤10
|
|
1777
|
+
* req/s gate + the 15s timeout; a slow body-read → abort → getEdgar's honest
|
|
1778
|
+
* upstream_unavailable throw (never a fake-empty). */
|
|
1779
|
+
async function fetchFullIndexText(
|
|
1780
|
+
built: string,
|
|
1781
|
+
year: number,
|
|
1782
|
+
quarter: number,
|
|
1783
|
+
): Promise<string> {
|
|
1784
|
+
const key = `${year}:${quarter}`;
|
|
1785
|
+
const now = Date.now();
|
|
1786
|
+
const hit = fullIndexCache.get(key);
|
|
1787
|
+
if (hit && hit.expiresAt > now) {
|
|
1788
|
+
fullIndexCache.delete(key); // LRU touch → re-insert as the newest
|
|
1789
|
+
fullIndexCache.set(key, hit);
|
|
1790
|
+
return hit.text;
|
|
1791
|
+
}
|
|
1792
|
+
if (hit) fullIndexCache.delete(key); // expired
|
|
1793
|
+
const r = await getEdgar(built, FULLINDEX_LABEL);
|
|
1794
|
+
const text = await r.text();
|
|
1795
|
+
const ttl = isCurrentQuarter(year, quarter)
|
|
1796
|
+
? FULLINDEX_TTL_CURRENT_MS
|
|
1797
|
+
: FULLINDEX_TTL_CLOSED_MS;
|
|
1798
|
+
fullIndexCache.set(key, { text, expiresAt: now + ttl });
|
|
1799
|
+
while (fullIndexCache.size > FULLINDEX_CACHE_MAX) {
|
|
1800
|
+
const oldest = fullIndexCache.keys().next().value as string | undefined;
|
|
1801
|
+
if (oldest === undefined) break;
|
|
1802
|
+
fullIndexCache.delete(oldest);
|
|
1803
|
+
}
|
|
1804
|
+
return text;
|
|
1805
|
+
}
|
|
1806
|
+
|
|
1807
|
+
/** For tests: evict the full-index bounded LRU (mirrors cache.ts `_clearCache` for
|
|
1808
|
+
* this dedicated cache, which the shared `_clearCache` does not touch). */
|
|
1809
|
+
export function _resetFullIndexCache(): void {
|
|
1810
|
+
fullIndexCache.clear();
|
|
1811
|
+
}
|
|
1812
|
+
|
|
1813
|
+
/**
|
|
1814
|
+
* Read the SEC EDGAR quarterly full-index for (year, quarter) and return the
|
|
1815
|
+
* filings matching the given CLIENT-SIDE filters (form / CIK / company substring /
|
|
1816
|
+
* date range), offset-paginated, with the EXACT total match count for the quarter.
|
|
1817
|
+
*
|
|
1818
|
+
* HONESTY (ADR-0026 v2):
|
|
1819
|
+
* - FULL-SCAN → `totalAvailable` is the EXACT filtered match count across the WHOLE
|
|
1820
|
+
* quarter (never a page length, never a byte-capped under-count — SEC ignores Range).
|
|
1821
|
+
* - A bounds-valid but unpublished quarter 403s (getEdgar mislabels it rate_limited);
|
|
1822
|
+
* TOOL-LOCAL reclassify to an AMBIGUOUS both-causes error (unpublished quarter OR
|
|
1823
|
+
* the 10 req/s rate-block) — never a bare rate-limit, never a fake-empty.
|
|
1824
|
+
* - Drift on header/dashes ABSENCE or an all-malformed body (THROW); header+0-rows ⇒
|
|
1825
|
+
* genuine-empty (complete:true). A future year / bad quarter ⇒ invalid_input, 0 fetch.
|
|
1826
|
+
* - CIK stays a STRING; every column via the module-local `str` (null-never-"").
|
|
1827
|
+
* - `companyContains` is a LITERAL case-insensitive substring (C110 N/A — no token split).
|
|
1828
|
+
*/
|
|
1829
|
+
export async function filingIndex(args: {
|
|
1830
|
+
year: number;
|
|
1831
|
+
quarter: number;
|
|
1832
|
+
formType?: string;
|
|
1833
|
+
cik?: string | number;
|
|
1834
|
+
companyContains?: string;
|
|
1835
|
+
dateFrom?: string;
|
|
1836
|
+
dateTo?: string;
|
|
1837
|
+
limit?: number;
|
|
1838
|
+
offset?: number;
|
|
1839
|
+
}): Promise<MetaBundle> {
|
|
1840
|
+
const { year, quarter } = args;
|
|
1841
|
+
// S1 — build + re-validate the path BEFORE any fetch (0 fetch on a bad year/quarter).
|
|
1842
|
+
const built = buildFullIndexUrl(year, quarter);
|
|
1843
|
+
|
|
1844
|
+
// Fetch the WHOLE quarter (getEdgar VERBATIM + `.text()`, bounded-LRU cached). A
|
|
1845
|
+
// bounds-valid 403 (an unpublished quarter) is reclassified TOOL-LOCAL — getEdgar
|
|
1846
|
+
// is UNTOUCHED (the other 5 edgar tools stay byte-identical).
|
|
1847
|
+
let body: string;
|
|
1848
|
+
try {
|
|
1849
|
+
body = await fetchFullIndexText(built, year, quarter);
|
|
1850
|
+
} catch (e) {
|
|
1851
|
+
if (
|
|
1852
|
+
e instanceof ToolErrorCarrier &&
|
|
1853
|
+
e.toolError.kind === "rate_limited" &&
|
|
1854
|
+
e.toolError.upstreamStatus === 403
|
|
1855
|
+
) {
|
|
1856
|
+
// 403-not-404 reclassify (fact #2): a bad/unpublished full-index quarter
|
|
1857
|
+
// returns 403 `AccessDenied` (NOT 404); getEdgar's F6 can't match /automated|
|
|
1858
|
+
// undeclared/ so it labels it rate_limited. Re-surface an AMBIGUOUS both-causes
|
|
1859
|
+
// error — NEVER a bare "rate limited", NEVER a fake-empty. A REAL 429 (status
|
|
1860
|
+
// 429, not 403) is NOT caught here → stays an honest rate_limited.
|
|
1861
|
+
throw new ToolErrorCarrier({
|
|
1862
|
+
kind: "upstream_unavailable",
|
|
1863
|
+
message: `SEC returned HTTP 403 for the full-index path ${FULLINDEX_BASE}/${year}/QTR${quarter}/${FULLINDEX_FILE}. This is AMBIGUOUS: EITHER the ${year} QTR${quarter} index is not published yet (a too-early / non-existent quarter returns 403, not 404) OR SEC is rate-limiting this IP at the 10 req/s ceiling (~10-minute block). NOT fabricated as empty. Verify the quarter is a real past/current EDGAR quarter and retry after ~10 minutes.`,
|
|
1864
|
+
retryable: true,
|
|
1865
|
+
retryAfterSeconds: 600,
|
|
1866
|
+
upstreamStatus: 403,
|
|
1867
|
+
upstreamEndpoint: FULLINDEX_LABEL,
|
|
1868
|
+
});
|
|
1869
|
+
}
|
|
1870
|
+
throw e; // schema_drift / invalid_input(UA) / upstream_unavailable / real 429 — loud
|
|
1871
|
+
}
|
|
1872
|
+
|
|
1873
|
+
// FULL-SCAN → parse ALL rows past the preamble. Drift (header-absent / all-malformed)
|
|
1874
|
+
// THROWS; header+0-rows ⇒ genuine-empty (below).
|
|
1875
|
+
const parsed = parseFullIndex(body);
|
|
1876
|
+
const all = parsed.rows;
|
|
1877
|
+
|
|
1878
|
+
// CLIENT-SIDE filters — ZERO query string (none reach the URL). cik via padCik
|
|
1879
|
+
// both-sides; companyContains a LITERAL case-insensitive substring (no token split);
|
|
1880
|
+
// dates are ISO string compares (the column is already YYYY-MM-DD).
|
|
1881
|
+
const formNeedle = args.formType?.trim()
|
|
1882
|
+
? args.formType.trim().toLowerCase()
|
|
1883
|
+
: null;
|
|
1884
|
+
const cikFilter =
|
|
1885
|
+
args.cik != null && String(args.cik).trim() !== ""
|
|
1886
|
+
? padCik(args.cik)
|
|
1887
|
+
: null;
|
|
1888
|
+
const companyNeedle = args.companyContains?.trim()
|
|
1889
|
+
? args.companyContains.trim().toLowerCase()
|
|
1890
|
+
: null;
|
|
1891
|
+
const dateFrom = args.dateFrom;
|
|
1892
|
+
const dateTo = args.dateTo;
|
|
1893
|
+
|
|
1894
|
+
const matches = all.filter((row) => {
|
|
1895
|
+
if (formNeedle !== null) {
|
|
1896
|
+
if (row.formType === null || row.formType.toLowerCase() !== formNeedle)
|
|
1897
|
+
return false;
|
|
1898
|
+
}
|
|
1899
|
+
if (cikFilter !== null) {
|
|
1900
|
+
if (row.cikPadded !== cikFilter) return false;
|
|
1901
|
+
}
|
|
1902
|
+
if (companyNeedle !== null) {
|
|
1903
|
+
if (
|
|
1904
|
+
row.companyName === null ||
|
|
1905
|
+
!row.companyName.toLowerCase().includes(companyNeedle)
|
|
1906
|
+
)
|
|
1907
|
+
return false;
|
|
1908
|
+
}
|
|
1909
|
+
if (dateFrom !== undefined) {
|
|
1910
|
+
if (row.dateFiled === null || row.dateFiled < dateFrom) return false;
|
|
1911
|
+
}
|
|
1912
|
+
if (dateTo !== undefined) {
|
|
1913
|
+
if (row.dateFiled === null || row.dateFiled > dateTo) return false;
|
|
1914
|
+
}
|
|
1915
|
+
return true;
|
|
1916
|
+
});
|
|
1917
|
+
|
|
1918
|
+
const limit = args.limit ?? 100;
|
|
1919
|
+
const offset = args.offset ?? 0;
|
|
1920
|
+
const totalAvailable = matches.length; // EXACT (full-scan) — never a page length
|
|
1921
|
+
const page = matches.slice(offset, offset + limit);
|
|
1922
|
+
const returned = page.length;
|
|
1923
|
+
const hasMore = offset + returned < totalAvailable;
|
|
1924
|
+
const nextOffset = hasMore ? offset + returned : null;
|
|
1925
|
+
|
|
1926
|
+
const filtersApplied: string[] = [];
|
|
1927
|
+
if (formNeedle !== null) filtersApplied.push("formType");
|
|
1928
|
+
if (cikFilter !== null) filtersApplied.push("cik");
|
|
1929
|
+
if (companyNeedle !== null) filtersApplied.push("companyContains");
|
|
1930
|
+
if (dateFrom !== undefined) filtersApplied.push("dateFrom");
|
|
1931
|
+
if (dateTo !== undefined) filtersApplied.push("dateTo");
|
|
1932
|
+
|
|
1933
|
+
const notes: string[] = [];
|
|
1934
|
+
// Index-vs-live-submissions caveat (always).
|
|
1935
|
+
notes.push(
|
|
1936
|
+
`The quarterly full-index (master.idx) is a POINT-IN-TIME snapshot of ${year} QTR${quarter}'s EDGAR dissemination feed — it covers ONLY that quarter. For the LATEST or complete-history filings of a KNOWN filer use edgar_company_filings (it fans out older shards); for a text query across 2001-present use edgar_full_text_search.`,
|
|
1937
|
+
);
|
|
1938
|
+
// Full-scan / exact-total disclosure + the ~33MB size caveat.
|
|
1939
|
+
notes.push(
|
|
1940
|
+
`The whole ${year} QTR${quarter} index was downloaded and FULL-SCANNED (~33MB / hundreds of thousands of rows); totalAvailable (${totalAvailable}) is the EXACT count of filings matching the filters across the ENTIRE quarter (not a page length, not a byte-capped subset — SEC ignores HTTP Range). This page contains ${returned} of ${totalAvailable}; page via _meta.pagination.nextOffset for the rest.`,
|
|
1941
|
+
);
|
|
1942
|
+
if (isCurrentQuarter(year, quarter)) {
|
|
1943
|
+
notes.push(
|
|
1944
|
+
`${year} QTR${quarter} is the CURRENT calendar quarter — it GROWS daily as new filings disseminate, so totalAvailable is EXACT AS-OF this (short-cached) snapshot, not exact-forever. A closed past quarter is immutable.`,
|
|
1945
|
+
);
|
|
1946
|
+
}
|
|
1947
|
+
if (companyNeedle !== null) {
|
|
1948
|
+
notes.push(
|
|
1949
|
+
`companyContains is a case-insensitive LITERAL substring match on the Company Name column — a multi-word value matches as ONE contiguous string, NOT AND/OR-tokenized.`,
|
|
1950
|
+
);
|
|
1951
|
+
}
|
|
1952
|
+
if (formNeedle !== null) {
|
|
1953
|
+
notes.push(
|
|
1954
|
+
`formType is a case-insensitive EXACT match on the Form Type column ("${args.formType?.trim()}" matches that form only — e.g. "8-K" does NOT match "8-K/A"). Pass each amendment variant separately.`,
|
|
1955
|
+
);
|
|
1956
|
+
}
|
|
1957
|
+
if (parsed.malformedRows > 0) {
|
|
1958
|
+
notes.push(
|
|
1959
|
+
`${parsed.malformedRows} row(s) did not split into 5 pipe fields and were skipped (tolerated as stray malformed rows; a body with ZERO valid rows would instead be refused as schema drift).`,
|
|
1960
|
+
);
|
|
1961
|
+
}
|
|
1962
|
+
if (parsed.totalIsLowerBound) {
|
|
1963
|
+
notes.push(
|
|
1964
|
+
`The scan hit the MAX_INDEX_ROWS safety ceiling (${MAX_INDEX_ROWS}); totals are a LOWER BOUND — the quarter index is larger than expected (possible format drift). See totalIsLowerBound.`,
|
|
1965
|
+
);
|
|
1966
|
+
}
|
|
1967
|
+
if (totalAvailable === 0) {
|
|
1968
|
+
notes.push(
|
|
1969
|
+
filtersApplied.length
|
|
1970
|
+
? `0 filings in ${year} QTR${quarter} matched the filters (${filtersApplied.join(", ")}). This is an EXACT ZERO over the full quarter index — NOT a truncation, NOT an outage.`
|
|
1971
|
+
: `0 filings in ${year} QTR${quarter} (the quarter index has no data rows). This is an EXACT ZERO over the full quarter index — NOT a truncation, NOT an outage.`,
|
|
1972
|
+
);
|
|
1973
|
+
}
|
|
1974
|
+
|
|
1975
|
+
const meta: Partial<ResponseMeta> = {
|
|
1976
|
+
returned,
|
|
1977
|
+
totalAvailable,
|
|
1978
|
+
filtersApplied,
|
|
1979
|
+
pagination: { offset, limit, hasMore, nextOffset },
|
|
1980
|
+
notes,
|
|
1981
|
+
};
|
|
1982
|
+
if (parsed.totalIsLowerBound) meta.totalIsLowerBound = true;
|
|
1983
|
+
|
|
1984
|
+
return withMeta(
|
|
1985
|
+
{
|
|
1986
|
+
year,
|
|
1987
|
+
quarter,
|
|
1988
|
+
indexFile: FULLINDEX_FILE,
|
|
1989
|
+
returned,
|
|
1990
|
+
totalAvailable,
|
|
1991
|
+
filings: page,
|
|
1992
|
+
},
|
|
1993
|
+
edgarMeta(meta),
|
|
1994
|
+
);
|
|
1995
|
+
}
|
|
1996
|
+
|
|
1997
|
+
// ─── Tool 7: edgar_daily_filing_index (ADR-0027) ──────────────────
|
|
1998
|
+
// The per-DAY sibling of edgar_filing_index: read the SEC EDGAR DAILY-index
|
|
1999
|
+
// (www.sec.gov/Archives/edgar/daily-index/<year>/QTR<n>/master.YYYYMMDD.idx — a
|
|
2000
|
+
// COMPLETE cross-filer index of EVERY filer's EVERY filing on ONE calendar day,
|
|
2001
|
+
// pipe-delimited, ~0.5–1.2MB / ~8K rows — ~30× smaller than the quarterly file),
|
|
2002
|
+
// FULL-SCAN it, apply CLIENT-SIDE filters, and return offset-paginated filings
|
|
2003
|
+
// with an EXACT total. It answers the monitoring/alerting question the quarterly
|
|
2004
|
+
// tool cannot without a whole-quarter download ("every 8-K filed on 2024-01-03").
|
|
2005
|
+
// Reuses getEdgar VERBATIM (UA + gzip + the ≤10 req/s throughGate — UNTOUCHED, so
|
|
2006
|
+
// the other 6 edgar tools stay byte-identical), `.text()`, padCik, the module-local
|
|
2007
|
+
// `str`, `driftError`, `buildMeta`/`withMeta`, `edgarMeta`, `splitOnFirstPipes`,
|
|
2008
|
+
// `MAX_INDEX_ROWS`, `currentUtcYear`, `FILING_ARCHIVE_BASE`, `FULLINDEX_HOST`. A
|
|
2009
|
+
// DEDICATED daily parser (do NOT reuse parseFullIndex — the daily header is
|
|
2010
|
+
// `…|Date Filed|File Name` WITH A SPACE, not the full-index `Filename`).
|
|
2011
|
+
//
|
|
2012
|
+
// LIVE-VERIFIED 2026-07-12 (SEC UA `cliwant-mcp-sam-gov research (…)`):
|
|
2013
|
+
// - master.20240103.idx (Wed) → 200; header VERBATIM `CIK|Company Name|Form Type|
|
|
2014
|
+
// Date Filed|File Name` (File Name WITH a space), then `----`; Date Filed column
|
|
2015
|
+
// compact `20240103`; 8426 data rows; 8-K = 220, 8-K/A = 5 (distinct/exact).
|
|
2016
|
+
// - master.20240106.idx (Sat) → 403 AccessDenied XML; index.json 2024/QTR1 does
|
|
2017
|
+
// NOT list it and its newest master is 20240329 ⇒ TRUE genuine-absent (found:false,
|
|
2018
|
+
// complete:true) — a real gap INSIDE the covered range.
|
|
2019
|
+
// - ★M1 recency (live Sun 2026-07-12): 2026/QTR3 index.json lists master ONLY through
|
|
2020
|
+
// master.20260709 (Thu); Fri 20260710 (a normal trading day) is UNLISTED and its
|
|
2021
|
+
// .idx 403s ⇒ requestedYyyymmdd > maxListedMasterDate ⇒ NOT-YET-DISSEMINATED
|
|
2022
|
+
// (found:false, complete:FALSE), NEVER a confident genuine-absent complete:true.
|
|
2023
|
+
//
|
|
2024
|
+
// ★HONESTY: the 403-disambiguation is DESIGN (b) — fetch the .idx FIRST (the happy
|
|
2025
|
+
// path pays ZERO oracle cost); consult the index.json existence oracle ONLY on a 403,
|
|
2026
|
+
// RECENCY-AWARE per M1 (maxListedMasterDate: newer-than-listed ⇒ not-yet-disseminated
|
|
2027
|
+
// complete:FALSE; listed-range gap ⇒ true genuine-absent complete:true; listed-but-403
|
|
2028
|
+
// ⇒ honest rate_limited; oracle-inconclusive ⇒ ambiguous both-causes upstream_unavailable).
|
|
2029
|
+
// The 403-reclassify is TOOL-LOCAL (getEdgar UNTOUCHED). M2: an EXACT date round-trip
|
|
2030
|
+
// PRE-fetch rejects Feb-30 / day-40 / non-leap-Feb-29 / a future date (invalid_input, 0 GET).
|
|
2031
|
+
|
|
2032
|
+
const DAILYINDEX_BASE = "https://www.sec.gov/Archives/edgar/daily-index";
|
|
2033
|
+
const DAILYINDEX_FILE = (yyyymmdd: string): string => `master.${yyyymmdd}.idx`; // the single backing file (pipe-delimited)
|
|
2034
|
+
const DAILYINDEX_LABEL = "edgar:daily-index"; // host+path only; keyless ⇒ no token can appear
|
|
2035
|
+
const DAILYINDEX_ORACLE_LABEL = "edgar:daily-index:index.json";
|
|
2036
|
+
// ★ File Name WITH A SPACE (the daily header) — the full-index uses `Filename` (no
|
|
2037
|
+
// space); a parser keying drift on `Filename` would FALSE-DRIFT on every valid daily index.
|
|
2038
|
+
const DAILYINDEX_HEADER = "CIK|Company Name|Form Type|Date Filed|File Name";
|
|
2039
|
+
const EDGAR_DAILY_START_YEAR = 1994; // EDGAR daily-index begins 1994 Q1 (conservative lower bound)
|
|
2040
|
+
|
|
2041
|
+
// A daily-specific bounded LRU of the RAW TEXT keyed by yyyymmdd (mirrors the shipped
|
|
2042
|
+
// fullIndexCache; NOT the shared unbounded `memoize`, which a many-day sweep would OOM).
|
|
2043
|
+
// Days are tiny (~0.5–1.2MB) → a larger cap than full-index's 3 is cheap. TTL splits:
|
|
2044
|
+
// TODAY may still be posting until ~22:00 ET → short TTL; a PAST day is immutable →
|
|
2045
|
+
// long TTL. The index.json oracle is fetched ONLY on the rare 403 path and is NOT cached.
|
|
2046
|
+
const DAILYINDEX_CACHE_MAX = 8;
|
|
2047
|
+
const DAILYINDEX_TTL_TODAY_MS = 5 * 60 * 1000; // today may still be posting → short TTL
|
|
2048
|
+
const DAILYINDEX_TTL_CLOSED_MS = 24 * 60 * 60 * 1000; // a past day is immutable → long TTL
|
|
2049
|
+
const dailyIndexCache = new Map<string, { text: string; expiresAt: number }>();
|
|
2050
|
+
|
|
2051
|
+
/** Throw the pre-fetch bounds/injection guard error (invalid_input, 0 fetch). */
|
|
2052
|
+
function dailyIndexInvalid(message: string): never {
|
|
2053
|
+
throw new ToolErrorCarrier({
|
|
2054
|
+
kind: "invalid_input",
|
|
2055
|
+
message,
|
|
2056
|
+
retryable: false,
|
|
2057
|
+
upstreamEndpoint: DAILYINDEX_LABEL,
|
|
2058
|
+
});
|
|
2059
|
+
}
|
|
2060
|
+
|
|
2061
|
+
/** Today's date as compact `YYYYMMDD` in UTC (call-time fresh — survives a day rollover). */
|
|
2062
|
+
function todayUtcYyyymmdd(): string {
|
|
2063
|
+
const now = new Date();
|
|
2064
|
+
const y = now.getUTCFullYear();
|
|
2065
|
+
const m = now.getUTCMonth() + 1;
|
|
2066
|
+
const d = now.getUTCDate();
|
|
2067
|
+
return `${y}${String(m).padStart(2, "0")}${String(d).padStart(2, "0")}`;
|
|
2068
|
+
}
|
|
2069
|
+
|
|
2070
|
+
/** True iff the compact `yyyymmdd` is TODAY (UTC) — the freshness/cache-TTL signal. */
|
|
2071
|
+
function isTodayUtc(yyyymmdd: string): boolean {
|
|
2072
|
+
return yyyymmdd === todayUtcYyyymmdd();
|
|
2073
|
+
}
|
|
2074
|
+
|
|
2075
|
+
/**
|
|
2076
|
+
* Normalize the row's compact `Date Filed` (`YYYYMMDD`, fact #3) to ISO `YYYY-MM-DD`
|
|
2077
|
+
* for the `dateFiled` output (so it matches edgar_filing_index / edgar_company_filings
|
|
2078
|
+
* and is human-readable). A non-8-digit value is kept as-is (defensive; via `str`).
|
|
2079
|
+
*/
|
|
2080
|
+
function normDailyDate(raw: unknown): string | null {
|
|
2081
|
+
const s = str(raw);
|
|
2082
|
+
if (s === null) return null;
|
|
2083
|
+
if (/^\d{8}$/.test(s)) return `${s.slice(0, 4)}-${s.slice(4, 6)}-${s.slice(6, 8)}`;
|
|
2084
|
+
return s;
|
|
2085
|
+
}
|
|
2086
|
+
|
|
2087
|
+
/**
|
|
2088
|
+
* Build the daily-index URL from the caller-influenced segments (year, quarter,
|
|
2089
|
+
* yyyymmdd — all DERIVED from one validated ISO date). BELT-AND-SUSPENDERS behind Zod
|
|
2090
|
+
* (a direct handler call could bypass it): re-run the integer/range guards + the
|
|
2091
|
+
* 8-digit-tied-to-year regex, then assert the built URL is https on the fixed
|
|
2092
|
+
* www.sec.gov host. The 8-digit regex admits NO `%`/`/`/`.`/`\`/`..`/`%2F` (JS `\d`
|
|
2093
|
+
* is ASCII-only). This does NOT reject a well-formed-but-NONEXISTENT day (Feb-30) —
|
|
2094
|
+
* the pre-fetch date round-trip (M2, in the handler) is the SOLE defense for that.
|
|
2095
|
+
*/
|
|
2096
|
+
function buildDailyIndexUrl(year: number, quarter: number, yyyymmdd: string): string {
|
|
2097
|
+
const maxYear = currentUtcYear();
|
|
2098
|
+
if (!Number.isInteger(year) || year < EDGAR_DAILY_START_YEAR || year > maxYear) {
|
|
2099
|
+
dailyIndexInvalid(
|
|
2100
|
+
`edgar_daily_filing_index: year ${JSON.stringify(year)} must be an integer in [${EDGAR_DAILY_START_YEAR}, ${maxYear}] (EDGAR daily-index begins 1994 Q1; a future year has no published daily index) — refused before any fetch (path-segment bounds guard).`,
|
|
2101
|
+
);
|
|
2102
|
+
}
|
|
2103
|
+
if (!Number.isInteger(quarter) || quarter < 1 || quarter > 4) {
|
|
2104
|
+
dailyIndexInvalid(
|
|
2105
|
+
`edgar_daily_filing_index: quarter ${JSON.stringify(quarter)} must be an integer in {1,2,3,4} — refused before any fetch (path-segment bounds guard).`,
|
|
2106
|
+
);
|
|
2107
|
+
}
|
|
2108
|
+
if (!/^\d{8}$/.test(yyyymmdd) || yyyymmdd.slice(0, 4) !== String(year)) {
|
|
2109
|
+
dailyIndexInvalid(
|
|
2110
|
+
`edgar_daily_filing_index: yyyymmdd ${JSON.stringify(yyyymmdd)} must be 8 digits whose year prefix === ${year} — refused before any fetch (path-segment injection guard; ties the compact day to the validated year, admits no slash/dot/percent/'..').`,
|
|
2111
|
+
);
|
|
2112
|
+
}
|
|
2113
|
+
const built = `${DAILYINDEX_BASE}/${year}/QTR${quarter}/${DAILYINDEX_FILE(yyyymmdd)}`;
|
|
2114
|
+
let parsed: URL;
|
|
2115
|
+
try {
|
|
2116
|
+
parsed = new URL(built);
|
|
2117
|
+
} catch {
|
|
2118
|
+
dailyIndexInvalid(
|
|
2119
|
+
`edgar_daily_filing_index: could not construct a valid URL from year=${year} quarter=${quarter} yyyymmdd=${yyyymmdd} — refused before any fetch.`,
|
|
2120
|
+
);
|
|
2121
|
+
}
|
|
2122
|
+
if (parsed.protocol !== "https:" || parsed.hostname !== FULLINDEX_HOST) {
|
|
2123
|
+
dailyIndexInvalid(
|
|
2124
|
+
`edgar_daily_filing_index: constructed URL host/scheme is not https://${FULLINDEX_HOST} (${parsed.protocol}//${parsed.hostname}) — refused before any fetch (fixed-host assertion).`,
|
|
2125
|
+
);
|
|
2126
|
+
}
|
|
2127
|
+
return built;
|
|
2128
|
+
}
|
|
2129
|
+
|
|
2130
|
+
/** Build the quarter's index.json existence-oracle URL (same year/quarter guard). */
|
|
2131
|
+
function buildDailyOracleUrl(year: number, quarter: number): string {
|
|
2132
|
+
const maxYear = currentUtcYear();
|
|
2133
|
+
if (!Number.isInteger(year) || year < EDGAR_DAILY_START_YEAR || year > maxYear) {
|
|
2134
|
+
dailyIndexInvalid(
|
|
2135
|
+
`edgar_daily_filing_index: oracle year ${JSON.stringify(year)} must be an integer in [${EDGAR_DAILY_START_YEAR}, ${maxYear}] — refused before any fetch (path-segment bounds guard).`,
|
|
2136
|
+
);
|
|
2137
|
+
}
|
|
2138
|
+
if (!Number.isInteger(quarter) || quarter < 1 || quarter > 4) {
|
|
2139
|
+
dailyIndexInvalid(
|
|
2140
|
+
`edgar_daily_filing_index: oracle quarter ${JSON.stringify(quarter)} must be an integer in {1,2,3,4} — refused before any fetch (path-segment bounds guard).`,
|
|
2141
|
+
);
|
|
2142
|
+
}
|
|
2143
|
+
const built = `${DAILYINDEX_BASE}/${year}/QTR${quarter}/index.json`;
|
|
2144
|
+
let parsed: URL;
|
|
2145
|
+
try {
|
|
2146
|
+
parsed = new URL(built);
|
|
2147
|
+
} catch {
|
|
2148
|
+
dailyIndexInvalid(
|
|
2149
|
+
`edgar_daily_filing_index: could not construct a valid oracle URL from year=${year} quarter=${quarter} — refused before any fetch.`,
|
|
2150
|
+
);
|
|
2151
|
+
}
|
|
2152
|
+
if (parsed.protocol !== "https:" || parsed.hostname !== FULLINDEX_HOST) {
|
|
2153
|
+
dailyIndexInvalid(
|
|
2154
|
+
`edgar_daily_filing_index: constructed oracle URL host/scheme is not https://${FULLINDEX_HOST} (${parsed.protocol}//${parsed.hostname}) — refused before any fetch (fixed-host assertion).`,
|
|
2155
|
+
);
|
|
2156
|
+
}
|
|
2157
|
+
return built;
|
|
2158
|
+
}
|
|
2159
|
+
|
|
2160
|
+
/**
|
|
2161
|
+
* Parse the raw daily master.YYYYMMDD.idx body into `FilingIndexRow[]` (DEDICATED —
|
|
2162
|
+
* NOT parseFullIndex). DRIFT keys on the ABSENCE of the daily `CIK|Company Name|Form
|
|
2163
|
+
* Type|Date Filed|File Name` header (★ File Name WITH A SPACE) + the `----` dashes
|
|
2164
|
+
* boundary ONLY (M1/fact #2) — a non-index / error / format-changed body served with
|
|
2165
|
+
* HTTP 200 → THROW `driftError`. A body WITH the header+dashes but ZERO data rows is a
|
|
2166
|
+
* GENUINE-EMPTY day → returned (NOT thrown; near-unreachable for a real trading day but
|
|
2167
|
+
* honest). A body WITH the header+dashes whose EVERY data row fails the 5-field split →
|
|
2168
|
+
* THROW `driftError` (all-malformed = format drift). Date Filed is normalized to ISO.
|
|
2169
|
+
*/
|
|
2170
|
+
export function parseDailyIndex(
|
|
2171
|
+
body: string,
|
|
2172
|
+
maxRows: number = MAX_INDEX_ROWS,
|
|
2173
|
+
): ParsedFullIndex {
|
|
2174
|
+
const lines = body.split("\n");
|
|
2175
|
+
// Locate the DAILY header line immediately followed by the `----` dashes boundary.
|
|
2176
|
+
let dashesIdx = -1;
|
|
2177
|
+
for (let i = 0; i < lines.length - 1; i++) {
|
|
2178
|
+
if ((lines[i] ?? "").replace(/\r$/, "").trim() === DAILYINDEX_HEADER) {
|
|
2179
|
+
const next = (lines[i + 1] ?? "").replace(/\r$/, "");
|
|
2180
|
+
if (/^-{5,}\s*$/.test(next.trim())) {
|
|
2181
|
+
dashesIdx = i + 1;
|
|
2182
|
+
break;
|
|
2183
|
+
}
|
|
2184
|
+
}
|
|
2185
|
+
}
|
|
2186
|
+
if (dashesIdx === -1) {
|
|
2187
|
+
throw driftError(
|
|
2188
|
+
DAILYINDEX_LABEL,
|
|
2189
|
+
`edgar:daily-index body is missing the '${DAILYINDEX_HEADER}' header / '----' dashes boundary (a non-index / error / format-changed body served with HTTP 200 — note the daily header is 'File Name' WITH a space, NOT the full-index 'Filename') — refusing to report an empty result (schema drift, NOT a genuine-absent day).`,
|
|
2190
|
+
);
|
|
2191
|
+
}
|
|
2192
|
+
const rows: FilingIndexRow[] = [];
|
|
2193
|
+
let malformedRows = 0;
|
|
2194
|
+
let totalIsLowerBound = false;
|
|
2195
|
+
for (let i = dashesIdx + 1; i < lines.length; i++) {
|
|
2196
|
+
const line = (lines[i] ?? "").replace(/\r$/, "");
|
|
2197
|
+
if (line.trim() === "") continue; // blank line (e.g. the trailing newline) — skip
|
|
2198
|
+
if (rows.length + malformedRows >= maxRows) {
|
|
2199
|
+
totalIsLowerBound = true; // safety ceiling — never a SILENT truncation
|
|
2200
|
+
break;
|
|
2201
|
+
}
|
|
2202
|
+
const parts = splitOnFirstPipes(line, 4); // bounded → a `|`-in-company row stays 5 fields
|
|
2203
|
+
if (parts.length < 5) {
|
|
2204
|
+
malformedRows++;
|
|
2205
|
+
continue;
|
|
2206
|
+
}
|
|
2207
|
+
const cik = str(parts[0]);
|
|
2208
|
+
const filename = str(parts[4]);
|
|
2209
|
+
rows.push({
|
|
2210
|
+
cik,
|
|
2211
|
+
cikPadded: cik === null ? null : padCik(cik),
|
|
2212
|
+
companyName: str(parts[1]),
|
|
2213
|
+
formType: str(parts[2]),
|
|
2214
|
+
dateFiled: normDailyDate(parts[3]), // compact YYYYMMDD → ISO YYYY-MM-DD
|
|
2215
|
+
filename,
|
|
2216
|
+
filingUrl: filename === null ? null : FILING_ARCHIVE_BASE + filename,
|
|
2217
|
+
});
|
|
2218
|
+
}
|
|
2219
|
+
if (rows.length === 0 && malformedRows > 0) {
|
|
2220
|
+
throw driftError(
|
|
2221
|
+
DAILYINDEX_LABEL,
|
|
2222
|
+
`edgar:daily-index has the header/dashes boundary but ALL ${malformedRows} data row(s) failed the 5-field pipe split (format drift) — refusing to report an empty result.`,
|
|
2223
|
+
);
|
|
2224
|
+
}
|
|
2225
|
+
return { rows, malformedRows, totalIsLowerBound };
|
|
2226
|
+
}
|
|
2227
|
+
|
|
2228
|
+
/**
|
|
2229
|
+
* Fetch the whole day's master.YYYYMMDD.idx text (getEdgar VERBATIM + `.text()`),
|
|
2230
|
+
* through the bounded LRU. getEdgar sets the mandatory UA + gzip + the ≤10 req/s gate
|
|
2231
|
+
* + the 15s timeout; a 403 throws (rate_limited/403 — caught tool-locally by the
|
|
2232
|
+
* handler for the oracle disambiguation), so ONLY a 200 body is ever cached. A slow
|
|
2233
|
+
* body-read → abort → getEdgar's honest upstream_unavailable throw (never a fake-empty).
|
|
2234
|
+
*/
|
|
2235
|
+
async function fetchDailyIndexText(built: string, yyyymmdd: string): Promise<string> {
|
|
2236
|
+
const key = yyyymmdd;
|
|
2237
|
+
const now = Date.now();
|
|
2238
|
+
const hit = dailyIndexCache.get(key);
|
|
2239
|
+
if (hit && hit.expiresAt > now) {
|
|
2240
|
+
dailyIndexCache.delete(key); // LRU touch → re-insert as the newest
|
|
2241
|
+
dailyIndexCache.set(key, hit);
|
|
2242
|
+
return hit.text;
|
|
2243
|
+
}
|
|
2244
|
+
if (hit) dailyIndexCache.delete(key); // expired
|
|
2245
|
+
const r = await getEdgar(built, DAILYINDEX_LABEL);
|
|
2246
|
+
const text = await r.text();
|
|
2247
|
+
const ttl = isTodayUtc(yyyymmdd) ? DAILYINDEX_TTL_TODAY_MS : DAILYINDEX_TTL_CLOSED_MS;
|
|
2248
|
+
dailyIndexCache.set(key, { text, expiresAt: now + ttl });
|
|
2249
|
+
while (dailyIndexCache.size > DAILYINDEX_CACHE_MAX) {
|
|
2250
|
+
const oldest = dailyIndexCache.keys().next().value as string | undefined;
|
|
2251
|
+
if (oldest === undefined) break;
|
|
2252
|
+
dailyIndexCache.delete(oldest);
|
|
2253
|
+
}
|
|
2254
|
+
return text;
|
|
2255
|
+
}
|
|
2256
|
+
|
|
2257
|
+
/** For tests: evict the daily-index bounded LRU (mirrors `_resetFullIndexCache`). */
|
|
2258
|
+
export function _resetDailyIndexCache(): void {
|
|
2259
|
+
dailyIndexCache.clear();
|
|
2260
|
+
}
|
|
2261
|
+
|
|
2262
|
+
/**
|
|
2263
|
+
* Consult the quarter's index.json existence oracle (getEdgar; ONLY on the 403 path).
|
|
2264
|
+
* Returns whether `master.<yyyymmdd>.idx` is listed AND `maxListed` = the newest
|
|
2265
|
+
* `YYYYMMDD` among items matching `/^master\.(\d{8})\.idx$/` (M1). Throws on a 403 /
|
|
2266
|
+
* unexpected shape — the caller converts that to the ambiguous both-causes
|
|
2267
|
+
* upstream_unavailable (the oracle itself is inconclusive).
|
|
2268
|
+
*/
|
|
2269
|
+
async function fetchDailyOracle(
|
|
2270
|
+
url: string,
|
|
2271
|
+
yyyymmdd: string,
|
|
2272
|
+
): Promise<{ listed: boolean; maxListed: string | null }> {
|
|
2273
|
+
const r = await getEdgar(url, DAILYINDEX_ORACLE_LABEL);
|
|
2274
|
+
const d = (await r.json()) as { directory?: { item?: Array<{ name?: unknown }> } };
|
|
2275
|
+
const items = d?.directory?.item;
|
|
2276
|
+
if (!Array.isArray(items)) {
|
|
2277
|
+
throw new ToolErrorCarrier({
|
|
2278
|
+
kind: "schema_drift",
|
|
2279
|
+
message: `edgar:daily-index index.json returned an unexpected shape (directory.item[] array missing) — the existence oracle is inconclusive.`,
|
|
2280
|
+
retryable: false,
|
|
2281
|
+
upstreamEndpoint: DAILYINDEX_ORACLE_LABEL,
|
|
2282
|
+
});
|
|
2283
|
+
}
|
|
2284
|
+
const wanted = DAILYINDEX_FILE(yyyymmdd);
|
|
2285
|
+
let listed = false;
|
|
2286
|
+
let maxListed: string | null = null;
|
|
2287
|
+
for (const it of items) {
|
|
2288
|
+
const name = typeof it?.name === "string" ? it.name : "";
|
|
2289
|
+
const mm = /^master\.(\d{8})\.idx$/.exec(name);
|
|
2290
|
+
if (!mm) continue;
|
|
2291
|
+
if (name === wanted) listed = true;
|
|
2292
|
+
const day = mm[1] as string;
|
|
2293
|
+
if (maxListed === null || day > maxListed) maxListed = day;
|
|
2294
|
+
}
|
|
2295
|
+
return { listed, maxListed };
|
|
2296
|
+
}
|
|
2297
|
+
|
|
2298
|
+
/** Honest found:false bundle for the daily-index absent/not-yet-disseminated states. */
|
|
2299
|
+
function dailyAbsentBundle(
|
|
2300
|
+
found: false,
|
|
2301
|
+
date: string,
|
|
2302
|
+
year: number,
|
|
2303
|
+
quarter: number,
|
|
2304
|
+
yyyymmdd: string,
|
|
2305
|
+
complete: boolean,
|
|
2306
|
+
note: string,
|
|
2307
|
+
): MetaBundle {
|
|
2308
|
+
return withMeta(
|
|
2309
|
+
{
|
|
2310
|
+
found,
|
|
2311
|
+
date,
|
|
2312
|
+
year,
|
|
2313
|
+
quarter,
|
|
2314
|
+
indexFile: DAILYINDEX_FILE(yyyymmdd),
|
|
2315
|
+
returned: 0,
|
|
2316
|
+
totalAvailable: 0,
|
|
2317
|
+
filings: [] as FilingIndexRow[],
|
|
2318
|
+
},
|
|
2319
|
+
edgarMeta({
|
|
2320
|
+
returned: 0,
|
|
2321
|
+
totalAvailable: 0,
|
|
2322
|
+
complete,
|
|
2323
|
+
notes: [note, DAILY_SNAPSHOT_NOTE],
|
|
2324
|
+
}),
|
|
2325
|
+
);
|
|
2326
|
+
}
|
|
2327
|
+
|
|
2328
|
+
const DAILY_SNAPSHOT_NOTE =
|
|
2329
|
+
"The daily-index is a point-in-time snapshot of ONE dissemination DAY. For the LATEST or complete-history filings of a KNOWN filer use edgar_company_filings; for a whole quarter use edgar_filing_index; for a text query across 2001-present use edgar_full_text_search.";
|
|
2330
|
+
|
|
2331
|
+
/**
|
|
2332
|
+
* Read the SEC EDGAR daily-index for one calendar `date` and return the filings
|
|
2333
|
+
* matching the given CLIENT-SIDE filters (form / CIK / company substring),
|
|
2334
|
+
* offset-paginated, with the EXACT total match count for the day.
|
|
2335
|
+
*
|
|
2336
|
+
* HONESTY (ADR-0027 v1 + M1 + M2):
|
|
2337
|
+
* - ★M2 — an EXACT date round-trip (Date.UTC component re-extraction) rejects
|
|
2338
|
+
* Feb-30 / day-40 / non-leap-Feb-29 / a malformed / a FUTURE date PRE-fetch
|
|
2339
|
+
* (invalid_input, 0 GET). NOT the `!isNaN(Date.UTC(...))` shortcut (it rolls overflow).
|
|
2340
|
+
* - Fetch the .idx FIRST (happy path pays zero oracle cost). 200 + parseable ⇒
|
|
2341
|
+
* found:true; FULL-SCAN → EXACT totalAvailable (byte-cap forbidden).
|
|
2342
|
+
* - ★M1 — on a 403 (getEdgar mislabels the daily AccessDenied XML rate_limited/403;
|
|
2343
|
+
* caught TOOL-LOCAL), consult index.json; maxListedMasterDate makes it recency-aware:
|
|
2344
|
+
* requestedYyyymmdd > maxListed ⇒ NOT-YET-DISSEMINATED (found:false, complete:FALSE)
|
|
2345
|
+
* requestedYyyymmdd ≤ maxListed & unlisted ⇒ TRUE genuine-absent (found:false, complete:true)
|
|
2346
|
+
* listed but .idx 403'd ⇒ honest rate_limited (retryable ~600s)
|
|
2347
|
+
* oracle itself 403/bad-shape ⇒ ambiguous both-causes upstream_unavailable
|
|
2348
|
+
* - A REAL 429 (status 429, not 403) is NOT caught → stays honest rate_limited.
|
|
2349
|
+
* - CIK stays a STRING; every column via `str`; companyContains is a LITERAL
|
|
2350
|
+
* case-insensitive substring (C110 N/A — no token split).
|
|
2351
|
+
*/
|
|
2352
|
+
export async function dailyFilingIndex(args: {
|
|
2353
|
+
date: string;
|
|
2354
|
+
formType?: string;
|
|
2355
|
+
cik?: string | number;
|
|
2356
|
+
companyContains?: string;
|
|
2357
|
+
limit?: number;
|
|
2358
|
+
offset?: number;
|
|
2359
|
+
}): Promise<MetaBundle> {
|
|
2360
|
+
// ★M2 — EXACT date round-trip PRE-fetch (before the URL builder), 0 GET on failure.
|
|
2361
|
+
const dm = /^(\d{4})-(\d{2})-(\d{2})$/.exec(args.date);
|
|
2362
|
+
if (!dm) {
|
|
2363
|
+
dailyIndexInvalid(
|
|
2364
|
+
`edgar_daily_filing_index: date ${JSON.stringify(args.date)} must be an ISO calendar day YYYY-MM-DD — refused before any fetch.`,
|
|
2365
|
+
);
|
|
2366
|
+
}
|
|
2367
|
+
const y = Number(dm[1]);
|
|
2368
|
+
const mo = Number(dm[2]);
|
|
2369
|
+
const d = Number(dm[3]);
|
|
2370
|
+
// Component re-extraction (NOT !isNaN(Date.UTC(...)) — JS silently rolls Feb-30 →
|
|
2371
|
+
// Mar-01, day-40, non-leap Feb-29, so the naive shortcut would FETCH a nonexistent day).
|
|
2372
|
+
const dt = new Date(Date.UTC(y, mo - 1, d));
|
|
2373
|
+
if (
|
|
2374
|
+
dt.getUTCFullYear() !== y ||
|
|
2375
|
+
dt.getUTCMonth() + 1 !== mo ||
|
|
2376
|
+
dt.getUTCDate() !== d
|
|
2377
|
+
) {
|
|
2378
|
+
dailyIndexInvalid(
|
|
2379
|
+
`edgar_daily_filing_index: date ${JSON.stringify(args.date)} is not a real calendar day (it does not round-trip through Date.UTC — e.g. 2024-02-30, 2024-01-40, or a non-leap 2023-02-29) — refused before any fetch.`,
|
|
2380
|
+
);
|
|
2381
|
+
}
|
|
2382
|
+
const yyyymmdd = `${dm[1]}${dm[2]}${dm[3]}`;
|
|
2383
|
+
const todayYyyymmdd = todayUtcYyyymmdd();
|
|
2384
|
+
if (yyyymmdd > todayYyyymmdd) {
|
|
2385
|
+
// A future day has not happened — you cannot ask for its filings. TODAY is allowed
|
|
2386
|
+
// (reachable; the oracle handles the not-yet-posted case as not-yet-disseminated).
|
|
2387
|
+
dailyIndexInvalid(
|
|
2388
|
+
`edgar_daily_filing_index: date ${args.date} is in the FUTURE (> today ${todayYyyymmdd.slice(0, 4)}-${todayYyyymmdd.slice(4, 6)}-${todayYyyymmdd.slice(6, 8)} UTC) — no filings can exist for a day that has not happened. Refused before any fetch.`,
|
|
2389
|
+
);
|
|
2390
|
+
}
|
|
2391
|
+
const quarter = Math.floor((mo - 1) / 3) + 1;
|
|
2392
|
+
|
|
2393
|
+
// S1 — build + re-validate the path BEFORE any fetch (0 fetch on a bad year/quarter/day).
|
|
2394
|
+
const built = buildDailyIndexUrl(y, quarter, yyyymmdd);
|
|
2395
|
+
|
|
2396
|
+
// Fetch the whole day (getEdgar VERBATIM + `.text()`, bounded-LRU cached). A 403 is
|
|
2397
|
+
// caught TOOL-LOCAL and disambiguated via the index.json oracle (M1) — getEdgar is
|
|
2398
|
+
// UNTOUCHED (the other 6 edgar tools stay byte-identical).
|
|
2399
|
+
let body: string;
|
|
2400
|
+
try {
|
|
2401
|
+
body = await fetchDailyIndexText(built, yyyymmdd);
|
|
2402
|
+
} catch (e) {
|
|
2403
|
+
if (
|
|
2404
|
+
e instanceof ToolErrorCarrier &&
|
|
2405
|
+
e.toolError.kind === "rate_limited" &&
|
|
2406
|
+
e.toolError.upstreamStatus === 403
|
|
2407
|
+
) {
|
|
2408
|
+
// ── ★M1 403-DISAMBIGUATION via the index.json existence oracle (recency-aware) ──
|
|
2409
|
+
const oracleUrl = buildDailyOracleUrl(y, quarter);
|
|
2410
|
+
let oracle: { listed: boolean; maxListed: string | null };
|
|
2411
|
+
try {
|
|
2412
|
+
oracle = await fetchDailyOracle(oracleUrl, yyyymmdd);
|
|
2413
|
+
} catch {
|
|
2414
|
+
// The oracle ITSELF 403'd / returned an unexpected shape ⇒ INCONCLUSIVE.
|
|
2415
|
+
throw new ToolErrorCarrier({
|
|
2416
|
+
kind: "upstream_unavailable",
|
|
2417
|
+
message: `SEC returned HTTP 403 for the daily-index ${DAILYINDEX_BASE}/${y}/QTR${quarter}/${DAILYINDEX_FILE(yyyymmdd)}, and the index.json existence oracle could not be consulted (it too 403'd / returned an unexpected shape). This is AMBIGUOUS: EITHER ${args.date} has no published daily index OR SEC is rate-limiting this IP at the 10 req/s ceiling (~10-minute block). NOT fabricated as empty — retry after ~10 minutes.`,
|
|
2418
|
+
retryable: true,
|
|
2419
|
+
retryAfterSeconds: 600,
|
|
2420
|
+
upstreamStatus: 403,
|
|
2421
|
+
upstreamEndpoint: DAILYINDEX_LABEL,
|
|
2422
|
+
});
|
|
2423
|
+
}
|
|
2424
|
+
if (oracle.maxListed === null || yyyymmdd > oracle.maxListed) {
|
|
2425
|
+
// ★M1 branch (a) — NEWER than anything the oracle has published yet (covers today
|
|
2426
|
+
// AND unlisted recent trading day(s) across weekends/holidays) ⇒ NOT-YET-DISSEMINATED.
|
|
2427
|
+
// NEVER complete:true — a day full of 8-Ks must not read as a confident empty.
|
|
2428
|
+
return dailyAbsentBundle(
|
|
2429
|
+
false,
|
|
2430
|
+
args.date,
|
|
2431
|
+
y,
|
|
2432
|
+
quarter,
|
|
2433
|
+
yyyymmdd,
|
|
2434
|
+
false, // complete:FALSE
|
|
2435
|
+
`${args.date} is NEWER than the newest daily index EDGAR has published for this quarter (${oracle.maxListed ? `${oracle.maxListed.slice(0, 4)}-${oracle.maxListed.slice(4, 6)}-${oracle.maxListed.slice(6, 8)}` : "none listed yet"}). EDGAR's daily index and its index.json listing LAG real filing activity and may not be posted for the most recent trading day(s), especially across weekends/holidays — this day may simply not be disseminated yet; retry later (EDGAR posts each day's index around 22:00 US-Eastern). This is NOT a confirmed empty day (complete:false).`,
|
|
2436
|
+
);
|
|
2437
|
+
}
|
|
2438
|
+
if (!oracle.listed) {
|
|
2439
|
+
// ★M1 branch (b) — requestedYyyymmdd ≤ maxListed AND still not listed ⇒ a TRUE
|
|
2440
|
+
// gap INSIDE the covered range (weekend / holiday / genuinely no-dissemination day).
|
|
2441
|
+
return dailyAbsentBundle(
|
|
2442
|
+
false,
|
|
2443
|
+
args.date,
|
|
2444
|
+
y,
|
|
2445
|
+
quarter,
|
|
2446
|
+
yyyymmdd,
|
|
2447
|
+
true, // complete:true — an HONEST genuine-absent, NOT an error/drift/fake-empty
|
|
2448
|
+
`master.${yyyymmdd}.idx is NOT published for ${args.date}: it is not listed in the quarter's index.json existence oracle, yet the oracle DOES list newer day(s) (newest ${oracle.maxListed.slice(0, 4)}-${oracle.maxListed.slice(4, 6)}-${oracle.maxListed.slice(6, 8)}) — so this is a genuine gap INSIDE the covered range (a weekend / holiday / no-dissemination day). This is an HONEST genuine-absent answer (found:false, complete:true), NOT an error, NOT a rate-block, NOT a fabricated empty.`,
|
|
2449
|
+
);
|
|
2450
|
+
}
|
|
2451
|
+
// listed === true but the .idx 403'd ⇒ the index EXISTS, SEC rate-blocked this IP.
|
|
2452
|
+
throw new ToolErrorCarrier({
|
|
2453
|
+
kind: "rate_limited",
|
|
2454
|
+
message: `The daily index master.${yyyymmdd}.idx for ${args.date} EXISTS (it IS listed in the quarter's index.json oracle) but the .idx fetch returned HTTP 403 — SEC is rate-limiting this IP at the 10 req/s ceiling (or a transient edge block). Slow down and retry after ~10 minutes. NOT a fake-empty, NOT a genuine-absent day.`,
|
|
2455
|
+
retryable: true,
|
|
2456
|
+
retryAfterSeconds: 600,
|
|
2457
|
+
upstreamStatus: 403,
|
|
2458
|
+
upstreamEndpoint: DAILYINDEX_LABEL,
|
|
2459
|
+
});
|
|
2460
|
+
}
|
|
2461
|
+
throw e; // schema_drift / invalid_input(UA) / upstream_unavailable / real 429 — loud
|
|
2462
|
+
}
|
|
2463
|
+
|
|
2464
|
+
// 200 path — FULL-SCAN → parse ALL rows past the preamble. Drift (header-absent /
|
|
2465
|
+
// all-malformed) THROWS; header+0-rows ⇒ genuine-empty (found:true, totalAvailable:0).
|
|
2466
|
+
const parsed = parseDailyIndex(body);
|
|
2467
|
+
const all = parsed.rows;
|
|
2468
|
+
|
|
2469
|
+
// CLIENT-SIDE filters — ZERO query string (none reach the URL). cik via padCik
|
|
2470
|
+
// both-sides; formType case-insensitive EXACT on col3 ('8-K' ≠ '8-K/A');
|
|
2471
|
+
// companyContains a LITERAL case-insensitive substring (no token split, C110 N/A).
|
|
2472
|
+
const formNeedle = args.formType?.trim() ? args.formType.trim().toLowerCase() : null;
|
|
2473
|
+
const cikFilter =
|
|
2474
|
+
args.cik != null && String(args.cik).trim() !== "" ? padCik(args.cik) : null;
|
|
2475
|
+
const companyNeedle = args.companyContains?.trim()
|
|
2476
|
+
? args.companyContains.trim().toLowerCase()
|
|
2477
|
+
: null;
|
|
2478
|
+
|
|
2479
|
+
const matches = all.filter((row) => {
|
|
2480
|
+
if (formNeedle !== null) {
|
|
2481
|
+
if (row.formType === null || row.formType.toLowerCase() !== formNeedle) return false;
|
|
2482
|
+
}
|
|
2483
|
+
if (cikFilter !== null) {
|
|
2484
|
+
if (row.cikPadded !== cikFilter) return false;
|
|
2485
|
+
}
|
|
2486
|
+
if (companyNeedle !== null) {
|
|
2487
|
+
if (
|
|
2488
|
+
row.companyName === null ||
|
|
2489
|
+
!row.companyName.toLowerCase().includes(companyNeedle)
|
|
2490
|
+
)
|
|
2491
|
+
return false;
|
|
2492
|
+
}
|
|
2493
|
+
return true;
|
|
2494
|
+
});
|
|
2495
|
+
|
|
2496
|
+
const limit = args.limit ?? 100;
|
|
2497
|
+
const offset = args.offset ?? 0;
|
|
2498
|
+
const totalAvailable = matches.length; // EXACT (full-scan) — never a page length
|
|
2499
|
+
const page = matches.slice(offset, offset + limit);
|
|
2500
|
+
const returned = page.length;
|
|
2501
|
+
const hasMore = offset + returned < totalAvailable;
|
|
2502
|
+
const nextOffset = hasMore ? offset + returned : null;
|
|
2503
|
+
|
|
2504
|
+
const filtersApplied: string[] = [];
|
|
2505
|
+
if (formNeedle !== null) filtersApplied.push("formType");
|
|
2506
|
+
if (cikFilter !== null) filtersApplied.push("cik");
|
|
2507
|
+
if (companyNeedle !== null) filtersApplied.push("companyContains");
|
|
2508
|
+
|
|
2509
|
+
const notes: string[] = [];
|
|
2510
|
+
notes.push(DAILY_SNAPSHOT_NOTE);
|
|
2511
|
+
notes.push(
|
|
2512
|
+
`The whole ${args.date} daily index was downloaded and FULL-SCANNED; totalAvailable (${totalAvailable}) is the EXACT count of filings matching the filters across the ENTIRE day (not a page length, not a byte-capped subset — SEC ignores HTTP Range). This page contains ${returned} of ${totalAvailable}; page via _meta.pagination.nextOffset for the rest.`,
|
|
2513
|
+
);
|
|
2514
|
+
notes.push(
|
|
2515
|
+
`dateFiled is normalized to ISO YYYY-MM-DD from the index's compact YYYYMMDD column; every row in this file shares the requested day (${args.date}).`,
|
|
2516
|
+
);
|
|
2517
|
+
if (isTodayUtc(yyyymmdd)) {
|
|
2518
|
+
notes.push(
|
|
2519
|
+
`${args.date} is TODAY (UTC) — EDGAR posts each day's index around 22:00 US-Eastern and it GROWS as filings disseminate, so totalAvailable is EXACT AS-OF this (short-cached) snapshot, not exact-forever. A closed past day is immutable.`,
|
|
2520
|
+
);
|
|
2521
|
+
}
|
|
2522
|
+
if (companyNeedle !== null) {
|
|
2523
|
+
notes.push(
|
|
2524
|
+
`companyContains is a case-insensitive LITERAL substring match on the Company Name column — a multi-word value matches as ONE contiguous string, NOT AND/OR-tokenized.`,
|
|
2525
|
+
);
|
|
2526
|
+
}
|
|
2527
|
+
if (formNeedle !== null) {
|
|
2528
|
+
notes.push(
|
|
2529
|
+
`formType is a case-insensitive EXACT match on the Form Type column ("${args.formType?.trim()}" matches that form only — e.g. "8-K" does NOT match "8-K/A"). Pass each amendment variant separately.`,
|
|
2530
|
+
);
|
|
2531
|
+
}
|
|
2532
|
+
if (parsed.malformedRows > 0) {
|
|
2533
|
+
notes.push(
|
|
2534
|
+
`${parsed.malformedRows} row(s) did not split into 5 pipe fields and were skipped (tolerated as stray malformed rows; a body with ZERO valid rows would instead be refused as schema drift).`,
|
|
2535
|
+
);
|
|
2536
|
+
}
|
|
2537
|
+
if (parsed.totalIsLowerBound) {
|
|
2538
|
+
notes.push(
|
|
2539
|
+
`The scan hit the MAX_INDEX_ROWS safety ceiling (${MAX_INDEX_ROWS}); totals are a LOWER BOUND — the daily index is larger than expected (possible format drift). See totalIsLowerBound.`,
|
|
2540
|
+
);
|
|
2541
|
+
}
|
|
2542
|
+
if (totalAvailable === 0) {
|
|
2543
|
+
notes.push(
|
|
2544
|
+
filtersApplied.length
|
|
2545
|
+
? `0 filings on ${args.date} matched the filters (${filtersApplied.join(", ")}). This is an EXACT ZERO over the full day index (found:true — the day IS published) — NOT a truncation, NOT an outage, NOT a genuine-absent day.`
|
|
2546
|
+
: `0 data rows in the ${args.date} daily index (found:true — the day IS published, but its index has no filing rows). This is an EXACT ZERO over the full day index — NOT a truncation, NOT an outage.`,
|
|
2547
|
+
);
|
|
2548
|
+
}
|
|
2549
|
+
|
|
2550
|
+
const meta: Partial<ResponseMeta> = {
|
|
2551
|
+
returned,
|
|
2552
|
+
totalAvailable,
|
|
2553
|
+
filtersApplied,
|
|
2554
|
+
pagination: { offset, limit, hasMore, nextOffset },
|
|
2555
|
+
notes,
|
|
2556
|
+
};
|
|
2557
|
+
if (parsed.totalIsLowerBound) meta.totalIsLowerBound = true;
|
|
2558
|
+
|
|
2559
|
+
return withMeta(
|
|
2560
|
+
{
|
|
2561
|
+
found: true,
|
|
2562
|
+
date: args.date,
|
|
2563
|
+
year: y,
|
|
2564
|
+
quarter,
|
|
2565
|
+
indexFile: DAILYINDEX_FILE(yyyymmdd),
|
|
2566
|
+
returned,
|
|
2567
|
+
totalAvailable,
|
|
2568
|
+
filings: page,
|
|
2569
|
+
},
|
|
2570
|
+
edgarMeta(meta),
|
|
2571
|
+
);
|
|
2572
|
+
}
|
|
2573
|
+
|
|
2574
|
+
// ─── Tool 8: edgar_company_concept (ADR-0041) ─────────────────────
|
|
2575
|
+
// ONE filer × ONE XBRL concept × the COMPLETE reported time-series (with the
|
|
2576
|
+
// amendment/restatement history + multi-unit disclosure) — the focused
|
|
2577
|
+
// financial-TREND / entity-vetting primitive that sits BETWEEN company_facts
|
|
2578
|
+
// (all curated concepts for one filer) and edgar_xbrl_frames (one concept across
|
|
2579
|
+
// ALL filers for one period). Reuses the C105/C106-hardened edgar.ts adapter
|
|
2580
|
+
// VERBATIM (getEdgar transport + self-throttle gate + UA + 403 disambiguation,
|
|
2581
|
+
// padCik + resolveCik, num/str null-never-0, edgarMeta, notFoundBundle, withMeta,
|
|
2582
|
+
// FRAMES_TAG_RE) — getEdgar UNTOUCHED, so the other 7 edgar tools stay byte-identical.
|
|
2583
|
+
//
|
|
2584
|
+
// LIVE-pinned (ADR-0041, 5 keyless GETs, SEC UA, data.sec.gov):
|
|
2585
|
+
// GET https://data.sec.gov/api/xbrl/companyconcept/CIK{cik10}/{taxonomy}/{Concept}.json
|
|
2586
|
+
// → { cik, taxonomy, tag, label, description, entityName, units:{ "USD":[…], "shares":[…] } }
|
|
2587
|
+
// Each unit key maps to its OWN row array; each row { start?, end, val, accn, fy, fp,
|
|
2588
|
+
// form, filed, (frame?) }. `unit` is a BODY key (filtered CLIENT-SIDE), NOT a path
|
|
2589
|
+
// segment. 404 (bad CIK/taxonomy/concept) → XML NoSuchKey → getEdgar throws not_found
|
|
2590
|
+
// BEFORE any JSON parse → notFoundBundle (never a fabricated val:0).
|
|
2591
|
+
//
|
|
2592
|
+
// ★M1 (period identity = the (start,end) PAIR, not `end` alone): every output row
|
|
2593
|
+
// carries `start` (null for INSTANT concepts, the ISO date for DURATION concepts).
|
|
2594
|
+
// The SAME `end` with a DIFFERENT `start` is a different-duration fact (a 3-month
|
|
2595
|
+
// quarter vs the 12-month year — LIVE: Apple NetIncomeLoss end=2009-09-26 carries
|
|
2596
|
+
// BOTH CY2009 (start 2008-09-28) + CY2009Q3 (start 2009-06-28), both frame-tagged
|
|
2597
|
+
// canonical), NOT a revision. A revision is ONLY multiple rows sharing the SAME
|
|
2598
|
+
// (start,end) with a differing accn/filed/val.
|
|
2599
|
+
// ★M2 (canonicalOnly dedup key = (unit,start,end)): partition by unit FIRST, then keep
|
|
2600
|
+
// ONE row per distinct (start,end) — never collapsing a whole unit's row.
|
|
2601
|
+
// ★S1 (unitsAvailable[].count = the RAW units[key].length, pre-filter).
|
|
2602
|
+
// ★S2 (CONCEPT_TAXONOMIES = {us-gaap, dei, ifrs-full} — all three live-confirmed; srt DROPPED).
|
|
2603
|
+
|
|
2604
|
+
/** The companyconcept endpoint base (same data.sec.gov the other edgar tools use). */
|
|
2605
|
+
const CONCEPT_BASE = "https://data.sec.gov/api/xbrl/companyconcept";
|
|
2606
|
+
|
|
2607
|
+
/**
|
|
2608
|
+
* The `taxonomy` path-segment enum — the SSRF guard for that segment (no free value
|
|
2609
|
+
* reaches the host). us-gaap + dei live-confirmed on Apple; ifrs-full live-confirmed on
|
|
2610
|
+
* Spotify (CIK0001639920 / ifrs-full / Assets → 200, unit EUR, frame CY2017Q4I). `srt`
|
|
2611
|
+
* is DROPPED (S2 — it was NOT probed to a resolving 200; a valid-but-unreported tuple is
|
|
2612
|
+
* an honest 404, so a slightly-broad enum can never fabricate — but ship only confirmed).
|
|
2613
|
+
*/
|
|
2614
|
+
export const CONCEPT_TAXONOMIES = ["us-gaap", "dei", "ifrs-full"] as const;
|
|
2615
|
+
|
|
2616
|
+
/** A 10-digit CIK path segment (post-padCik; rejects an 11-digit overflow). */
|
|
2617
|
+
const CIK10_RE = /^\d{10}$/;
|
|
2618
|
+
|
|
2619
|
+
/** Throw the pre-fetch injection-guard error (invalid_input, 0 fetch). */
|
|
2620
|
+
function conceptInvalid(message: string): never {
|
|
2621
|
+
throw new ToolErrorCarrier({
|
|
2622
|
+
kind: "invalid_input",
|
|
2623
|
+
message,
|
|
2624
|
+
retryable: false,
|
|
2625
|
+
upstreamEndpoint: "edgar:companyconcept",
|
|
2626
|
+
});
|
|
2627
|
+
}
|
|
2628
|
+
|
|
2629
|
+
/**
|
|
2630
|
+
* Build the companyconcept URL from the THREE validated path segments (COPIES the
|
|
2631
|
+
* buildFramesUrl S1/S2 doctrine — with THREE segments, NOT four; `unit` is a BODY key,
|
|
2632
|
+
* never a path segment). BELT-AND-SUSPENDERS: re-run the cik regex + the taxonomy enum
|
|
2633
|
+
* + the concept regex and hard-throw invalid_input (0 fetch) on any mismatch — do NOT
|
|
2634
|
+
* trust that Zod already ran, and do NOT rely on the hostname assertion alone (it passes
|
|
2635
|
+
* a same-host `../` traversal). THEN assert the built URL is https on the fixed
|
|
2636
|
+
* data.sec.gov host (guards host-escape/downgrade).
|
|
2637
|
+
*/
|
|
2638
|
+
function buildConceptUrl(cik10: string, taxonomy: string, tag: string): string {
|
|
2639
|
+
if (!CIK10_RE.test(cik10)) {
|
|
2640
|
+
conceptInvalid(
|
|
2641
|
+
`edgar_company_concept: CIK ${JSON.stringify(cik10)} must be exactly 10 digits after padding (an 11-digit overflow is rejected) — refused before any fetch (path-segment injection guard).`,
|
|
2642
|
+
);
|
|
2643
|
+
}
|
|
2644
|
+
if (!(CONCEPT_TAXONOMIES as readonly string[]).includes(taxonomy)) {
|
|
2645
|
+
conceptInvalid(
|
|
2646
|
+
`edgar_company_concept: taxonomy ${JSON.stringify(taxonomy)} is not one of {${CONCEPT_TAXONOMIES.join(", ")}} — refused before any fetch (path-segment injection guard).`,
|
|
2647
|
+
);
|
|
2648
|
+
}
|
|
2649
|
+
if (!FRAMES_TAG_RE.test(tag)) {
|
|
2650
|
+
conceptInvalid(
|
|
2651
|
+
`edgar_company_concept: concept ${JSON.stringify(tag)} must match ^[A-Za-z0-9]+$ (XBRL tags are alphanumeric CamelCase, e.g. NetIncomeLoss; slash/dot/percent/backslash/'..'/%2F/%2E/%00 are rejected) — refused before any fetch (path-segment injection guard).`,
|
|
2652
|
+
);
|
|
2653
|
+
}
|
|
2654
|
+
const built = `${CONCEPT_BASE}/CIK${cik10}/${taxonomy}/${tag}.json`;
|
|
2655
|
+
let parsed: URL;
|
|
2656
|
+
try {
|
|
2657
|
+
parsed = new URL(built);
|
|
2658
|
+
} catch {
|
|
2659
|
+
conceptInvalid(
|
|
2660
|
+
`edgar_company_concept: could not construct a valid URL from the segments — refused before any fetch.`,
|
|
2661
|
+
);
|
|
2662
|
+
}
|
|
2663
|
+
if (parsed.protocol !== "https:" || parsed.hostname !== "data.sec.gov") {
|
|
2664
|
+
conceptInvalid(
|
|
2665
|
+
`edgar_company_concept: constructed URL host/scheme is not https://data.sec.gov (${parsed.protocol}//${parsed.hostname}) — refused before any fetch (fixed-host assertion).`,
|
|
2666
|
+
);
|
|
2667
|
+
}
|
|
2668
|
+
return built;
|
|
2669
|
+
}
|
|
2670
|
+
|
|
2671
|
+
/** One output row: unit-tagged, (start,end)-keyed, canonical = frame present. */
|
|
2672
|
+
export type ConceptRow = {
|
|
2673
|
+
unit: string;
|
|
2674
|
+
start: string | null; // null for INSTANT concepts; the ISO period-start for DURATION concepts (M1)
|
|
2675
|
+
end: string | null;
|
|
2676
|
+
val: number | null; // num() null-never-0 (a real reported 0 survives)
|
|
2677
|
+
accn: string | null;
|
|
2678
|
+
fy: number | null;
|
|
2679
|
+
fp: string | null;
|
|
2680
|
+
form: string | null;
|
|
2681
|
+
filed: string | null;
|
|
2682
|
+
frame: string | null; // SEC's canonical-consolidation tag, verbatim (null when absent)
|
|
2683
|
+
canonical: boolean; // frame != null — SEC's OWN consolidated-value marker
|
|
2684
|
+
};
|
|
2685
|
+
|
|
2686
|
+
type ConceptRowRaw = {
|
|
2687
|
+
start?: unknown;
|
|
2688
|
+
end?: unknown;
|
|
2689
|
+
val?: unknown;
|
|
2690
|
+
accn?: unknown;
|
|
2691
|
+
fy?: unknown;
|
|
2692
|
+
fp?: unknown;
|
|
2693
|
+
form?: unknown;
|
|
2694
|
+
filed?: unknown;
|
|
2695
|
+
frame?: unknown;
|
|
2696
|
+
};
|
|
2697
|
+
type ConceptBody = {
|
|
2698
|
+
cik?: unknown;
|
|
2699
|
+
taxonomy?: unknown;
|
|
2700
|
+
tag?: unknown;
|
|
2701
|
+
label?: unknown;
|
|
2702
|
+
description?: unknown;
|
|
2703
|
+
entityName?: unknown;
|
|
2704
|
+
units?: unknown;
|
|
2705
|
+
};
|
|
2706
|
+
|
|
2707
|
+
/**
|
|
2708
|
+
* The all-rows default disclosure note (M1-corrected). Rides on EVERY found:true
|
|
2709
|
+
* response. The load-bearing correction vs v1: for flow/duration concepts the same
|
|
2710
|
+
* `end` with a different `start` is a DIFFERENT-DURATION fact, NOT a revision.
|
|
2711
|
+
*/
|
|
2712
|
+
const AMENDMENT_DISCLOSURE_NOTE =
|
|
2713
|
+
"Rows are SEC's COMPLETE reported history for this concept, each tagged with its unit. Period identity is the (start,end) PAIR: `start` is null for INSTANT (balance-sheet) concepts and the ISO period-start date for DURATION (flow) concepts. For flow/duration concepts the same `end` with a different `start` is a different-duration fact (a 3-month quarter vs the 12-month year), NOT a revision; a revision is only multiple rows sharing the same (start,end) with different accn/filed/val — read the filed dates. `canonical:true` (a `frame` tag present) marks SEC's consolidated value for that (start,end) period; `canonical:false` rows are earlier/superseded/intra-year reports. Values are surfaced verbatim (val is null-never-0); none is recomputed.";
|
|
2714
|
+
|
|
2715
|
+
/** The (unit,start,end) group key (M1/M2). A space joiner keeps the three fields distinct. */
|
|
2716
|
+
function periodKey(r: ConceptRow): string {
|
|
2717
|
+
return `${r.unit} ${r.start ?? ""} ${r.end ?? ""}`;
|
|
2718
|
+
}
|
|
2719
|
+
|
|
2720
|
+
/**
|
|
2721
|
+
* Deterministic canonicalOnly tiebreak within a (unit,start,end) group: prefer a
|
|
2722
|
+
* frame-tagged row, then max `filed`, then max `accn`. ALWAYS a verbatim selection of
|
|
2723
|
+
* an existing row (never a merge/recompute).
|
|
2724
|
+
*/
|
|
2725
|
+
function preferCanonical(a: ConceptRow, b: ConceptRow): ConceptRow {
|
|
2726
|
+
const af = a.frame != null;
|
|
2727
|
+
const bf = b.frame != null;
|
|
2728
|
+
if (af !== bf) return af ? a : b;
|
|
2729
|
+
const afiled = a.filed ?? "";
|
|
2730
|
+
const bfiled = b.filed ?? "";
|
|
2731
|
+
if (afiled !== bfiled) return afiled > bfiled ? a : b;
|
|
2732
|
+
const aaccn = a.accn ?? "";
|
|
2733
|
+
const baccn = b.accn ?? "";
|
|
2734
|
+
return aaccn >= baccn ? a : b;
|
|
2735
|
+
}
|
|
2736
|
+
|
|
2737
|
+
/**
|
|
2738
|
+
* One filer × one XBRL concept × the COMPLETE reported time-series. Reuses resolveCik
|
|
2739
|
+
* (ticker→CIK path EXISTS) + buildConceptUrl (the frames path-segment SSRF doctrine,
|
|
2740
|
+
* THREE segments) + getEdgar VERBATIM. `unit`/`form`/`fy` are CLIENT-SIDE filters;
|
|
2741
|
+
* `canonicalOnly` (default false) dedups to one canonical row per (unit,start,end),
|
|
2742
|
+
* FULLY DISCLOSED. limit/offset window the already-fully-fetched set.
|
|
2743
|
+
*
|
|
2744
|
+
* HONESTY (ADR-0041 v2):
|
|
2745
|
+
* - ★M1 — every row carries `start`; period identity is the (start,end) PAIR. A
|
|
2746
|
+
* same-`end` different-`start` pair is a different-duration fact, NOT a revision.
|
|
2747
|
+
* - ★M2 — canonicalOnly dedup key = (unit,start,end): partition by unit first, keep
|
|
2748
|
+
* one canonical row per distinct (start,end) — never dropping a whole unit's row.
|
|
2749
|
+
* - ★S1 — unitsAvailable[].count = the RAW units[key].length (pre-filter).
|
|
2750
|
+
* - val null-never-0 via num(); every row unit-tagged (no USD↔shares conflation).
|
|
2751
|
+
* - 404 (bad CIK/taxonomy/concept) → notFoundBundle (never a fabricated val:0);
|
|
2752
|
+
* 5xx/timeout/non-JSON/units-shape-drift → THROW; a bad `unit` filter → honest empty
|
|
2753
|
+
* + the available-units note (unit is CLIENT-SIDE, never a path segment).
|
|
2754
|
+
*/
|
|
2755
|
+
export async function companyConcept(args: {
|
|
2756
|
+
cikOrTicker: string;
|
|
2757
|
+
concept: string;
|
|
2758
|
+
taxonomy?: string;
|
|
2759
|
+
unit?: string;
|
|
2760
|
+
form?: string;
|
|
2761
|
+
fy?: number;
|
|
2762
|
+
canonicalOnly?: boolean;
|
|
2763
|
+
limit?: number;
|
|
2764
|
+
offset?: number;
|
|
2765
|
+
}): Promise<MetaBundle> {
|
|
2766
|
+
const resolved = await resolveCik(args.cikOrTicker);
|
|
2767
|
+
if (!resolved) {
|
|
2768
|
+
return notFoundBundle(
|
|
2769
|
+
args.cikOrTicker,
|
|
2770
|
+
`Could not resolve "${args.cikOrTicker}" to a CIK (not a numeric CIK and no exact-ticker/title match in company_tickers.json).`,
|
|
2771
|
+
);
|
|
2772
|
+
}
|
|
2773
|
+
const cik = resolved.cik;
|
|
2774
|
+
const taxonomy = args.taxonomy ?? "us-gaap";
|
|
2775
|
+
const concept = args.concept;
|
|
2776
|
+
|
|
2777
|
+
// S1/S2 — build (and re-validate) the path BEFORE any fetch. Throws invalid_input
|
|
2778
|
+
// with 0 fetches on any bad segment or a non-fixed-host URL.
|
|
2779
|
+
const url = buildConceptUrl(cik, taxonomy, concept);
|
|
2780
|
+
|
|
2781
|
+
let body: ConceptBody;
|
|
2782
|
+
try {
|
|
2783
|
+
const r = await getEdgar(url, "edgar:companyconcept");
|
|
2784
|
+
// A 200 body that is NOT valid JSON (an HTML/XML outage slipping through with 200)
|
|
2785
|
+
// → SyntaxError → schema_drift THROW (ADR-0003 doctrine), never a fake empty.
|
|
2786
|
+
try {
|
|
2787
|
+
body = (await r.json()) as ConceptBody;
|
|
2788
|
+
} catch {
|
|
2789
|
+
throw new ToolErrorCarrier({
|
|
2790
|
+
kind: "schema_drift",
|
|
2791
|
+
message: `edgar:companyconcept returned an HTTP 200 body that is not valid JSON for CIK ${cik} / ${taxonomy} / ${concept} — an outage/error page masquerading as a 200. Refusing rather than fabricating an empty result.`,
|
|
2792
|
+
retryable: false,
|
|
2793
|
+
upstreamEndpoint: "edgar:companyconcept",
|
|
2794
|
+
});
|
|
2795
|
+
}
|
|
2796
|
+
} catch (e) {
|
|
2797
|
+
// 404 (bad CIK / bad taxonomy / a concept the filer never reported) ⇒ getEdgar throws
|
|
2798
|
+
// not_found BEFORE any .json() (the XML NoSuchKey body is never parsed) ⇒ notFoundBundle.
|
|
2799
|
+
if (e instanceof ToolErrorCarrier && e.toolError.kind === "not_found") {
|
|
2800
|
+
return notFoundBundle(
|
|
2801
|
+
`${cik}/${taxonomy}/${concept}`,
|
|
2802
|
+
`No XBRL companyconcept matched CIK ${cik} / ${taxonomy} / ${concept} (HTTP 404). The filer did NOT report this concept under this exact taxonomy — this is NOT a value of 0. Check the tag spelling/case (XBRL tags are CamelCase, e.g. NetIncomeLoss) and the taxonomy (us-gaap vs dei vs ifrs-full).`,
|
|
2803
|
+
);
|
|
2804
|
+
}
|
|
2805
|
+
throw e;
|
|
2806
|
+
}
|
|
2807
|
+
|
|
2808
|
+
// Drift guard: `units` MISSING / not an object / an array ⇒ schema_drift THROW (the
|
|
2809
|
+
// envelope changed) — never a fabricated empty.
|
|
2810
|
+
const units = body.units;
|
|
2811
|
+
if (units === null || typeof units !== "object" || Array.isArray(units)) {
|
|
2812
|
+
throw new ToolErrorCarrier({
|
|
2813
|
+
kind: "schema_drift",
|
|
2814
|
+
message: `edgar:companyconcept returned HTTP 200 without a units{} object for CIK ${cik} / ${taxonomy} / ${concept} — the companyconcept envelope changed.`,
|
|
2815
|
+
retryable: false,
|
|
2816
|
+
upstreamEndpoint: "edgar:companyconcept",
|
|
2817
|
+
});
|
|
2818
|
+
}
|
|
2819
|
+
const unitsObj = units as Record<string, unknown>;
|
|
2820
|
+
const unitKeys = Object.keys(unitsObj);
|
|
2821
|
+
|
|
2822
|
+
// ★S1 — unitsAvailable[].count = the RAW per-unit array length, computed BEFORE any
|
|
2823
|
+
// unit/form/fy/canonicalOnly/limit/offset filtering (a filtered count is a masquerade).
|
|
2824
|
+
const unitsAvailable = unitKeys.map((k) => ({
|
|
2825
|
+
unit: k,
|
|
2826
|
+
count: Array.isArray(unitsObj[k]) ? (unitsObj[k] as unknown[]).length : 0,
|
|
2827
|
+
}));
|
|
2828
|
+
|
|
2829
|
+
// Build ALL rows, TAGGED with unit (P3 — never conflate USD with shares). Iterate the
|
|
2830
|
+
// unit keys in a DETERMINISTIC order (ascending) for a stable window; each unit's rows
|
|
2831
|
+
// stay in SEC's returned order (no blind re-sort — the edgar_company_filings doctrine).
|
|
2832
|
+
const sortedUnitKeys = [...unitKeys].sort();
|
|
2833
|
+
const allRows: ConceptRow[] = [];
|
|
2834
|
+
for (const uKey of sortedUnitKeys) {
|
|
2835
|
+
const arr = unitsObj[uKey];
|
|
2836
|
+
if (!Array.isArray(arr)) continue; // defensive — a non-array unit contributes no rows
|
|
2837
|
+
for (const raw of arr as ConceptRowRaw[]) {
|
|
2838
|
+
const frame = str(raw.frame);
|
|
2839
|
+
allRows.push({
|
|
2840
|
+
unit: uKey,
|
|
2841
|
+
start: str(raw.start), // null for INSTANT concepts; ISO date for DURATION (M1)
|
|
2842
|
+
end: str(raw.end),
|
|
2843
|
+
val: num(raw.val), // null-never-0
|
|
2844
|
+
accn: str(raw.accn),
|
|
2845
|
+
fy: typeof raw.fy === "number" ? raw.fy : null,
|
|
2846
|
+
fp: str(raw.fp),
|
|
2847
|
+
form: str(raw.form),
|
|
2848
|
+
filed: str(raw.filed),
|
|
2849
|
+
frame,
|
|
2850
|
+
canonical: frame != null,
|
|
2851
|
+
});
|
|
2852
|
+
}
|
|
2853
|
+
}
|
|
2854
|
+
|
|
2855
|
+
// Defensive empty (Q5): a 200 whose units{} has ZERO keys or ALL-empty arrays ⇒ an
|
|
2856
|
+
// honest empty (found:false, complete:true), NOT a crash and NOT a fabricated row.
|
|
2857
|
+
if (allRows.length === 0) {
|
|
2858
|
+
return withMeta(
|
|
2859
|
+
{
|
|
2860
|
+
found: false,
|
|
2861
|
+
cik,
|
|
2862
|
+
entityName: str(body.entityName),
|
|
2863
|
+
taxonomy,
|
|
2864
|
+
concept,
|
|
2865
|
+
label: str(body.label),
|
|
2866
|
+
description: str(body.description),
|
|
2867
|
+
unitsAvailable,
|
|
2868
|
+
rows: [] as ConceptRow[],
|
|
2869
|
+
},
|
|
2870
|
+
edgarMeta({
|
|
2871
|
+
returned: 0,
|
|
2872
|
+
totalAvailable: 0,
|
|
2873
|
+
complete: true,
|
|
2874
|
+
filtersApplied: ["concept", "taxonomy"],
|
|
2875
|
+
notes: [
|
|
2876
|
+
`The companyconcept document for CIK ${cik} / ${taxonomy} / ${concept} has no reported data points (units{} is empty). This is an honest empty — NOT a value of 0 and NOT an outage.`,
|
|
2877
|
+
],
|
|
2878
|
+
}),
|
|
2879
|
+
);
|
|
2880
|
+
}
|
|
2881
|
+
|
|
2882
|
+
// ── CLIENT-SIDE filters (unit/form/fy) — none reaches the URL. ──
|
|
2883
|
+
const unitFilter = args.unit?.trim() ? args.unit.trim() : null;
|
|
2884
|
+
const formFilter = args.form?.trim() ? args.form.trim().toLowerCase() : null;
|
|
2885
|
+
const fyFilter = typeof args.fy === "number" ? args.fy : null;
|
|
2886
|
+
const canonicalOnly = args.canonicalOnly ?? false;
|
|
2887
|
+
|
|
2888
|
+
let filtered = allRows;
|
|
2889
|
+
if (unitFilter !== null) filtered = filtered.filter((r) => r.unit === unitFilter);
|
|
2890
|
+
if (formFilter !== null)
|
|
2891
|
+
filtered = filtered.filter((r) => r.form != null && r.form.toLowerCase() === formFilter);
|
|
2892
|
+
if (fyFilter !== null) filtered = filtered.filter((r) => r.fy === fyFilter);
|
|
2893
|
+
|
|
2894
|
+
// Group the FILTERED rows by (unit,start,end) ONCE — used for BOTH revision detection
|
|
2895
|
+
// (over the pre-dedup set) and the canonicalOnly dedup (M1/M2).
|
|
2896
|
+
const groups = new Map<string, ConceptRow[]>();
|
|
2897
|
+
const groupOrder: string[] = [];
|
|
2898
|
+
for (const r of filtered) {
|
|
2899
|
+
const key = periodKey(r);
|
|
2900
|
+
let g = groups.get(key);
|
|
2901
|
+
if (!g) {
|
|
2902
|
+
g = [];
|
|
2903
|
+
groups.set(key, g);
|
|
2904
|
+
groupOrder.push(key);
|
|
2905
|
+
}
|
|
2906
|
+
g.push(r);
|
|
2907
|
+
}
|
|
2908
|
+
|
|
2909
|
+
// ★M1 revision detection: a GENUINE restatement is ≥2 rows sharing the SAME
|
|
2910
|
+
// (unit,start,end) with ≥2 DISTINCT non-null vals. A same-`end` different-`start`
|
|
2911
|
+
// pair lands in DIFFERENT groups (singletons) ⇒ NOT flagged as a revision.
|
|
2912
|
+
const revisedPeriods = groupOrder
|
|
2913
|
+
.map((k) => groups.get(k) as ConceptRow[])
|
|
2914
|
+
.filter((g) => {
|
|
2915
|
+
if (g.length < 2) return false;
|
|
2916
|
+
const vals = new Set(g.filter((r) => r.val !== null).map((r) => r.val));
|
|
2917
|
+
return vals.size >= 2;
|
|
2918
|
+
})
|
|
2919
|
+
.map((g) => {
|
|
2920
|
+
const first = g[0] as ConceptRow;
|
|
2921
|
+
return { unit: first.unit, start: first.start, end: first.end };
|
|
2922
|
+
});
|
|
2923
|
+
|
|
2924
|
+
// ★M2 canonicalOnly — partition by unit FIRST (already reflected in the (unit,start,end)
|
|
2925
|
+
// key), keep ONE row per distinct (start,end): frame-tagged, else the latest-filed
|
|
2926
|
+
// fallback (marked canonical:false). Deterministic tiebreak via preferCanonical.
|
|
2927
|
+
if (canonicalOnly) {
|
|
2928
|
+
filtered = groupOrder.map((k) => (groups.get(k) as ConceptRow[]).reduce(preferCanonical));
|
|
2929
|
+
}
|
|
2930
|
+
|
|
2931
|
+
// ── Pagination — client-side window over the (unit,start,end)-keyed filtered set. ──
|
|
2932
|
+
const totalAvailable = filtered.length;
|
|
2933
|
+
const limit = args.limit ?? 100;
|
|
2934
|
+
const offset = args.offset ?? 0;
|
|
2935
|
+
const page = filtered.slice(offset, offset + limit);
|
|
2936
|
+
const returned = page.length;
|
|
2937
|
+
const hasMore = offset + returned < totalAvailable;
|
|
2938
|
+
const nextOffset = hasMore ? offset + returned : null;
|
|
2939
|
+
|
|
2940
|
+
const filtersApplied: string[] = ["concept", "taxonomy"];
|
|
2941
|
+
if (unitFilter !== null) filtersApplied.push("unit");
|
|
2942
|
+
if (formFilter !== null) filtersApplied.push("form");
|
|
2943
|
+
if (fyFilter !== null) filtersApplied.push("fy");
|
|
2944
|
+
if (canonicalOnly) filtersApplied.push("canonicalOnly");
|
|
2945
|
+
|
|
2946
|
+
const notes: string[] = [AMENDMENT_DISCLOSURE_NOTE];
|
|
2947
|
+
|
|
2948
|
+
// Unit-filter disclosure (Q2-c) — NEVER hide the other units.
|
|
2949
|
+
if (unitFilter !== null) {
|
|
2950
|
+
if (unitKeys.includes(unitFilter)) {
|
|
2951
|
+
const others = unitKeys.filter((k) => k !== unitFilter);
|
|
2952
|
+
notes.push(
|
|
2953
|
+
others.length
|
|
2954
|
+
? `unit filter '${unitFilter}' applied; this concept is ALSO reported in unit(s) ${others.join(", ")} for this filer (not shown). Re-request without the unit filter, or with a different unit, to see them.`
|
|
2955
|
+
: `unit filter '${unitFilter}' applied; it is the only unit this filer reports for this concept.`,
|
|
2956
|
+
);
|
|
2957
|
+
} else {
|
|
2958
|
+
notes.push(
|
|
2959
|
+
`unit filter '${unitFilter}' matched NONE of this filer's reported unit(s) for this concept (available: ${unitKeys.join(", ") || "none"}). Returning 0 rows for that filter — this is NOT a value of 0; re-request with one of the available units, or omit unit.`,
|
|
2960
|
+
);
|
|
2961
|
+
}
|
|
2962
|
+
}
|
|
2963
|
+
|
|
2964
|
+
// ★M1 revision note — a distinctive phrase so a caller (and the fault fixtures) can
|
|
2965
|
+
// tell a GENUINE restatement from ordinary duration-multiplicity.
|
|
2966
|
+
if (revisedPeriods.length) {
|
|
2967
|
+
const ex = revisedPeriods
|
|
2968
|
+
.slice(0, 3)
|
|
2969
|
+
.map(
|
|
2970
|
+
(p) =>
|
|
2971
|
+
`(unit ${p.unit}, ${p.start != null ? `start ${p.start}, ` : ""}end ${p.end})`,
|
|
2972
|
+
)
|
|
2973
|
+
.join("; ");
|
|
2974
|
+
notes.push(
|
|
2975
|
+
`Restatement/revision detected: ${revisedPeriods.length} (unit,start,end) period(s) carry MULTIPLE rows with a DIFFERING val — the figure WAS revised for ${ex}${revisedPeriods.length > 3 ? " (and more)" : ""}. Read the filed dates; the frame-tagged (canonical) row carries SEC's consolidated value.`,
|
|
2976
|
+
);
|
|
2977
|
+
}
|
|
2978
|
+
|
|
2979
|
+
// canonicalOnly disclosure (Q3) — the dedup is FULLY disclosed, never silent.
|
|
2980
|
+
if (canonicalOnly) {
|
|
2981
|
+
notes.push(
|
|
2982
|
+
"canonicalOnly=true: reduced to ONE row per distinct (unit,start,end) period — the frame-tagged canonical value, or (for a period SEC has not yet consolidated) the latest-filed row, marked canonical:false. SUPERSEDED/amendment rows sharing the same (start,end) were REMOVED; a value may have been revised — re-request with canonicalOnly=false to see the full amendment history. NOTE: the same `end` with a different `start` is a DIFFERENT period (kept separately), not a duplicate.",
|
|
2983
|
+
);
|
|
2984
|
+
}
|
|
2985
|
+
|
|
2986
|
+
// Raw-vs-filtered total disclosure (Q4) — fetch-once, exact filtered total.
|
|
2987
|
+
notes.push(
|
|
2988
|
+
`served ${allRows.length} row(s) across unit(s) ${unitKeys.join(", ")}; after filters (${filtersApplied.join(", ")}) → ${totalAvailable} pageable; totalAvailable reflects the FILTERED (unit,start,end)-keyed set. The full per-unit series was fetched in ONE request (SEC does not paginate companyconcept); limit/offset is a client-side window.`,
|
|
2989
|
+
);
|
|
2990
|
+
|
|
2991
|
+
if (resolved.title)
|
|
2992
|
+
notes.push(`Resolved "${args.cikOrTicker}" → ${resolved.title} (CIK ${cik}).`);
|
|
2993
|
+
|
|
2994
|
+
return withMeta(
|
|
2995
|
+
{
|
|
2996
|
+
found: true,
|
|
2997
|
+
cik,
|
|
2998
|
+
entityName: str(body.entityName),
|
|
2999
|
+
taxonomy,
|
|
3000
|
+
concept,
|
|
3001
|
+
label: str(body.label),
|
|
3002
|
+
description: str(body.description),
|
|
3003
|
+
unitsAvailable,
|
|
3004
|
+
rows: page,
|
|
3005
|
+
},
|
|
3006
|
+
edgarMeta({
|
|
3007
|
+
returned,
|
|
3008
|
+
totalAvailable,
|
|
3009
|
+
filtersApplied,
|
|
3010
|
+
pagination: { offset, limit, hasMore, nextOffset },
|
|
3011
|
+
notes,
|
|
3012
|
+
}),
|
|
3013
|
+
);
|
|
3014
|
+
}
|