@cyanheads/secedgar-mcp-server 0.17.0 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +18 -15
- package/CLAUDE.md +18 -15
- package/README.md +12 -9
- package/changelog/0.17.x/0.17.1.md +36 -0
- package/dist/config/server-config.js +1 -1
- package/dist/config/server-config.js.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/mcp-server/tools/definitions/company-search.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/company-search.tool.js +6 -4
- package/dist/mcp-server/tools/definitions/company-search.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/compare-companies.tool.d.ts +8 -4
- package/dist/mcp-server/tools/definitions/compare-companies.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/compare-companies.tool.js +169 -39
- package/dist/mcp-server/tools/definitions/compare-companies.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.d.ts +4 -4
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.js +24 -8
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/find-holders.tool.d.ts +14 -0
- package/dist/mcp-server/tools/definitions/find-holders.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/find-holders.tool.js +34 -7
- package/dist/mcp-server/tools/definitions/find-holders.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.d.ts +2 -0
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.js +146 -77
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-filing.tool.d.ts +4 -4
- package/dist/mcp-server/tools/definitions/get-filing.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-filing.tool.js +305 -81
- package/dist/mcp-server/tools/definitions/get-filing.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.d.ts +14 -0
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.js +8 -0
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-material-events.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-material-events.tool.js +3 -1
- package/dist/mcp-server/tools/definitions/get-material-events.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/index.d.ts +453 -423
- package/dist/mcp-server/tools/definitions/index.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/search-filings.tool.d.ts +33 -0
- package/dist/mcp-server/tools/definitions/search-filings.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/search-filings.tool.js +31 -2
- package/dist/mcp-server/tools/definitions/search-filings.tool.js.map +1 -1
- package/dist/mcp-server/tools/limit-clamp.d.ts +50 -18
- package/dist/mcp-server/tools/limit-clamp.d.ts.map +1 -1
- package/dist/mcp-server/tools/limit-clamp.js +51 -15
- package/dist/mcp-server/tools/limit-clamp.js.map +1 -1
- package/dist/services/edgar/edgar-api-service.d.ts +75 -11
- package/dist/services/edgar/edgar-api-service.d.ts.map +1 -1
- package/dist/services/edgar/edgar-api-service.js +273 -50
- package/dist/services/edgar/edgar-api-service.js.map +1 -1
- package/dist/services/edgar/filing-to-text.d.ts +4 -3
- package/dist/services/edgar/filing-to-text.d.ts.map +1 -1
- package/dist/services/edgar/filing-to-text.js +12 -0
- package/dist/services/edgar/filing-to-text.js.map +1 -1
- package/package.json +4 -4
- package/server.json +3 -3
- package/skills/secedgar-guide/SKILL.md +1 -0
package/CLAUDE.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Agent Protocol
|
|
2
2
|
|
|
3
3
|
**Server:** secedgar-mcp-server
|
|
4
|
-
**Version:** 0.17.
|
|
4
|
+
**Version:** 0.17.1
|
|
5
5
|
**Framework:** [@cyanheads/mcp-ts-core](https://www.npmjs.com/package/@cyanheads/mcp-ts-core) `^0.13.18`
|
|
6
6
|
**Engines:** Bun ≥1.4.0, Node ≥24.0.0
|
|
7
7
|
**MCP SDK:** `@modelcontextprotocol/server` ^2.4.0
|
|
@@ -59,7 +59,7 @@ Tailor suggestions to what's actually missing or stale — don't recite the full
|
|
|
59
59
|
| `secedgar_get_beneficial_owners` | 5%+ blockholders of an issuer from structured SCHEDULE 13D/13G XML | `issuer`, `form_kind?`, `include_amendments?`, `limit?` |
|
|
60
60
|
| `secedgar_get_fund_holdings` | Fund portfolio holdings from the quarterly NPORT-P report | `fund`, `series_id?`, `report_date?`, `limit?`, `offset?` |
|
|
61
61
|
| `secedgar_fetch_frames` | Fetch SEC XBRL frames for one concept × one period across all reporting companies | `concept`, `period`, `taxonomy?`, `unit?`, `limit?`, `offset?`, `sort?` |
|
|
62
|
-
| `secedgar_compare_companies` | Period-aligned comparison of 2-10 named companies across 1-8 concepts | `companies`, `concepts`, `taxonomy?`, `period_type?`, `periods?` |
|
|
62
|
+
| `secedgar_compare_companies` | Period-aligned comparison of 2-10 named companies across 1-8 concepts; a longer list (up to 100) is answered to the cap, the rest returned unread in `deferred_companies` / `deferred_concepts` | `companies`, `concepts`, `taxonomy?`, `period_type?`, `periods?` |
|
|
63
63
|
| `secedgar_search_concepts` | Discover supported XBRL concept names or reverse-lookup a raw tag | `search?`, `group?`, `taxonomy?` |
|
|
64
64
|
| `secedgar_dataframe_describe` | List canvas dataframes with provenance, TTL, and schema | `name?` |
|
|
65
65
|
| `secedgar_dataframe_query` | Run a single-statement SELECT across dataframes (DuckDB SQL) | `sql`, `preview?`, `register_as?` |
|
|
@@ -82,15 +82,17 @@ Tailor suggestions to what's actually missing or stale — don't recite the full
|
|
|
82
82
|
|
|
83
83
|
## Domain Notes
|
|
84
84
|
|
|
85
|
-
- **Rate limit:** 10 req/s per IP. Every SEC request goes through one process-wide `createPacer` queue in `EdgarApiService` that spaces request *starts* `1000 / EDGAR_RATE_LIMIT_RPS` ms apart, with no concurrency cap and no wait budget — an ordinary burst drains at the configured rate and is never shed, and this server's own traffic can never exceed it, so `rateLimitRps` stays at 10 and headroom for a co-tenant on the outbound IP is the operator's call via `EDGAR_RATE_LIMIT_RPS`. A 429 is **not** retried: SEC holds the block until the request rate has stayed under the threshold for ten minutes and every request sent meanwhile restarts that clock, so the ~3s retry budget (`MAX_RETRIES` 3 × `BASE_BACKOFF_MS` 1000, exponential) cannot outlast it (#112). Instead a 429 closes the service's **block gate** for `EDGAR_RATE_LIMIT_COOLDOWN_SECONDS` (default 600). While it is closed nothing is sent: a call fails locally as `RateLimited` with `retryAfter` set to the whole seconds remaining (never below 1), and the gate is re-read at dispatch, so a call already queued in the pacer when the 429 landed — or a 5xx, no-response, or body-failure retry sleeping through its backoff — is refused too. The first call after the cool-down is sent alone as a probe while every other call waits on it: any HTTP answer but a 429 reopens normal pacing, a probe 429 closes the gate for the full cool-down again, and a probe that gets no HTTP answer (a network error, or no headers within the per-attempt cap) proves nothing, so the next call probes in its place — and every call that waited on it counts that probe as one of its own no-response attempts, so calls queued behind a silent SEC each settle within their own three-attempt ladder instead of probing in turn, 15 s apiece (#173). **The gate lives in the service, not in the pacer's `cooldown` option** — that gate holds queued calls until it reopens and then releases them all at the pacing rate, so it can neither refuse locally nor keep the reopening to one request. The upstream 429 and the local refusal carry one `data.reason: 'rate_limited'`, declared (`thrownBy: 'service'`, `retryable: true`) on every tool that reaches SEC, plus `data.retryable: true` — `ctx.fail` copies a contract entry's `retryable` onto `data`, a service throw does not, so both throw sites set it themselves (#122) — and one recovery hint naming SEC's ten-minute window and the seconds to wait; SEC's HTML block page stays out of the payload: the 429's body is cancelled unread (#188). Mirror-served reads (the ticker base behind `resolveCik`, and `tryGetCompanyFacts` / `tryGetCompanyConcept` / `tryGetFrames` through `mirrorOrLive`) never reach `rawFetch`, so they keep answering during a block. 500/502/503/504 still retry with the backoff, and so does an attempt that gets no HTTP response at all — a `fetch` rejection (Bun sets the transport code on the error itself, Node's undici on its `cause`) or no response headers within `ATTEMPT_HEADERS_TIMEOUT_MS` (15 s from dispatch; pacer queue time and the body read are outside it) (#173). Each tool call's `CallScope` carries a failure budget, `CALL_FAILURE_BUDGET_MS` (40 s, `call-scope.ts`): the time SEC spends failing the call — unanswered-attempt and 5xx waits, backoff, body-idle time — is charged to it and healthy progress never is, attempt and body-idle waits are clamped to what is left, a backoff is slept only while the budget outlasts it, and once it is spent no further attempt or backoff starts and the request fails as `sec_unreachable` or its 5xx mapping, so a call against an unresponsive SEC settles inside the MCP client's 60 s default; the caller's `ctx.signal` reaches every pacer wait, fetch, body read, and backoff, and a cancel rethrows untouched, never retried or counted as an SEC failure (#179). Every body is read inside that ladder, in `rawFetch`'s one body reader (`readBody`), so a 2xx whose body fails after its headers — the connection breaks off, invalid chunk framing or compression, or no bytes for `BODY_IDLE_TIMEOUT_MS` (15 s between reads, never across the whole body, so a body still arriving is never cut) — retries the same way; a probe whose headers arrive has already reopened the gate, and only its own call retries the body. The error body is read there too: a non-retryable status's or a last-attempt 5xx's body is captured for `data.body` / `data.responseBody` through `readBody` (`errorBodyHead`: the first 500 bytes, `…` when cut short, never a partial code point), so one that stalls after its headers ends at the 15 s idle limit or the budget left — keeping what arrived, its wait charged to the budget — and the caller's cancel ends it at once as cancellation (#202). A body the ladder has no use for — a 404's, a retried 5xx's (released before its backoff), a 429's, an answer landing as the caller cancels — is cancelled unread (#188); an error a spent budget ends with from a retried 5xx therefore has no `body`. A 200 that reads whole but is not JSON where JSON is expected fails on its one request, unretried (#181). So does a submissions document or archive page whose JSON lacks what its readers index — `filings.recent`'s (or the page's) six columns `accessionNumber`, `filingDate`, `form`, `primaryDocument`, `primaryDocDescription`, `reportDate` as arrays, an array `filings.files`, array `tickers` and `exchanges` — checked in `getSubmissions` / `fetchArchivePage` before the cache write, so nothing is cached and the next call refetches (#186). Empty arrays pass; lengths and optional columns are not checked. When the last attempt gets no complete, readable answer — whatever the earlier ones got — the call fails as `ServiceUnavailable` with `data.reason: 'sec_unreachable'`, `retryable: true`, `url`, and a hint, declared beside `rate_limited` on the same tools; the message says which case (no response, a body that failed to read, a body that is not the JSON the request reads) and names host, attempts, and the transport code or the timeout, never the request URL, and the runtime's own text rides `cause` to the log only (#173, #181). `createApp({ teardown })` disposes the pacer through `disposeEdgarApiService()` (#116)
|
|
86
|
-
- **Parameter names:** one name per concept across tools — `company` (the entity a tool reads; on `get_institutional_holdings` that is the 13F filer), `filed_after` / `filed_before` (inclusive filing-date bounds), `forms`. Every tool carrying one of those keys declares `inputAliases` for the other spellings in use — `ticker`, `cik`, `ticker_or_cik` → `company`; `start_date`, `date_from` → `filed_after`; `end_date`, `date_to` → `filed_before`; `form_types` → `forms` — so a caller carrying a name from one tool to the next is not rejected (#115). An alias is declared only where it maps to exactly one key of that tool and its value shape fits: a singular `form_type` cannot fill the `forms` array, so it is left out rather than widening the schema. A tool whose only company input carries another name takes the same company-identifier spellings onto it — `company`, `ticker`, `cik`, `ticker_or_cik` → `issuer` on `find_holders` and `get_beneficial_owners`, and those plus `name` and `search` → `query` on `company_search` — since the key it moves into has no other reading there; the canonical names stay, because renaming them would break existing callers (#152). `fund` (a fund or series) and `companies` (an array) are not aliased. Every tool with the date pair accepts either bound alone. `search_filings` fills the missing one before routing, since EDGAR full-text search ignores a one-sided custom range: a lone `filed_after` runs through today (UTC), a lone `filed_before` starts at 2001-01-01, or at 1993-01-01 when it falls before 2001, and the full-text path discloses a filled window in `effectiveQuery` as `(filed <after> to <before>)` (#185)
|
|
85
|
+
- **Rate limit:** 10 req/s per IP. Every SEC request goes through one process-wide `createPacer` queue in `EdgarApiService` that spaces request *starts* `1000 / EDGAR_RATE_LIMIT_RPS` ms apart, with no concurrency cap and no wait budget — an ordinary burst drains at the configured rate and is never shed, and this server's own traffic can never exceed it, so `rateLimitRps` stays at 10 and headroom for a co-tenant on the outbound IP is the operator's call via `EDGAR_RATE_LIMIT_RPS`. A 429 is **not** retried: SEC holds the block until the request rate has stayed under the threshold for ten minutes and every request sent meanwhile restarts that clock, so the ~3s retry budget (`MAX_RETRIES` 3 × `BASE_BACKOFF_MS` 1000, exponential) cannot outlast it (#112). Instead a 429 closes the service's **block gate** for `EDGAR_RATE_LIMIT_COOLDOWN_SECONDS` (default 600). While it is closed nothing is sent: a call fails locally as `RateLimited` with `retryAfter` set to the whole seconds remaining (never below 1), and the gate is re-read at dispatch, so a call already queued in the pacer when the 429 landed — or a 5xx, no-response, or body-failure retry sleeping through its backoff — is refused too. The first call after the cool-down is sent alone as a probe while every other call waits on it: any HTTP answer but a 429 reopens normal pacing, a probe 429 closes the gate for the full cool-down again, and a probe that gets no HTTP answer (a network error, or no headers within the per-attempt cap) proves nothing, so the next call probes in its place — and every call that waited on it counts that probe as one of its own no-response attempts, so calls queued behind a silent SEC each settle within their own three-attempt ladder instead of probing in turn, 15 s apiece (#173). **The gate lives in the service, not in the pacer's `cooldown` option** — that gate holds queued calls until it reopens and then releases them all at the pacing rate, so it can neither refuse locally nor keep the reopening to one request. The upstream 429 and the local refusal carry one `data.reason: 'rate_limited'`, declared (`thrownBy: 'service'`, `retryable: true`) on every tool that reaches SEC, plus `data.retryable: true` — `ctx.fail` copies a contract entry's `retryable` onto `data`, a service throw does not, so both throw sites set it themselves (#122) — and one recovery hint naming SEC's ten-minute window and the seconds to wait; SEC's HTML block page stays out of the payload: the 429's body is cancelled unread (#188). Mirror-served reads (the ticker base behind `resolveCik`, and `tryGetCompanyFacts` / `tryGetCompanyConcept` / `tryGetFrames` through `mirrorOrLive`) never reach `rawFetch`, so they keep answering during a block. 500/502/503/504 still retry with the backoff, and so does an attempt that gets no HTTP response at all — a `fetch` rejection (Bun sets the transport code on the error itself, Node's undici on its `cause`) or no response headers within `ATTEMPT_HEADERS_TIMEOUT_MS` (15 s from dispatch; pacer queue time and the body read are outside it) (#173). Each tool call's `CallScope` carries a failure budget, `CALL_FAILURE_BUDGET_MS` (40 s, `call-scope.ts`): the time SEC spends failing the call — unanswered-attempt and 5xx waits, backoff, body-idle time — is charged to it and healthy progress never is, attempt and body-idle waits are clamped to what is left, a backoff is slept only while the budget outlasts it, and once it is spent no further attempt or backoff starts and the request fails as `sec_unreachable` or its 5xx mapping, so a call against an unresponsive SEC settles inside the MCP client's 60 s default; the caller's `ctx.signal` reaches every pacer wait, fetch, body read, and backoff, and a cancel rethrows untouched, never retried or counted as an SEC failure (#179). Every body is read inside that ladder, in `rawFetch`'s one body reader (`readBody`), so a 2xx whose body fails after its headers — the connection breaks off, invalid chunk framing or compression, or no bytes for `BODY_IDLE_TIMEOUT_MS` (15 s between reads, never across the whole body, so a body still arriving is never cut) — retries the same way; a probe whose headers arrive has already reopened the gate, and only its own call retries the body. The error body is read there too: a non-retryable status's or a last-attempt 5xx's body is captured for `data.body` / `data.responseBody` through `readBody` (`errorBodyHead`: the first 500 bytes, `…` when cut short, never a partial code point), so one that stalls after its headers ends at the 15 s idle limit or the budget left — keeping what arrived, its wait charged to the budget — and the caller's cancel ends it at once as cancellation (#202). A body the ladder has no use for — a 404's, a retried 5xx's (released before its backoff), a 429's, an answer landing as the caller cancels — is cancelled unread (#188); an error a spent budget ends with from a retried 5xx therefore has no `body`. A 200 that reads whole but is not JSON where JSON is expected fails on its one request, unretried (#181). EDGAR full-text search answers a failure of its search backend with a 200 whose JSON carries `errorType` (its OpenSearch client's error class) in place of hits; `searchFilings` and `resolveEntityByName` opt in to classifying that body (`ReadOptions.backendFailure`), so `ConnectionError`, `TimeoutError`, and `NoLivingConnectionsError` fail the attempt on this ladder like a 5xx — same attempts, backoff, block-gate and pacer re-admission, budget, and cancel — and end as `efts_degraded_response` with `retryable: true`, while any other `errorType` (a `ResponseError` is the cluster refusing the request) fails on its one request with `retryable: false`; both carry `upstreamErrorType` and `upstreamErrorMessage`, bounded like `data.body`, never the body's `trace` or `requestId`, and the entity lookup throws instead of reading the body as no match (#206). So does a submissions document or archive page whose JSON lacks what its readers index — `filings.recent`'s (or the page's) six columns `accessionNumber`, `filingDate`, `form`, `primaryDocument`, `primaryDocDescription`, `reportDate` as arrays, an array `filings.files`, array `tickers` and `exchanges` — checked in `getSubmissions` / `fetchArchivePage` before the cache write, so nothing is cached and the next call refetches (#186). Empty arrays pass; lengths and optional columns are not checked. When the last attempt gets no complete, readable answer — whatever the earlier ones got — the call fails as `ServiceUnavailable` with `data.reason: 'sec_unreachable'`, `retryable: true`, `url`, and a hint, declared beside `rate_limited` on the same tools; the message says which case (no response, a body that failed to read, a body that is not the JSON the request reads) and names host, attempts, and the transport code or the timeout, never the request URL, and the runtime's own text rides `cause` to the log only (#173, #181). `createApp({ teardown })` disposes the pacer through `disposeEdgarApiService()` (#116)
|
|
86
|
+
- **Parameter names:** one name per concept across tools — `company` (the entity a tool reads; on `get_institutional_holdings` that is the 13F filer), `filed_after` / `filed_before` (inclusive filing-date bounds), `forms`. Every tool carrying one of those keys declares `inputAliases` for the other spellings in use — `ticker`, `cik`, `ticker_or_cik` → `company`; `start_date`, `date_from` → `filed_after`; `end_date`, `date_to` → `filed_before`; `form_types` → `forms` — so a caller carrying a name from one tool to the next is not rejected (#115). An alias is declared only where it maps to exactly one key of that tool and its value shape fits: a singular `form_type` cannot fill the `forms` array, so it is left out rather than widening the schema. A tool whose only company input carries another name takes the same company-identifier spellings onto it — `company`, `ticker`, `cik`, `ticker_or_cik` → `issuer` on `find_holders` and `get_beneficial_owners`, and those plus `name` and `search` → `query` on `company_search` — since the key it moves into has no other reading there; the canonical names stay, because renaming them would break existing callers (#152). `fund` (a fund or series) and `companies` (an array) are not aliased. `company_search`'s `filing_limit` → `limit` on `search_filings`, `get_material_events`, and `get_beneficial_owners`, whose one count input counts filings; rewritten before validation, it meets `limit`'s own cap and bound in either `EDGAR_CLAMP_OVERSIZED_LIMITS` mode. A `limit` counting managers, transactions, positions, companies, or data points takes no such alias, and `company_search` takes no `limit` alias, since there it would read as a count of companies (#212). Every tool with the date pair accepts either bound alone. `search_filings` fills the missing one before routing, since EDGAR full-text search ignores a one-sided custom range: a lone `filed_after` runs through today (UTC), a lone `filed_before` starts at 2001-01-01, or at 1993-01-01 when it falls before 2001, and the full-text path discloses a filled window in `effectiveQuery` as `(filed <after> to <before>)` (#185)
|
|
87
87
|
- **User-Agent required:** `"AppName contact@email.com"` on every SEC request or IP gets blocked
|
|
88
|
-
- **CIK zero-padding:** the `data.sec.gov` APIs (submissions, companyfacts, companyconcept) require the 10-digit zero-padded CIK (`String(cik).padStart(10, '0')`). Archive paths under `www.sec.gov/Archives/edgar/data/` take it unpadded — SEC answers the padded path with a 301, and `fetch` follows it inside the same pacer slot, so every archive read would cost two requests. `filingArchiveUrl` (exported from `edgar-api-service.ts`) strips the zeros for all
|
|
88
|
+
- **CIK zero-padding:** the `data.sec.gov` APIs (submissions, companyfacts, companyconcept) require the 10-digit zero-padded CIK (`String(cik).padStart(10, '0')`). Archive paths under `www.sec.gov/Archives/edgar/data/` take it unpadded — SEC answers the padded path with a 301, and `fetch` follows it inside the same pacer slot, so every archive read would cost two requests. `filingArchiveUrl` (exported from `edgar-api-service.ts`) strips the zeros for all seven archive reads and for `get_filing`'s `filing_url` (#156)
|
|
89
89
|
- **ETF/mutual-fund tickers:** `loadTickerCache` fetches `company_tickers_mf.json` alongside `company_tickers.json` and merges fund symbols (VOO, SCHD, JEPI…) into `byTicker`. Fund CIKs are registrant trusts (1:many with series), so fund rows go into `byTicker` only — never `byCik`, whichever source they come from. Resolved fund matches carry `seriesId`/`classId` from the live MF file, the only source of either. On a mirror-served ticker base the live MF fetch is merged in when `EDGAR_MIRROR_FALLBACK_LIVE` is on, so fund tickers resolve even if the mirror predates MF ingestion (#43); strict mirror-only (`FALLBACK_LIVE=false`) uses the mirror's own MF rows (ticker→CIK, no series/class). The mirror stores a fund symbol with an empty name — its schema has no series column — and `mirrorIndexRow` reads that empty name as the fund marker; a symbol claims `byTicker` in the order equity → live fund → mirror fund, so a live entry supersedes the mirror's bare row and keeps its series (#135). **The live fund slice has its own failure window:** a failed MF load (any HTTP failure, a malformed body, the block gate's refusal) is not cached for the ticker-cache TTL — `loadMfTickers` returns no rows and a `retryAt` `FUND_SLICE_RETRY_MS` (60 s) out, or the error's `retryAfter` when longer, so nothing is retried while the gate is closed. The first resolution after it refetches the file once, single-flight, onto the cached base (`reloadFundSlice`), which keeps its own load time: the equity index is not refetched and its TTL does not move. A per-resolution retry was rejected — against a persistent 5xx it adds three requests and ~3 s of backoff to every lookup (#119). Each failed load logs one warning through the framework's global `logger` (no request context to attach to) carrying `endpoint: 'company_tickers_mf'` (never the URL — framework log guidance keeps URLs out of log fields), the HTTP status or `malformed_body`, and the retry delay (#43).
|
|
90
|
-
- **Name and ticker resolution:** `resolveCik` walks numeric CIK → the `/^[A-Z]{1,5}$/` ticker fast path → the name passes (exact → prefix → substring over `allEntries`) → a catch-all `byTicker` lookup for symbols whose shape skipped the fast path.
|
|
90
|
+
- **Name and ticker resolution:** `resolveCik` walks numeric CIK → the `/^[A-Z]{1,5}$/` ticker fast path → the name passes (exact → exact once folded → prefix once folded → substring over `allEntries`) → a catch-all `byTicker` lookup for symbols whose shape skipped the fast path. Three normalizations hang off that chain. The name passes' **exact** tier also compares suffix-normalized forms (`normalizeCompanySuffix`): the terminal whitespace-delimited token only, a trailing `.`/`,` stripped as part of recognizing it, expanded to one of four canonical long forms — `corp`→`corporation`, `inc`→`incorporated`, `co`→`company`, `ltd`→`limited`. **The four buckets are never merged.** The registry carries distinct registrants differing only in which suffix they use (`TORO CO` 0000737758 vs `TORO CORP.` 0001941131, and the same shape on Blue Owl and Fluent), so a single "has a corporate suffix" marker would compare two companies equal. Only the terminal token is rewritten — the same words sit mid-name in titles ending in a different suffix (`American Water Works Company, Inc.`) — and a terminal token that merely contains one (`Ltd./ADR`) is not a suffix. Punctuation elsewhere in a name is the fold's (below). `llc`/`plc` and the `lp`/limited-partnership family get no bucket and must never fold into the `ltd` bucket (#107); they drop for a bare query (below) but never normalize into one of the four. The name passes **resolve within the best tier a current name reaches**: several CIKs there are the ambiguity, listed without the tiers below, so `Itron` resolves to ITRON, INC. instead of listing SOLITRON and POSITRON beside it. The exact tier also matches a registry name minus its terminal suffix and a comma before it (`stripCompanySuffix` over `DROPPABLE_SUFFIXES`: the four buckets plus `plc`, `llc`, `lp`, `nv`, `sa`, `ag`, and `se`, with or without periods), so `Apple` names `Apple Inc.` exactly. The wider set is deliberate: with the four buckets alone, `Rio Tinto` silently picked RIO TINTO LTD over RIO TINTO PLC; now both drop to an exact hit and tie. Only the registry side is stripped, so `Toro Co` never meets `TORO CORP.` and `Rio Tinto plc` never meets RIO TINTO LTD, and two suffix variants of one base name (`Blue Owl Capital`) tie. Current names (registry rows, which carry a ticker) choose the tier. A former name joins at that tier or better but never outranks one — otherwise `Travelers` resolves to Citigroup through its former name `travelers inc` — and decides alone only when no current name matches (`Facebook Inc`). Against the 2026-10-02 registry plus the former-names asset, 930 of 34,865 name queries move from ambiguous to resolved and no resolved CIK changes (#155). The **punctuation fold** (`foldNamePunctuation`, #210) deletes `,` `.` `'` `’` from the query and from every current and former name, so `QXO Inc` meets `QXO, Inc.`, `Fluent Inc` meets `Fluent, Inc.`, and `Jeffs Brands` the former name `jeffs' brands ltd`; a period is deleted, never spaced, so `U.S. Bancorp` meets `US BANCORP \DE\`. It is comparison-only — returned names and `suggestCompanies` never see it — and a helper of its own: `normalizeCompanySuffix` and `stripCompanySuffix` read the as-written strings, and the folded tiers run them again on the folded forms. Tier order: exact as written, exact once folded, prefix once folded, substring as written. Exact as written ranks first so a title typed as listed never ties a former name equal to it only once folded (`McGraw Hill, Inc.` stays alone beside S&P Global's former `mcgraw hill inc`), and substring stays as written, since a folded `Embassy Bancorp, Inc.` contains `sy bancorp inc` and would answer `S.Y. Bancorp Inc` with another bank. Without the fold a weaker tier answered a punctuation-free spelling with another registrant (`BlackRock Inc` reached BLACKROCK INCOME TRUST, INC. by prefix, `Block Inc` H&R BLOCK INC by substring). Against the 2026-11-03 registry plus the former-names asset, no title queried as listed changes its answer and no title or former name changes CIK; six former names typed as listed move to `multiple_matches` with the current registrant first, since a current title now matches them too (`energizer holdings inc` lists ENERGIZER HOLDINGS, INC. ahead of the former holder). `\DE\`-style state tags, `&` `-` `/`, and conformed names that space their initials (`J M SMUCKER Co`) stay out of scope. The **catch-all ticker lookup** retries a dotted symbol with `.`→`-` on a miss — brokers and market data write `BRK.B` where `company_tickers.json` lists `BRK-B`, and that file carries exactly one dotted ticker (a placeholder), so the substitution cannot land on the wrong registrant. The early gate stays untouched: it is a fast path, not the only ticker lookup, and `search_filings`' `ticker:` token inherits the fix through `resolveCik` with no change of its own (#110)
|
|
91
91
|
- **Former-name resolution:** `src/services/edgar/data/former-names.json` is a committed asset (generated offline via `bun run gen:former-names`) containing `[lowercasedFormerName, zeroPaddedCIK]` tuples. `buildTickerCache` folds them into `allEntries` for name search and the trigram pass. Regenerate on demand after major M&A waves — former names are immutable, so the file rarely needs updating.
|
|
92
92
|
- **Near-match suggestions:** `suggestCompanies(query, allEntries)` runs a Dice-coefficient trigram pass on the zero-hit search path, returning up to 3 candidates above a threshold. Each entry is scored on its **name and its ticker**, keeping the higher of the two — one ranked, deduped, top-N list rather than two disjoint ones, so a strong ticker match can outrank a weak name match and vice versa, and a ticker-shaped miss gets help instead of a bare `no_match` (#111). An entry missing either field is simply not scored on it: former-name entries carry a name and no ticker, and MF rows carry neither a name nor a place in `allEntries`, so fund tickers stay structurally unreachable here regardless of what the scorer does. Suggestions appear in the `no_match` error `data.suggestions` and in the error message. Never auto-resolves. Only the throw that attaches suggestions names the field in its hint (`company_search` passes its own `recovery.hint`: that sentence, then the contract text); the contract's `no_match` recovery never mentions it, since a zero-hit miss with no near match and the bare-CIK 404 carry no `suggestions` (#153).
|
|
93
93
|
- **XBRL friendly names:** `concept-map.ts` maps `"revenue"` → real XBRL tags; handles historical tag changes (ASC 606), including both assessed-tax variants — a filer presenting revenue gross of excise/sales tax reports only `RevenueFromContractWithCustomerIncludingAssessedTax`, and without it the walk fell through to tags SEC retired in 2018 (#98). The Including variant sits BEHIND `Revenues`, not ahead of it — the two disagree for filers that tag gross sales and net-of-excise sales under separate elements, so promoting it over `Revenues` swaps the definition partway back through the history and prints a step change that is a tag switch, not a business fact; behind `Revenues` it only fills frames no current tag covers. The same precedent places successor tags: `capex` walks `PaymentsToAcquireProductiveAssets` behind the PP&E-only tag (it also counts software and intangibles) and `interest_expense` walks `InterestExpenseNonoperating` behind both older tags, so NVIDIA, Amazon, Visa, and Microsoft reach current values and no frame the older tags cover changes (#125); the banks' `InterestExpenseOperating` (JPMorgan, Bank of America, and Wells Fargo report interest expense only under it from 2024) goes behind that, since it matches `InterestExpense` for 33 of 37 CY2025Q2 filers reporting both but `InterestExpenseNonoperating` for 0 of 4 (#147). SEC serves both successors with no label, so the line falls back to the concept's label. A different-definition tag still goes in `relatedTags`, never `tags` — `ppe_net`'s finance-lease-inclusive total is the primary line for Alphabet, Meta, and Tesla (#36, #130). `ifrsTags` covers every concept with an IFRS element confirmed present in a live 20-F filer's companyfacts; `notes_payable` is deliberately unmapped (IFRS presents borrowings as one caption, with no notes/debt split) and `shares_outstanding` is `dei`. Never add an `ifrsTags` entry from the taxonomy alone — an element existing is not evidence filers tag it (#99). `ifrsTags` order is semantic the same way `tags` is: index 0 is the counterpart of the us-gaap index-0 tag, so the same concept answers the same question under either taxonomy (`accounts_receivable` leads with the trade-only caption, not the combined trade-and-other one; `share_repurchases` with the financing outflow, not the treasury-only line a share-cancelling filer tags zero). Verify a candidate order against filers that report BOTH tags — presence alone does not settle which one leads. One concept has no right order: IFRS filers split between the employee-scheme share-based-payment element and the IFRS 2.51(a) expense total, and the two invert between filers (SAP's total is its real line with the employee element a two-period fringe; Sanofi is the exact opposite), so `stock_based_compensation` carries `ifrsTagSelection: 'coverage'` and picks per filer by which tag covers more standard periods (#101). It is the only concept that does, and the flag is scoped to `ifrsTags` on purpose — its us-gaap pair IS a ladder, since `AllocatedShareBasedCompensationExpense` is the disaggregated line and out-covers the total for Alphabet, IBM, Tesla, and P&G.
|
|
94
|
+
- **`fetch_frames` reads one tag:** a friendly name sends one frames request, for its first tag; the rest come back in `unqueried_tags` on a success and on `no_data`, whose hint says to query each. Revenue's first tag has no annual frame before CY2014, so an older revenue screen names one of the listed tags directly. The hint names no start year: frame coverage is per tag (#207).
|
|
95
|
+
- **`compare_companies` lists:** `companies` takes the first 10 inputs as given and `concepts` the first 8 distinct known concepts (merged spellings count once, unknown names none); the rest come back in `deferred_companies` / `deferred_concepts` behind `EDGAR_CLAMP_OVERSIZED_LIMITS`, through `batchClamp` in `limit-clamp.ts`, and the 10 x 8 bound keeps the inline-matrix cell ceiling valid (#201). The notice names the re-call — three when both lists defer: deferred companies with the 8 compared concepts, the 10 companies read with deferred concepts, and the two deferred lists together — since a re-call pairing one deferred list with the other full input list passes the other cap and repeats cells another re-call reads; a lone deferred company is paired with one of the 10 in its two re-calls, and that company leaves the second. An input resolving to a CIK an earlier input already holds is read once, under the earlier input, with a caveat like the concept merge's; it still takes a slot (#217).
|
|
94
96
|
- **Unknown concept names:** `findUnknownConcept` in `concept-map.ts` runs in `get_financials`, `compare_companies`, and `fetch_frames` before company resolution or any SEC request. After trimming, a catalog name in any case or separator form passes, and so does anything matching `/^[A-Z][A-Za-z0-9]*$/` — every item element in us-gaap, ifrs-full, dei, and srt has that shape and SEC matches tags case-sensitively, so no input that could return data is rejected, and a well-formed tag that 404s stays `no_data` / `no_concept_data` (#45). Everything else fails as `unknown_concept` (`NotFound`) and never reaches a request URL, where the tag is interpolated as a path segment. The hint is a formula from the `DERIVATIONS` table (`free_cash_flow`, `ebitda`, `working_capital`; read before the tag-shape test, since no taxonomy element is named `EBITDA`) or up to three catalog names by Dice trigram score ≥ 0.3 against name and label, ties by name — the label is the only route from `capital_expenditures` to `capex`. `total_debt` deliberately has no formula: `debt` is long-term only. `compare_companies` fails only when every concept is unknown; otherwise it answers the rest and lists each unknown once in `unknown_concepts`, never as a per-company gap (#128). Known concepts are deduped the same way, by what they resolve to — the catalog entry, or the trimmed raw tag — so `revenue`, `Revenue`, and ` revenue` are compared once under the first spelling and a caveat names the merged inputs; a catalog name and a raw tag it walks (`revenue` and `Revenues`) are different concepts and stay separate (#145). `trigramSimilarity` lives in the `trigram-similarity.ts` leaf so the catalog scores names without importing the HTTP service.
|
|
95
97
|
- **Taxonomy retry on `no_concept_data`:** when no tag answers under the requested taxonomy, `get_financials` reads companyfacts once (the #13 probe) and leads the hint with the retry that payload supports — `Retry with taxonomy: "ifrs-full" — eps_diluted resolves there to DilutedEarningsLossPerShare.` for a friendly name (resolved under each other accepted taxonomy through `resolveConceptTarget`, first tag in ladder order carrying a framed value), `DilutedEarningsLossPerShare is reported under ifrs-full — retry with taxonomy: "ifrs-full".` for a raw tag. It names only `us-gaap`, `ifrs-full`, or `dei` (companyfacts also carries `srt`, `invest`, `ffd`), fires in every direction whether or not the requested namespace is present (Toyota reports both and `cogs` only under us-gaap), and the namespace summary follows, or stands alone when no retry applies. It never answers from the other taxonomy itself: `taxonomy` defaults to `us-gaap`, so an explicit choice cannot be told from an omitted one (#164).
|
|
96
98
|
- **Series staleness:** two causes, one symptom, one caveat — `seriesStalenessCaveats(tag, label, newest, reference)` in `concept-series.ts` owns both, so a series is never flagged twice in different words. A friendly name's tag array is walked in order, so a tag SEC retired can win when every current tag reports nothing for the filer; SEC ships that tell in the taxonomy `label` (`"… (Deprecated 2018-01-31)"`), already in hand — no extra upstream call. Generalized rather than pinned to one pair: `revenue` and `cogs` both carry retired fallbacks (#98). A *current* tag whose series simply ends carries no tell at all, so the fallback signal is the gap itself: **730 days**, two full fiscal years behind the reference. The floor has to clear one year plus a filing window — an annual-only concept sits a fiscal year back until the next report lands, and a 20-F is due four months after year end — which puts a healthy filer around 500 days. Each tool passes the freshest reference it has without an extra call: `get_snapshot` and `compare_companies` pass `newestReportedPeriod(facts, taxonomy)`, the newest standard period the filer reports across the whole concept catalog — it isolates a line lagging its siblings and collapses to silence for a filer that stopped filing, whose every line is equally old, and it deliberately reads the whole catalog rather than the requested concepts, or a single-concept comparison would measure a series against itself. The reference skips the `dei` namespace — cover-page facts track the filing, not the statements, so they stay current for a registrant that stopped reporting under this taxonomy; counting them put Toyota's us-gaap profile (last reported fiscal 2020) six years behind a cover-page date and flagged all 28 of its lines at once. `get_financials` passes today, because one companyconcept payload holds no filer-wide period. All three measure the *full deduped set*, not the period-filtered slice — an annual view trailing a live quarterly series is a property of the filter. The retirement stamp wins when both apply; it names the concrete cause. `get_financials` therefore reads the clock, so its tests pin one (`vi.useFakeTimers({ toFake: ['Date'] })`) (#102).
|
|
@@ -98,17 +100,18 @@ Tailor suggestions to what's actually missing or stale — don't recite the full
|
|
|
98
100
|
- **Fiscal-period caveats:** SEC reports a filer's fiscal Q4 as the 10-K residual, never as a discrete quarterly fact, so the calendar quarter that a filer's fiscal Q4 *spans* carries no frame-tagged quarterly value. That is not always the quarter holding the fiscal year-end date — a January year-end closes a fiscal Q4 running Nov–Jan, which SEC frames as calendar Q4 — and calendar-year filers are affected too (no discrete Q4). `fiscal-periods.ts` owns both directions: `fiscalQ4Caveats(period)` for the cross-company frame view (`fetch_frames`), `missingQuarterCaveats(frames)` for the per-filer series view (`get_financials`, `get_snapshot`, `compare_companies`) (#95). The per-filer detection is data-driven — no maintained filer list — and reads only the newest 4 years carrying quarterly frames, applying the per-year quorum inside that window rather than before it, because SEC frame-tagged all four calendar quarters before ~CY2021 and drops the fiscal-Q4 quarter after; a whole-history read would let the older years mask the live gap, and filtering first would let a filer whose recent years are too sparse silently describe the older regime. **Keep that slice-then-filter order.** The detector reports a *set* of absent quarters, not one: a filer whose remaining fiscal quarters also span non-calendar durations loses two (Costco tags only calendar Q1 and Q4), and the per-year quorum is two of four — low enough to admit that shape, high enough to bound the absent set at two and keep a one-quarter stub year from dragging a third in (#100). Message wording and the derivation advice differ between the one- and two-quarter cases; the three consuming tools all iterate `caveats`, so nothing downstream assumes at most one entry.
|
|
99
101
|
- **`get_financials` period default:** with `period_type` unset the series defaults to `annual` (clean FY series); if the annual filter empties a non-empty series whose frames are all instant (`CY####Q#I` — balance-sheet, shares-outstanding, raw instant tags), it falls back to the full series so the first call returns data (#48). An explicit `period_type` is honored. `annual` on an instant series returns the framed instants whose `end` is one of the filer's fiscal-year ends (`filerFiscalYearEndsWithComparatives` — each annual report's own year, the #148 rule, plus the comparative years it restates, the same set `compare_companies` aligns on, so MSFT reaches its first XBRL 10-K's `CY2009Q2I` and `CY2008Q2I`), read from companyfacts only on that path — which otherwise fails — and reusing a payload already read. Selection keys on `end`, never on the frame holder's form or quarter: BSX's 2025-12-31 instant, framed on its Q2-2026 10-Q while the 10-K's own fact carries no frame, is kept with the 10-Q's provenance, and a June year end lands on `Q2I`. A cover-date concept (`shares_outstanding` for a 10-K filer, dated weeks after the year end) has no year-end instant, and a failed or `null` read selects nothing; both still fail `no_period_data`, with a hint naming which (#167). Any other explicit `period_type` that excludes everything still errors.
|
|
100
102
|
- **8-K item codes:** SEC replaced the original single-integer item numbering with the current `x.xx` scheme in Release 33-8400, effective **2004-08-23**, so EDGAR's `items` field carries two disjoint vocabularies. `eight-k-items.ts` decodes by the code's **shape**, never the filing date — dotted is current, a bare integer is legacy — because filers reported pre-changeover events under the old numbers for months after the effective date, and a date-keyed table would mislabel exactly those. The two vocabularies reuse integers for unrelated events (legacy `5` is Other Events, current `5.02` is officer departures; legacy `12` is the ancestor of `2.02`), so a dotted `items` filter never matches a pre-2004 filing — `get_material_events`' zero-hit notice says so and echoes the codes actually present. The tables back three surfaces: the tool's `items` enum, the decoded per-filing rows, and the `secedgar://filing-types` resource. A code matching a shape but absent from that regime's table comes back with no label rather than a guess.
|
|
101
|
-
- **EFTS quirks:** `dateRange=custom` must be set when using date params; the singular `entity` param is ignored — `cik:`/`ticker:` targeting passes the resolved CIK via the plural `ciks` param (server-side scope, independent of the filing's name text, so former-name filings on the same CIK are matched); `size` is ignored entirely and every request answers with 100 hits, so `find_holders` pages with `from` alone and expresses its budget in pages (5 → 500 rows). A hit is one matching **document**, not a filing: `_id` is `<accession>:<filename>`, a filing whose body and exhibits both match arrives once per document, and `hits.total.value` and the `form_filter` aggregation (keyed on `root_forms`, amendments folded in) count documents. `search_filings` collapses hits to one row per accession (first-ranked hit's fields, plus `matched_documents` in rank order) and counts `total` / `form_distribution` from those filings; no EFTS parameter yields a search's filing count (`collapse`/`aggs` are ignored), so `total` is exact only when one window held every match, else a lower bound beside `total_documents`. **A browse is the exception:** a forms- or entity-only request (no `q`) matches each filing's primary document alone (S-1 January 2025: 200 hits over two pages, 200 distinct accessions, every `file_type` the form), so `hits.total.value` is the filing count and `total` is EDGAR's count — exact when the relation is `eq`, at any offset — trusted while the window shows one document per accession, else back to the window lower bound; `dataset.truncated` still flags a window short of it, and `form_distribution` covers the filings in hand. The straddling merge keeps the window count for its 2001-onward side, since its `total` counts assembled rows. Under `sort=relevance`, `from` counts documents and the 100-hit window runs past the page, so a page lists the filings among its own `limit` documents (the window's first rows, which can number fewer than `limit`) and only a filing straddling a page edge recurs on the next page (#124)
|
|
103
|
+
- **EFTS quirks:** `dateRange=custom` must be set when using date params; the singular `entity` param is ignored — `cik:`/`ticker:` targeting passes the resolved CIK via the plural `ciks` param (server-side scope, independent of the filing's name text, so former-name filings on the same CIK are matched); `size` is ignored entirely and every request answers with 100 hits, so `find_holders` pages with `from` alone and expresses its budget in pages (5 → 500 rows). The result window is 10,000 documents, so a `from` past 9,900 is refused as a `ResponseError` body, with or without `q` and whatever the match total; `search_filings` sends `offset` as `from` only under `sort=relevance` with no entity scope on a 2001-onward range, fails a deeper `offset` there as `offset_out_of_range` before any request (`MAX_RELEVANCE_OFFSET`, which `find_holders`' continuation recipe also names), and keeps the schema's 9,999 for its other paths, which page the rows they assembled (#214). A hit is one matching **document**, not a filing: `_id` is `<accession>:<filename>`, a filing whose body and exhibits both match arrives once per document, and `hits.total.value` and the `form_filter` aggregation (keyed on `root_forms`, amendments folded in) count documents. `search_filings` collapses hits to one row per accession (first-ranked hit's fields, plus `matched_documents` in rank order) and counts `total` / `form_distribution` from those filings; no EFTS parameter yields a search's filing count (`collapse`/`aggs` are ignored), so `total` is exact only when one window held every match, else a lower bound beside `total_documents`. **A browse is the exception:** a forms- or entity-only request (no `q`) matches each filing's primary document alone (S-1 January 2025: 200 hits over two pages, 200 distinct accessions, every `file_type` the form), so `hits.total.value` is the filing count and `total` is EDGAR's count — exact when the relation is `eq`, at any offset — trusted while the window shows one document per accession, else back to the window lower bound; `dataset.truncated` still flags a window short of it, and `form_distribution` covers the filings in hand. The straddling merge keeps the window count for its 2001-onward side, since its `total` counts assembled rows. Under `sort=relevance`, `from` counts documents and the 100-hit window runs past the page, so a page lists the filings among its own `limit` documents (the window's first rows, which can number fewer than `limit`) and only a filing straddling a page edge recurs on the next page (#124). A backend failure arrives as a 200 too, with `errorType`, `errorMessage`, `trace`, and `requestId` in place of `hits`; its handling is in the rate-limit note above (#206). Caller text is escaped before it reaches full-text search: `searchFilings`' `q` and `resolveEntityByName`'s `keysTyped` both go through `escapeEftsText`, which doubles each backslash and writes each control character (category Cc) as `\uXXXX`, so a term holding either is searched as typed. The escape lives in the service, not a tool, so `find_holders`' `search_key` re-sent through `search_filings` runs the same search. `find_holders` quotes the conformed name minus one trailing qualifier — a `\` or `/`, one to six letters, an optional closing `\` or `/` (`/[\\/][A-Za-z]{1,6}[\\/]?$/`, then a trim) — since 13F information tables print the issuer without it: `"US BANCORP"` matches 2,315 documents in the 2026-Q2 filing window, the phrase with its `\DE\` qualifier 2, and `PROGRESSIVE CORP/OH/` is searched as `"PROGRESSIVE CORP"`. A slash inside a name (`CULLEN/FROST BANKERS, INC.`) and an undelimited state token (`UNIVERSAL DISPLAY CORP PA`) stay, and `resolved_issuer_name` keeps the registry title (#205). A notice naming a `secedgar_search_filings` call as JSON — `find_holders`' first-page and continuation recipes, `company_search`'s withheld-filings recipe — puts it in a code span (`markdown().inlineCode`, fenced past any backtick run), since `content[]` renders as Markdown and outside code the quoted phrase's `\"` loses its backslash, as would a caller's form type typed into `company_search`'s `forms` (#220)
|
|
102
104
|
- **Submissions archive walk:** the submissions feed's `filings.recent` holds the last year or 1,000 filings, whichever is more — so 26,000+ rows for JPMorgan, and a filing day is never split (Microsoft's window runs to 1,002 rows to finish its oldest day); everything older lives on `filings.files[]` archive pages of 200–360 KB each. Every tool that reads past `recent` goes through `SubmissionsArchiveWalk` (`submissions-archive.ts`): one `ARCHIVE_PAGE_SCAN_CAP` (10) for all, pages read one at a time through the service's pacer and page cache, and no page selected when the lower bound falls on or after `recent`'s oldest date — so a call answered by `recent` sends no archive request. The caller picks the order — `newest-first` (`company_search`, `search_filings`' pre-2001 arm, `get_material_events`, `get_insider_transactions`' window), or `oldest-first` forward from a date (`get_institutional_holdings`' quarter lookup, since newest-first cannot reach JPMorgan's 2020-Q4 13F-HR on page 41 of 70 under the cap) — and can stop after any page; `scannedThrough` (the oldest page start read) and `truncated` (selected pages left unread, whether the cap, an early stop, or a caller that never started the walk left them) are reported the same way everywhere. Worst case per call is the submissions document plus 10 pages plus the tool's own document reads (`get_insider_transactions`: 111). `get_material_events` without a date window walks only to fill `limit`: it enters the walk when fewer than `limit` `recent` rows pass the `items` filter — the same count as the stop test, so unrelated recent 8-Ks never stand in for matches (#139) — and stops on the page that fills it with rows passing that filter, and its `dataset.truncated` is true whenever selected pages went unread — including when `recent` alone fills `limit` and no page is read, since the dataframe then holds only the recent window (#134); a result short of `limit` with selected pages unread (only the cap ends a walk that short) gets a notice naming the pages read, the oldest date reached, and the lever — `filed_before=<that date>` alone or in a window when undated, narrowing the window with `filed_before=<that date>` when dated, since a newest-first walk reaches older filings only through an earlier upper bound — appended to the zero-match or empty-window notice when one fires, since `ctx.enrich.notice` is last-wins; an empty-window notice the cap cut short claims no 8-K only "as far back as the scan reached" and drops its widen-the-range advice (#139); `get_institutional_holdings` stops on the first page holding a 13F-HR or 13F-NT for the quarter, and runs the #86 operating-company routing only when neither `recent` nor the pages read hold any 13F-HR or 13F-NT — a notice-only manager asked for a quarter it filed nothing for gets the plain quarter miss (#117, #133); without a `quarter`, a manager whose `recent` holds 13F-NT notices and no 13F-HR gets the notice message for its newest notice rather than the #86 classification (#133). **Page selection widens both date bounds by a day:** a manifest page's `filingFrom` / `filingTo` can sit a day off its own rows in either direction (JPMorgan's page 041 is listed to 2021-02-17 but holds 42 filings dated 2021-02-18; Apple's page 001 is listed a day past its newest row), so an exact overlap test skipped the page holding a window's edge day. The slack costs at most one extra page per bound, and every caller filters rows by their own filing dates (#137)
|
|
103
105
|
- **13F amendment composition:** `get_institutional_holdings` composes a quarter from the original 13F-HR and its same-period 13F-HR/A amendments (`composeHoldings`), ordered by `acceptanceDateTime`, then filing date, then accession — never `amendmentNo`, since filers skip numbers and two amendments can land on one day. A `RESTATEMENT` replaces everything composed so far, earlier `NEW HOLDINGS` included, so the base is the last restatement whose table reads (the original when none does) and nothing before it is read (`superseded`). A `NEW HOLDINGS` after the base appends its rows, unless its table repeats every composed row (CUSIP, class, put/call, discretion, shares, share type; value is left out because its unit changed on 2023-01-03) — a mislabeled resubmission that would double the portfolio — in which case it applies as a restatement. Each accession's values normalize by its own filing date. An amendment whose type, index, or table cannot be read, or whose table has no rows, is `not_applied` instead of blanking the quarter. The original's table is always read (it backs `total_holdings_in_filing`); each amendment table is two more requests on the call's `CallScope`, and a 429 or a spent budget stops further optional reads (#198).
|
|
104
|
-
- **Filing metadata outside `recent`:** `get_filing` takes `form`, `filing_date`, and `period_ending` from `recent` when the accession is there, else from the filing's own SEC header — the raw `<TYPE>` / `<FILING-DATE>` / `<PERIOD>` in `index-headers.html`, which the tool already fetches to type the documents, and only when that page 404s (common before 2014) the sub-1 KB `<accession>.hdr.sgml`. `FILING-DATE` is the filing date, not the acceptance timestamp, and matches the submissions feed; an S-8 or Form 4 header has no `PERIOD`. `company_name` stays the current registrant name (#126)
|
|
105
|
-
- **
|
|
106
|
-
- **
|
|
106
|
+
- **Filing metadata outside `recent`:** `get_filing` takes `form`, `filing_date`, and `period_ending` from `recent` when the accession is there, else from the filing's own SEC header — the raw `<TYPE>` / `<FILING-DATE>` / `<PERIOD>` in `index-headers.html`, which the tool already fetches to type the documents, and only when that page 404s (common before 2014) the sub-1 KB `<accession>.hdr.sgml`. `FILING-DATE` is the filing date, not the acceptance timestamp, and matches the submissions feed; an S-8 or Form 4 header has no `PERIOD`. `company_name` stays the current registrant name (#126), read from the feed of the CIK whose directory served the filing
|
|
107
|
+
- **Filer lookup (#203, #215):** without `cik`, `get_filing`'s candidate CIKs are the filers EDGAR full-text search records for the accession, then the accession number's own prefix (the filer's CIK, or a filing agent's). A search SEC cannot answer (`ServiceUnavailable` — `sec_unreachable`, a 5xx, `efts_degraded_response` — or a 504's `Timeout`) degrades to the prefix alone, so `get_filing` never surfaces `efts_degraded_response` and does not declare it; the caller's cancellation and `rate_limited` still end the call. When no candidate serves the filing — a directory listed the target and the archive answered 404, or no directory answered at all — `resolveFilingArchive` reads `<accession>-index.html` under the prefix CIK (`tryGetFilingPartyCiks`): SEC serves it there for any existing accession, an agent's prefix included even when that directory has no `index.json` (a 1999 SC 13D's agent path, a 2025 Donnelley 10-K's), and answers 404 for an accession that does not exist. Its party links (`browse-edgar?CIK=<10 digits>`: Filer, Subject, Filed by) are scanned in one forward pass (`filingPartyCiks`), and up to three CIKs not yet tried are searched the same way, #158 fallback included. A document no listing names skips the page, since every party's directory lists the same files. The page costs one request on a path that failed before and none when a candidate serves the filing; an unavailable page read degrades like the search. Misses after that name `cik` in their hints, never the document that just failed, and say when a filer lookup was unavailable. `tryGetFilingIndex` drops the entries SEC lists with no filename (a pre-2001 submission's documents, which exist only inside `<accession>.txt`), and a blank `document` reads as omitted
|
|
108
|
+
- **Primary document (#161):** the primary `get_filing` reads when `document` is omitted, lists under `documents.primary`, and names in its error catalogs is the name SEC records (`recordedPrimaryNames`): the submissions feed's `primaryDocument` when the accession sits in `recent` (last path segment — an XML form is recorded through its stylesheet path, `xslF345X06/form4.xml`), else the document the header page types with the submission's form. Only with neither does the largest readable document stand in, and the viewer-page guard there matches `R\d+.html?`, not every name starting with `R` (`Report10K.htm`). Size alone picked an 8-K's press-release exhibit over the form and a 13F-HR's information table over its cover page. The header page and the feed are read together right after an index is found and before the body, since every outcome — the error catalogs included — names the primary from them. **An XML form reads as SEC renders it (#216):** the stylesheet path the feed records serves SEC's HTML rendering of the form under the filing's archive directory, while the raw XML's tags strip to one unlabeled line — Maplebear's SCHEDULE 13D/A (0001193125-26-442148) reads as 13,443 characters on one line raw and as 857 labeled lines rendered, Items 1 and 4–7 detected as headings. The rendering prints filer-written text (a 13D's Items, a reporting person's cover-page note) in `div.largetext`, styled `white-space: pre-wrap`, so that div takes html-to-text's `pre` formatter in `CONVERT_OPTIONS`, with a plain div's one line break around it: collapsed, Item 4 ran `as follows: On October 28` together. The 13D and 13G stylesheets carry that one pre-wrap class; the Form 4, Form 144, and Form D ones none. A selector, not a regex pre-pass: a pass matching `<div class="largetext">` missed the cover-page notes, whose div carries a `style` attribute. With `document` omitted, `searchArchive` reads the rendering first (`renderingToRead`) and the raw file when the rendering answers 404 or SEC is unavailable for it, degrading like the #203 index-page read (`unlessSecUnavailable`), so a rate-limit refusal or the caller's cancel still ends the call. `primary_document` and `filing_url` name the path read; `documents` keeps the raw file as the primary; a caller-named raw file reads as it always has, and a caller-named stylesheet path the feed records for the filing reads the rendering. The rendering is read only while the index lists the raw file at `MAX_RENDERED_RAW_BYTES` (600,000) or less — a raw file listed with no size reads as filed: a holdings report renders at about 20 times its raw size (Vanguard 500 Index's NPORT-P 0000036405-26-000325: 500,525 bytes raw, 9,777,862 rendered, 4.2 million characters of text), and `html-to-text` silently cuts input past 16,777,216 characters, 28 times the bound, while a 13D/A renders at under 3 times (25,851 → 69,523 bytes) and a Form 4 rendering runs about 16 KB. The success path still sends one document request. Only the feed records the path, so a filing outside `recent` reads its raw file. `get_insider_transactions`, `get_beneficial_owners`, and `get_fund_holdings` parse the raw XML through `rawDocumentName` and are unaffected
|
|
109
|
+
- **Filing content:** HTML → text via `html-to-text` library; pre-2005 filings produce noisier output. The library walks the DOM recursively, so `CONVERT_OPTIONS` caps it at `limits.maxDepth: 512` with a `[…]` ellipsis where the cut falls — without it deep markup overflowed the stack and one such candidate failed `search_filings`' whole pre-2001 text scan (#118). The bound must hold on both runtimes the server ships under: a fresh Node process overflows near 1,360 levels of nested lists (its most stack-hungry shape), Bun near 13,000; real HTML filings measure under 25 levels. Legacy SGML `<PAGE>` markers are replaced with a line break before parsing (`neutralizePageMarkers`) — the parser never closes them, so a text document otherwise nests one level per page and a long one would be cut at the cap; with them gone the deepest measured `.txt` sits under 100 (an EX-27 schedule's unclosed tags). Both changes leave every sampled filing's text byte-identical. `limits` deep-merges with the library defaults, so its 16,777,216-character `maxInputLength` (silent truncation, no ellipsis) still applies. **Plain-text SGML bodies keep their lines (#136):** the parser collapses every newline run to a space, so a plain `<TEXT>` body — a pre-2001 full submission, a 2000s ASCII primary document — lost the line structure `detectHeadings` anchors on (Apple's 1996 10-K came back as 104 lines with an empty outline). `preservePlainTextBodies` runs inside the one `convert()` call: a body is HTML when `<html`, `<body`, `<p`, `<div`, `<br`, `<font`, `<tr`, `<td`, or `<xml` opens anywhere in it (`\b`-bounded, so `<PAGE>`, `<PDF>`, and EX-27's `<PP&E>` don't trip `<p`; no `<table`, since EDGAR ASCII tables are `<TABLE>`); any other body has every tag dropped, `&`/`<`/`>` escaped, and goes in a `<pre>`. The list is broad because misreading HTML as plain wrecks it, while the reverse is only the old output. Tags are dropped rather than parsed: the parser formats `<TABLE>`/`<CAPTION>` as blocks inside the `<pre>`, and a stray `<word` in prose would swallow text up to the next `>`. The `<TEXT>` wrapper matches case-sensitively — EDGAR writes it uppercase, and an inline SVG `<text>` is not a body — and the scan is one forward `indexOf` pass, since a lazy `<TEXT>([\s\S]*?)</TEXT>` regex goes quadratic on openers with no closer. Input with no plain body reaches the parser unchanged, so current HTML and 2000–2004 SGML-wrapped HTML convert byte-identically; their outlines change only where an unpunctuated Item cross-reference opens a line and drops out as wrapped prose (see Heading detection). Kept indentation grows plain extractions about 19%. **Every document of a submission holding an HTML document (#159):** the parser converts only `<body>` elements when the input has one, so an HTML-era submission (HTML primary, plain EX-10/EX-27 exhibits) lost every plain exhibit, and the pre-2001 scan could not match exhibit text. When the input holds `<body`, each plain body's `<pre>` goes inside a `<body>` of its own, so every document converts in document order, separated by the block break the parser already puts between two HTML documents — no per-document label, as multi-HTML submissions never had one. Uuencoded payloads (GRAPHIC, a PDF behind `<PDF>`, whose encoded bytes can pass the HTML test) and machine-readable files by `<FILENAME>` extension (`.xml`/`.xsd`/`.css`/`.js`/`.json` — a modern full submission's XBRL and renderer files read as plain) stay outside, dropped as before, and so does the SEC header, as it always was for these submissions. Input with no `<body>` — a plain-only submission — converts whole, header included. Over 93 corpus files only the 10 submissions mixing HTML and plain documents change: each old text is a prefix of the new one and all 186 old outline entries keep their offsets. The primary is document 1, so its headings come first and exhibit headings only fill what the 50-entry cap leaves; cross-document keep-later dedup moved none. **The XBRL renderer's files are cut (#160):** in the same `<body>` case, a document whose `<TYPE>` is `XML` is removed from the input. That is the type EDGAR gives every file its renderer generates (`R1.htm`…, `Show.js`, `report.css`, `FilingSummary.xml`, described `IDEA: XBRL DOCUMENT`). The viewer pages carry `<body>` elements, and so does 2011's `Show.js`, in a string literal, so the R-file tables followed the filed documents. The rule keys on the type, not the `R\d+.htm` name: across 79 sampled 2014–2026 indexes every one of 1,081 R files is typed `XML`, no filed document is, and inline-XBRL primaries carry their form type. Only the 4 corpus submissions holding renderer files change, each new text a prefix of the old. Indented headings stay undetected (centered S-4 headings; Items indented in 2 of 29 sampled 10-Ks): admitting indentation fills the outline cap with centered cover legends and promotes indented TOC rows. The one exception is a same-line titled Item opening on a single NBSP (#176, see Heading detection). **A primary the archive does not serve (#158, #213):** many 2000–2001 indexes list sequence-numbered documents (`0001.htm`, `0001.txt`, `0002.txt`) that answer 404 under those names. SEC serves them under the accession-prefixed name (`<accession>-0001.htm`, each a `<DOCUMENT>`-wrapped copy: TOSCO's 10-Q 0000899681-00-000406 answers 404 as `0001.htm` and 200 with 74,863 bytes prefixed), so a sequence-numbered name that 404s is read again under that spelling in the same directory before the next candidate CIK is tried (`archiveSpellings`). `primary_document` and `filing_url` then name the spelling read, `documents` keeps the index's names, and a caller-named `document` resolves in either spelling, the given one read first (`requestedReadNames`). With `document` omitted, the prefixed read is optional like the rendering read (#216, `unlessSecUnavailable`): one SEC cannot serve (no response, a 5xx, a 504) counts as not served and leads on to the full submission below, while a rate-limit refusal, a 403, or the caller's cancel still ends the call; a caller-named spelling's read fails as any read does. Only for a filing serving neither spelling, when no candidate CIK serves the index's primary, does `resolveFilingArchive` read the full submission `<accession>.txt` instead: `primary_document` and `filing_url` name the `.txt`, and the notice (composed into the truncation guidance, since `notice` is last-wins) names the unserved file. The fallback runs after the CIK loop, so a CIK that serves the primary still wins, and a served primary costs no extra request. A caller-named `document` is never swapped — a listed one served under neither spelling fails `document_not_found` with a hint naming `<accession>.txt`. The extract cache stores the document and CIK each text came from, so a cache hit reports the same primary, notice, `cik`, and `filing_url` as the miss that filled it, with no candidate-CIK search
|
|
107
110
|
- **Heading detection:** `detectHeadings` runs three line-anchored patterns — the all-caps forms, the mixed-case same-line `Item N. Title` / `Part II` forms (#71), and a bare `Item N` marker whose title lands on a later line. **One NBSP may open a same-line Item line (#176):** both Item arms admit a single U+00A0 ahead of the marker, since an ` ` before a styled span left Horizon Space's 8-K line `<NBSP>Item 1.01. Entry into a Material Definitive Agreement.` (0001929980-26-000542) out of an outline that listed Items 7.01 and 9.01. A lookahead holds the NBSP to an Item marker; the entry's offset is the marker's (1960, not the NBSP's 1959), `isWrappedProse` and `followsMidSentence` still read the line from its real start, and a line that is a bare marker once its page cell is stripped (`<NBSP>ITEM 2.02`, `<NBSP>Item 1. 3`) yields nothing (`itemEntryOffset`). Nothing wider: ASCII-space or tab indentation, two NBSPs, and NBSP-led Part lines stay out — Microsoft's 2004 10-K (0001193125-04-150689) stamps an NBSP-led `Part I` on every page, and admitting those turns its body `PART I` (3832), `PART II` (64076), and `PART IV` (308565) into running-header furniture. Across the 67-filing corpus the change adds that one entry and removes none. Every whitespace match is horizontal-only: `\s` matches `\n`, which fused runs of consecutive all-caps lines into composite entries (`"TABLE OF CONTENTS\n\nPART I"`) (#105). Horizontal-only is not space-or-tab — filings put NBSP inside all-caps headings too (`MICROSOFT<NBSP>CORPORATION`), so the all-caps arm nests the class-set difference `[\s--\n]` in its class (the `v` flag). It is one class, not the class alternated with `[^\S\n]`: the quantified alternation slows sharply on very long lines, while a single class scans in linear time. The all-caps line admits `-`, `–`, `—`, digits, and apostrophes, and `isAllCapsHeadingText` bounds it and narrows the last two. Its heading text runs at most `MAX_TITLE_CHARS` (140): past it the line is a paragraph set in capitals, which the widened class otherwise admitted (Alternative Ballistics' 308-character risk-factor coda took `section: "business"` by substring from `OUR BUSINESS`; EuroSPAC's 860-character tax legend; GT Biopharma's 288- and 326-character voting notices). A digit is admitted only inside a whitespace-delimited `\d{1,2}[A-Z]?\.?` token judged on the whole line, a TOC page cell included (`NOTE 10`, `SCHEDULE 14A`; `USD 2,929`, `FMR LLC(4)`, `WASHINGTON, D.C. 20549`, and `…ACT OF 1934` stay out), and an apostrophe only inside a word or after a word-final `S` (`MANAGEMENT’S`, `STOCKHOLDERS’ EQUITY`), never opening or closing the heading. The all-caps `ITEM …` arm takes the rest of its line whatever it holds, and neither Item arm admits a line that runs a table of contents together: a second Item marker — after whitespace, or abutting the period, digit, or parenthesis that ends the cell before it (`Item 2.Item 3.`, `Item 9.01Item 9.02`, #180), while an opening quote or parenthesis before it is a title citing another Item (`(Item 408 of Regulation S-K)`) — or a page cell mid-line followed by two or more spaces and the next cell (`isRunTogetherToc`). Without it a 238-character 8-K table of contents (`ITEM 8.01. OTHER EVENTS. 3 ITEM 9.01. …`, Amazon's 0001104659-26-107526) was one heading and took `item 8`, `item 9`, and `exhibits`. The test is on shape, not length: a 140-character bound on that arm dropped a proxy's one proposal heading, Modine's 185-character `ITEM 1 – TO APPROVE THE PROPOSAL TO AMEND THE Amended and Restated Articles…` (DEF 14A 0001104659-26-115309), and turned `item 1` and `proposal` into misses. Across 104 cached filings (244 all-caps and 568 mixed-case Item lines) the shape test flags only the Amazon line, and no all-caps Item line runs past 240 characters, so the arm carries no outer length bound. The mixed-case same-line arm keeps its title bound as its prose guard. A colon stays out: every colon line it admitted across 26 sampled documents was a statement caption (`CURRENT ASSETS:`). A Part line may carry a title — whitespace, an optional `-`/`–`/`—`/`:`/`.`, then an uppercase letter, at most 140 characters — and is a cross-reference rather than a heading (`isPartCrossReference`), on the mixed-case and the all-caps arm alike, when it sits under a line ending on a comma or semicolon like an unpunctuated Item, or when its title opens on another marker (`Part II — Item 5. Market for Registrant’s Common Equity`, a bank 10-K's cross-reference index row), which would otherwise also displace the bare body Part it names. After dedup, a bare Part entry gives way to a later titled heading for the same numeral (`dropBarePartsTitledLater`, which applies the same marker-title test): a 10-Q's TOC row `Part II` and its body `PART II — OTHER INFORMATION` have different keys, so without the merge `section: "part ii"` landed on the TOC row 50,045 characters early (Apple Q3 FY2026). Over the 26-document corpus these rules add 72 headings and remove 4 TOC Part rows: 66 section headings (all 18 of Microsoft FY2026's `NOTE N — TITLE` lines, statement titles, titled Parts) and 6 furniture (statement captions, `SCHEDULE 14A`, a `PAGE` stamp, a wrapped legend line), no numeric table row (#170). The Item code spans `16A`–`16K` and the current 8-K's dotted `N.NN` (`ITEM_CODE`, shared by all four Item patterns), and the marker's period is optional — 20-F filers write the lettered markers unpunctuated. A dotted code may also abut its title or carry a paragraph suffix (`Item 9.01Financial Statements…`, `Item 9.01(d). Financial…`, the shapes html-to-text leaves when the marker and title are separate spans), and then the title must start with a letter, so a bare `Item 5.02.` line never matches on its own. **Any other code may abut its title too (#180):** past its period (`Item 2.Management’s Discussion…`, `Item 1A.Risk Factors`, `Item 16A.Audit…`), the title again opening on a letter, while a digit after the period stays part of the code (`Item 2.02` and the dotted forms above read as before; `Item 2.0 something` is no heading). With no period, a code or Part numeral runs into a title only where a capital followed by a lowercase letter opens it (`TITLE_RUN_IN`, a case-sensitive `(?-i:)` lookahead inside the case-insensitive patterns): `Item 1Financial Statements` is Item `1` and `Part IFinancial Information` Part `I`, while a lettered code or a longer numeral reads whole (`Item 16KCybersecurity`, `Part IVExhibits`) and an all-caps run-in (`Item 1ARISK FACTORS`) matches nothing. New Peoples Bankshares' 10-Q (0001903596-26-000307) prints every body marker this way and had no Item in its outline; it now lists all eleven Items and Part I (28 → 40 entries; the period form alone 28 → 38), `item 1` lands on Part I's `Item 1Financial Statements` (4167) instead of Part II's `Item 1.Legal Proceedings` (113660), and `risk factors` and `management's discussion` land at 114156 and 73945 instead of missing. Across the other 25 corpus filings no outline entry or landing moved. Every new alternative is a fixed-width lookahead, so the scans stay linear. A code in `CURRENT_EIGHT_K_ITEMS` takes a 160-character title bound (`EIGHT_K_MAX_TITLE_CHARS`) on the same-line arm and in `usableTitle` — Item 5.02's 146-character title is the longest across the 53 sampled 8-Ks, and the 14-character margin admits a short trailing note while a paragraph opening on the marker line (`Item 5.02(b) On June 2, 2026, the Board…`) runs to hundreds of characters and stays out — and every other marker keeps the 140-character prose guard, a dotted code outside the table (`Item 7.09`) included. Across 53 current 8-Ks the outline holds all 112 item codes EDGAR lists for them (24 before); without the bound exemption it holds 89, without the abutting/suffix form 103 (#165). When an all-caps marker line (`ITEM 2.02`) both matches the all-caps arm alone and joins the title below it, and both entries survive dedup at one offset, the bare entry is dropped — one heading, listed once. Because the period is optional, an unpunctuated same-line Item is dropped as wrapped prose (`isWrappedProse`) when its next word starts lowercase or the line above ends on a comma or semicolon, a title run straight into its code included (`…in Part II,` over `Item 8Financial Statements of this report`, #180). A line ending on a lowercase word is no evidence: a running page header (`Table of Contents`) sits directly above an unpunctuated body heading (`Item 5 — Other information`) at the top of a page. A plain-text 10-K's cross-reference that wraps to start a line with `Item 8 on this Form 10-K…` otherwise takes the `item 8` landing ahead of the real heading (#136). **A bare marker is admitted on the shape of what follows, not just on a title being there:** a blank line between marker and title admits the pair; without one, the pair is admitted only when no bare page-number line sits under the title, since marker/title/page-number on three tight lines is the TOC-cell artifact #71's guard exists to reject. That test is one-way — it rejects a TOC row, it does not certify a heading — so a marker line anywhere else (a page footer, an unwrapped paragraph) is admitted too, taking the next line as its title; the entry carries the marker's own offset, so `section` still lands in the right place. A heading text repeated `RUNNING_HEADER_MIN_OCCURRENCES` (5) times at distinct offsets is page furniture — C3is' FY2025 20-F repeats `TABLE OF CONTENTS` 129 times and a 10-Q stamps `PART I` on all 47 of its pages, while a heading printed once in the TOC and once in the body occurs twice. **Furniture is then handled by shape, two ways:** a bare marker (`Item 1`, `PART I`) carries no navigable text, so no occurrence of it earns an entry and every one is dropped; anything else keeps its FIRST occurrence and drops the rest, inverting the keep-later rule — a filing that runs a section's own heading across every page of that section (`NOTES TO THE CONSOLIDATED FINANCIAL STATEMENTS` through an S-1's financial statements) prints it first at the section start, and keep-later would land it on the section's last page instead. So the outline carries no fused composite and at most one entry per running header, at its first occurrence. **A bare marker is counted on its own line, never on the joined heading:** a page footer's marker takes its title from whatever prose the next page opens with, so every composite it produces is a different string and only the marker line repeats — count composites instead and a 10-Q's per-page `Item 1` footer yields one bogus entry per page, crowding the real Items past `maxEntries`. That count is per rendered form (`Item 1` and `Item 1.` are separate), since a filing stamping the bare form on every page still writes the punctuated one for the real heading. Dedup (keep the later, i.e. body, occurrence) keys on `headingKey` — `foldForHeadingMatch`, a terminal-punctuation strip, and the separator between an Item/Part marker and its title folded to one space — never raw lowercase: the same filing prints one marker with different space characters and a trailing period in its TOC and its body (`Item 3.` with U+0020 vs U+2009; `…Policies` vs `…Policies.`), and a raw key leaves both in the outline. The separator is the third difference: Worthington Steel's 10-Q (0001968487-26-000036) prints `Item 1A. Risk Factors` in its TOC and `Item 1A. – Risk Factors` in its body, so both kept an entry and `risk factors` landed on the TOC row at 4292 instead of 250397. A marker key is rebuilt as the marker, one space, and the title (`item 1a risk factors`), whatever period, dash, or colon stood between them (`MARKER_TITLE_RE`, exported for the section matcher); across the 67-filing corpus that drops the 10-Q's 11 TOC copies (34 → 23 entries) and moves no other outline entry (#174). A title run straight into its marker gets its space where detection ends the marker, before the fold lowercases the capital that marks it (`RUN_IN_MARKER_RE`): `Item 1Financial Statements` keys as `item 1 financial statements`, not `item 1f inancial statements`, so it dedups against a TOC row's `Item 1. Financial Statements`, `dropBarePartsTitledLater` reads `Part IFinancial Information` as a titled Part I, and `section: "part i"` lands on it as Part I's heading (#180). `section` reads the same space in through `spaceRunInMarker`: the Item/Part test (`isStructuralHeading`, which the outline cut shares) and `sectionMatches`' operands (`foldForSectionMatch`, heading and value alike), so the numeral ends there too — `part i` reaches New Peoples' `Part IFinancial Information` (4136) in tier 2 instead of through the line tier with a line notice, and a run-in Part past the 50th heading stays in a cut outline, while `part i` never reaches `Part IIOther Information` or `Part IVExhibits` and `item 1` never `Item 12Security Ownership`. A run-in line keys one character longer than it prints, so the line scan's length short-circuit allows that one character. The furniture count, and the copy furniture keeps, key on the printed form instead — `headingKey` short of the separator fold (`printedKey`) — so a TOC row and a body heading never pool with a running header printed with another separator and lose the body heading to the TOC row. A TOC row's trailing page cell — a page number or a lettered page (`F-3`), optionally after a `Page` word (`Page 1`), set off by a tab or two or more spaces (`TOC_PAGE_TAIL_RE`) — is stripped from every heading for the same reason. Without the `Page` form, First Bancorp's 8-K TOC row `Item 8.01 Other Events Page 1` (0000765207-26-000110) kept its cell, never deduped against the body `Item 8.01 Other Events.`, and took `item 8` at 1998 instead of 2227. A number after a single space is the heading's own (`PROPOSAL NO. 1`, `ARTICLE 5`, `…Securities Exchange Act of 1934`): the single-space strip cut it off, and dedup then merged GT Biopharma's three `PROPOSAL NO. N` headings into one `PROPOSAL NO.` at the last (#172). Across 81 cached filings the change moved nothing else in any outline. Lettered sub-item markers (a bare `A.`/`D.` inside an Item) are deliberately not detected — single letters recur throughout filing prose — and neither is a digit-led sub-item line (`3D. RISK FACTORS`; the all-caps arm needs a leading letter): `section` reaches a labelled line by stripping the label at comparison time instead (#175, see Section matching), so no detected heading is added.
|
|
108
111
|
- **Section matching:** `get_filing`'s `section` compares through `foldForHeadingMatch` (`filing-to-text.ts`) on **both** operands — Unicode whitespace runs collapse to one space, U+2018/U+2019 → `'`, U+201C/U+201D → `"`, then lowercase. Comparison-time only: `FilingHeading.heading`, the `outline` output, `content[]`, and every character offset keep the document's own bytes. EDGAR headings carry NBSP runs and typographic quotes, so a caller re-sending an outline heading through a UI that normalizes either one never byte-matched the heading it came from — which broke the `section_not_found` recovery loop the error's own hint points at (#106). No fuzzy matching — `"item 8"` resolves only to Item 8 — but every tier runs on folded operands, so a single-spaced needle also reaches a heading whose marker carries an NBSP run or a double space (`Item 9.` in a 2004 10-K). Folding only one operand does not fix the round trip. One boundary (`sectionMatches`): a needle ending in a number never continues into a longer one. A trailing digit never runs on into another digit, so `item 1` matches `Item 1.` and `Item 1A` but never `Item 12`, and `item 5` reaches an 8-K's `Item 5.02` while `item 5.0` does not. A trailing roman numeral (a final word of `i`, `v`, `x`) must end at a space, punctuation, or the heading's end, so `part i` never matches `PART II`, `PART III`, `PART IV`, or `PART II, III`: Sono-Tek's 10-Q (0001171520-26-000305), which prints Part I only inside a one-line TOC, landed `part i` on `PART II - OTHER INFORMATION`, and Microsoft's FY2026 10-K on `PART II, III` (348204). `part i` now falls through to the line tier (Microsoft: the body `PART I` at 6900) or misses with the outline (Sono-Tek; an S-1, whose Part I is the unlabeled prospectus). Plain-text outlines keep TOC rows worded differently from their body heading, and a TOC row for Item 10–16 precedes the body Item 1 — a bare substring test landed `item 1` on another Item in 6 of 37 plain 10-K/10-Q outlines and 1 of 270 HTML landings (#136). **Resolution order (#169, #166, #177)**, first hit wins, over every detected heading (`detectHeadings(text)`, which takes no cap, run once per call — the 50-entry cap belongs to the rendered outline alone; with the first 50 only, a heavy 10-K's financial-statement captions put Items 9B–16 out of reach) and the document's own lines: (1) **exact** (`resolveSection`) — a heading whose `headingKey` (the dedup key: the fold, a terminal-punctuation strip, and the marker separator folded to one space) equals the needle's, or whose title after its Item/Part marker does — failing that, the same with a leading sub-item label removed (below); a bare marker (`ITEM 1.`, `Part II`) has no title and is never a candidate, or GSBD's `item 1` jumps to a bare TOC `ITEM 1.`. The tier exists for #170's widened all-caps class: CCC's `SPECIAL NOTE REGARDING FORWARD-LOOKING STATEMENTS AND RISK FACTORS` (7562) otherwise takes `risk factors` from `Item 1A. Risk Factors.` (67351). (2) **Item/Part substring** — the first Item or Part heading (`isStructuralHeading`, bare and run-in markers included) that `sectionMatches`: a value inside an Item title names that Item, while a line equal to it is often a table-row label (CCC's `compensation` → `Item 11. Executive Compensation` at 493087, not the `Compensation` cell at 418658). (3) **an equal line** and (5) **a table-row label, else a prefix line** — a document line (`matchSectionLine`, `filing-to-text.ts`, which reports whether its line ranks in tier 3; `findSectionLine` is its offset alone), for a mixed-case subheading the outline never lists (`Liquidity and Capital Resources`): a line whose `headingKey` equals the needle's and that labels no table row (3), else — only when no such line exists — the first equal line that labels one (5, below), else one that starts with it (5) at a word boundary within `MAX_TITLE_CHARS` (a date-suffixed title: `Consolidated Schedule of Investments as of June 30, 2026`), unless a one-word needle runs on into a longer Title Case title (`continuesIntoLongerTitle`: the next word opens on a capital in a line that is not all caps). One word heads many unrelated titles — `business` landed on `Business Combinations.` (C3is), `Business Seasonality and Product Introductions` (Apple 10-Q), and a cross-reference line `Business Overview—Governmental regulations…` (DRDGOLD); a needle of two or more words keeps its continuation (`Human Capital` → `Human Capital Management`), and a suffix opening lowercase or on punctuation (`(continued)`, `—Parent Company Only`) always does. The cost: `forward-looking` no longer reaches `Forward-Looking Statements`, nor `revenue` `Revenue Recognition`; both miss with the outline. Equality goes first because a prefix match alone lands `Competition` on an earlier line that merely begins with the word. **A table-row label ranks in tier 5, ahead of a prefix line (#184):** an equal line labels a row when its next line with content (spacer lines passed over) is a figure row, or is a label — at most 60 characters, opening uppercase, ending on no `.`, `:`, `;`, or `,`, closing quotes and brackets passed over (a cross-reference `See “Item 18. Financial Statements.”` over its page number is no label) — whose next line with content is one (`labelsTableRow`; Microsoft's segment table `Revenue` / `Client` / `$ 9,350 …`). A figure row is a lone currency or nil cell (`$`, `-`, `–`, `—`), or a line whose text after its last letter holds a digit and, when the line has a letter, opens on a dotted leader, a tab, or two or more spaces behind a label of at most 60 characters (`isFigureRow`: `Product sales....... $4,759`; ``$ 7,746 $ `7,835``, a stray backtick in a 2004 cell); one space never qualifies (`The effective rate was 21%.`), and past the label bound the line is a table set on one line, which a heading may sit over (AMC Robotics' S-1 0001493152-26-047958: MD&A `Revenue` at 177926 over `Three months ended June 30, 2026 2025 $ Change …`). All 13 `Revenue` lines in Microsoft's 2004 10-K (0001193125-04-150689) label rows, and as tier-3 lines took `revenue` from `SEGMENT PRODUCT REVENUE/OPERATING INCOME (LOSS)` (81018) to the selected-data row at 66987; Aware's `Revenue:` over `Product sales....... $4,759` (112947) took it from `CONTRACT REVENUE` (49489). Demoted rather than skipped, a row label still lands when no heading contains the value (`total revenue` keeps its income-statement row), under the plain line notice. The test reads at most two content lines past an equal line. Over 26 corpus filings × 104 section values it labels 124 lines on the resolution path, none a detected heading, and moves 22 landings and no outline entry: 15 from a row label onto a heading or prose subheading for the value (Microsoft FY2026 `revenue` → its MD&A `Revenue` subheading at 217873; `earnings per share` → the EPS note in both Microsoft 10-Ks), 2 onto a table's caption line, 1 onto another row label (BAB's `goodwill`, from its balance-sheet row to a note table's), and 4 onto a heading that only contains the words (Arxis `net income` → `NOTE 16. NET INCOME (LOSS) PER SHARE`, and Nutanix's → `NOTE 11. NET INCOME (LOSS) PER SHARE` past a `Net income` row of `—` cells; CCC `net income` → the all-caps row label `NET INCOME (LOSS) INCLUDING NON-CONTROLLING INTEREST`, a detected heading; CCC `revenues` → `COST OF REVENUES` at 332106, since all four of its `Revenues` lines label rows). In both line tiers a line is skipped when it opens lowercase (a wrapped sentence or bullet); when it sits under a line ending on a comma or semicolon, or opens on an unpunctuated Item marker followed by a lowercase word (a wrapped cross-reference, the `followsMidSentence` / `isWrappedProse` guards heading detection applies: `…in Part II,\nItem 8 on this Form 10-K…`); or when it is a TOC row (`isTocRow`). A TOC row ends in a page cell set off by **two or more** spaces or a tab (`TOC_PAGE_TAIL_RE`; one space would read the year in `as of June 30, 2026` as a page number), or its next line with content is a page cell (`PAGE_CELL_LINE_RE`: a number, `F-3`, or a lowercase roman `ii`, which counts only on its own line — `Part iv` is not a page cell); either form may put a `Page` word before the number (`Page 3`), spacer lines passed over (an HTML TOC table puts NBSP rows between its cells). A line that is not a marker, followed directly by an Item/Part marker, is a TOC title cell with no page column; a marker run straight into its title counts (`MARKER_LINE_RE` takes `TITLE_RUN_IN`, #180: `Item 2Properties`). A marker line — a Part over its first Item — is a TOC row when that Item row is: the Item line ends in a page cell, the line below it holds or ends in one (a title cell, or Aware's wrapped `of Equity Securities....... 12`), or, for a bare marker, the line below its title holds one. Microsoft's FY2026 10-K prints body `PART IV` only as page furniture, so `part iv` reaches the line tier, where `PART IV / Item 15. / Exhibit and Financial Statement Schedules / 93` was taken; it now lands on the body's first `PART IV` (352884). A body Part sits over its Item too, but prose follows, not a page cell. The first remaining line wins, at its first non-blank character; one linear pass serves tiers 3 and 5, run only after tiers 1 and 2 miss, so a value landing on a tier-4 heading now pays that pass (7–11 ms on a 1 MB S-1). A cut page's guidance after a `section` landing says to pass `next_offset` as `offset` without `section`, which outranks `offset` — kept, it returns the same page. A line landing says so in the notice, composed into the truncation guidance when the page is cut (`notice` is last-wins): `No detected heading matches section '<value>', so this page starts at the first document line that does.` — no new output field. An equal line that outranked a heading merely containing the value says `No detected heading equals section '<value>', so this page starts at the first document line that does, ahead of headings that only contain it.` instead, since that heading does match by substring. Every #166 acceptance offset holds. (4) **other substring** — the first other heading that `sectionMatches`, after the equal line: a heading that merely holds the word took Arxis' S-1 `business` to `NOTE 3. BUSINESS COMBINATIONS` (544705) past its `Business` heading line (323578), and CCC's `revenue` to `COST OF REVENUES` (332106) past the MD&A `Revenue` subheading (256790). An exact or Item/Part heading still beats an equal line (Reddit's `use of proceeds` → the `USE OF PROCEEDS` heading); every other heading ranks after one. Over 67 corpus filings × 103 section values the order moved 24 landings, each one the validation predicted: 17 onto the section named (Microsoft FY2026 `general` → `GENERAL` at 8720, not `GENERAL RISKS`; DRDGOLD's `risk`, `directors`, `management`, and `employees` → body Items instead of TOC rows), and 7 onto a different place that is defensible but that a reader may like less, since an Item heading or a line equal to the value now outranks a heading that only holds it — JPM's 10-Q `risk` lands on Item 3, a one-sentence cross-reference, not `FIRMWIDE RISK MANAGEMENT`; `revenue` in Microsoft's 2004 and FY2026 10-Ks and Aware's on a selected-data, MD&A-table, or income-statement row label rather than a heading containing the word — the three landings tier 5's row labels move (#184, above); `summary` on Item 16 `Form 10-K Summary`; Intel's 2000 `consolidated balance sheets` on an Item 14 index row. (6) `section_not_found`. **A proxy statement keeps its headings ahead of the lines:** for a proxy or information statement form (`PROXY_FORM_RE`: DEF 14A, DEFA14A, DEFM14A, PRE 14A, DEF 14C, …), tier 1 is followed by the first heading of any kind that `sectionMatches`, then the line tiers — its ITEM headings are ballot proposals, not the Regulation S-K item taxonomy, and its lines equal to a value are mostly table cells. Under tiers 2–4 Goldman Sachs' DEF 14A (0001193125-26-117433) took `compensation` from the CD&A (123680) to `ITEM 2. … (SAY ON PAY)` (274744) and `summary` from `EXECUTIVE SUMMARY` (16953) to a Pay versus Performance column header (282286). The form is read before `section` resolves, so the `.hdr.sgml` fallback request (#126) now precedes a `section_not_found`. **Inside each heading tier a body heading ranks ahead of a table-of-contents row (#174):** a heading is a TOC row when its title line — its own line, or the next line with content after a bare or joined marker — is one by `isTocRow` (the line tiers' test), or the next line with content below it ends in a page cell (a TOC title wrapped onto a second line, Aware's `of Equity Securities....... 12`), or the title wraps once with its page cell on its own line under the wrap — the title line, one continuation line, and a `PAGE_CELL_LINE_RE` line on three consecutive lines, the continuation in the title line's case: no lowercase letter under an all-caps title line, an opening lowercase one under a mixed-case one (`of Equity Securities`) (#183). DRDGOLD's TOC row `ITEM 14.` / `MATERIAL MODIFICATIONS TO THE RIGHTS OF SECURITY HOLDERS AND USE OF` / `PROCEEDS` / `79` took `item 14` and `material modifications to the rights of security holders` at 8146 from the body heading at 433144. A body heading over one line of body text and a page number has the same three lines: across 26 filings the step without the consecutive-line and case conditions flags 47 body headings (`Item 4. Mine Safety Disclosures` / `Not applicable.` / `31`, spacer lines between or not, and DRDGOLD's `NOTE 22 ASSET HELD FOR SALE` over its page footer and `F-50`); with them it flags only that TOC row (8146) and its title cell (8155), moves those two landings, and changes no outline. An all-caps body heading over a one-word all-caps line and a page number on consecutive lines (`ITEM 4. MINE SAFETY DISCLOSURES` / `NONE` / `31`) still reads as a TOC row and lands as its tier's only match; `isTocHeading` in `filing-to-text.ts` owns the test, finding the line start from the newline before the offset since an NBSP-led Item's offset sits one past it. A TOC row still lands when it is its tier's only match. Three TOC shapes survive dedup: a row worded apart from its body heading (First Bancorp's 8-K `Item 9.01 Financial Statements and Exhibits. Page 1` took `item 9` at 2041 from the body Item at 2402), a bare all-caps marker over its title and page cells (DRDGOLD's FY2026 20-F `ITEM 3.` / `KEY INFORMATION` / `6` took `item 3` at 6174 from `ITEM 3. KEY INFORMATION` at 34346, and nearly every other `item N` the same way), and the title cell itself (`key information`). TOC rows are ranked, never dropped or pushed behind the line tiers: the test also reads a body heading over a page number (CCC's `Item 6. Reserved.` at 234199, Reddit's S-1 `PART II` at 1021631), and behind the line tiers CCC's and Microsoft's `item 6` left the body heading for a bare TOC marker line. With the dedup-key fold, the ranking moved 83 landings over the corpus, each from a TOC row onto a body heading (DRDGOLD 35, Worthington Steel 20, GSBD 9, Amazon's 2003 10-K 8, Aware 6, First Bancorp 4, CCC 1), and no other: 78 onto the body heading the row copies, 5 onto another the value names (Amazon 2003's `part ii`–`part iv` onto its body Parts, Aware's `management` from its TOC Item 12 to body Item 7, Worthington Steel's `other information` from its TOC Item 5 to body `PART II. OTHER INFORMATION`). **A leading 20-F sub-item label is ignored (#175):** 20-F filers label sub-items (`3D. RISK FACTORS`, `B. Business Overview`), so a plain-phrase value never equalled the line. Tier 1 and both line tiers also compare the heading or line key with a leading label removed (`withoutSubItemLabel`: `/^(?:\d{1,2}[a-z]|[a-z])\. /` on the folded key, anchored, so constant-time), and the one-word longer-title rule judges the line after its label — Regencell's `B. Business Overview` keeps `business` a miss while `business overview` lands (151573). The value itself is never stripped, so `3D. RISK FACTORS` still matches its own line; a line opening on an initial (`L. John Doerr`) strips the same way and so matches only a value equal to the rest of the line. The landing is the line's first character, label included (DRDGOLD `risk factors` → 34503). Tier 1 strips only once no heading equals the value as printed, so an earlier `A. RISK FACTORS` never takes `risk factors` from a later `Item 1A. Risk Factors`; a labelled body heading still ranks ahead of a TOC row. The line tiers keep the first line equal either way: across the corpus the labelled line they land on is the 20-F sub-item itself and a later unlabelled one is narrower (DRDGOLD `business overview` → `4B. BUSINESS OVERVIEW` at 176975, not `Business overview` at 269017). Over the corpus the strip moved 23 landings, all in the three 20-Fs: 19 misses now land on the labelled sub-item, and 4 earlier landings move onto one (DRDGOLD `compensation` from the `Compensation of key management` subheading inside Item 6B, 328266, to `6B. COMPENSATION` at 324690); no filing without labelled sub-item lines changed. **The outline cut (#169):** past 50 detected headings the outline is every Item/Part heading (`isStructuralHeading`, bare and run-in markers included) plus the earliest other headings up to 50, in offset order (`selectOutline`) — shared by the first-page `outline` and the `section_not_found` message and data. When the Item/Part headings alone pass 50, each bare marker that a later titled heading for the same Item or Part repeats is left out first (`outlineStructuralHeadings`): DRDGOLD's FY2026 20-F renders all 31 TOC Items as bare markers (`ITEM 3.`), which filled the cut and pushed body Items 16A–19 out; its outline now runs `ITEM 1.` through `ITEM 19. EXHIBITS` from the body. A bare marker no titled heading repeats stays, and `section` still matches every heading. The miss message heads a cut outline `Outline (50 of N detected headings):` (else `Outline:`), and a cut first page appends `The outline lists 50 of N detected headings, every Item and Part heading among them; section matches all of them.` to the truncation guidance — `…among them, bare table-of-contents copies of titled ones aside; …` when bare markers were left out (DRDGOLD: `The outline lists 50 of 142 detected headings, every Item and Part heading among them, bare table-of-contents copies of titled ones aside; section matches all of them.`), and no Item/Part clause when the kept Item/Part headings alone pass 50, which reaches `structuredContent.notice` and `content[]` — no new output field, since the outline is emitted only on a truncated first page. A document of 50 or fewer headings renders as before.
|
|
109
112
|
- **Binary filing documents:** a filing index routinely carries scanned pages, PDF exhibits, `Financial_Report.xlsx`, and `-xbrl.zip` — `filingToExtract` runs `html-to-text` over whatever bytes arrive and never inspects a content type, so requesting one used to return decoded binary as `content` with no error (#96). `get-filing.tool.ts` detects them with one rule, `binaryType`: the filename extension (`binaryTypeFromName`), the one signal every filing carries, plus a header-page `GRAPHIC` type for a scan named oddly — the header page is read before the body (#161). Two halves, both needed — `documents[]` entries carry `binary: true`, and `resolveFilingArchive` throws `binary_document` before `tryGetFilingDocument` runs, since the `document` input is an unconstrained string and a catalog marker alone cannot stop a caller. Extension-driven and bucket-independent on purpose: a PDF exhibit is typed `EX-99.*` and files under `exhibits`, not among the graphics.
|
|
110
|
-
- **Dataframes:** `secedgar_search_filings`, `secedgar_get_financials`, `secedgar_fetch_frames`, `secedgar_compare_companies`, `secedgar_get_insider_transactions`, `secedgar_get_institutional_holdings`, `secedgar_get_material_events`, `secedgar_find_holders`, `secedgar_get_beneficial_owners`, and `secedgar_get_fund_holdings` materialize their full result set as `df_<id>` on a shared DuckDB-backed canvas (one per tenant). The two ownership tools register the full parsed set (inline response is a preview capped at `limit`); `get_insider_transactions` scans at least `INSIDER_CANVAS_FILING_SCAN` recent filings when canvas is available (so small inline limits don't under-scan the window), capped at 100, and flags `dataset.truncated` when more Form 4 filings exist beyond the scanned window (#63); with a `filed_after` / `filed_before` window it reads the in-window Form 4 / 4-A rows with an XML primary document — `recent`, then the archive pages overlapping the window, newest first, stopping once the scan cap plus a sentinel row are in hand — and with a canvas parses every one up to 100 instead of the 40-filing floor, since the window already bounds the set; `dataset.truncated` is then the sentinel or unread overlapping pages, and `history_scanned_through` the oldest filing parsed (#127); a company's submissions feed also lists the Form 4s it filed as a reporting owner of another issuer (JPMorgan's 10% stake in a BlackRock fund), so a parsed document whose `issuerCik` differs from the resolved CIK, compared with zero-padding stripped, adds no row inline or to the dataframe and is counted in `filings_other_issuer` — it still counts toward `filings_scanned`, the scan floor, and `history_scanned_through`, and the no-canvas stop rule is unchanged — though a skipped filing adds no transactions, so a no-canvas call reads on through such filings until `limit` transactions are in hand, within the same scan cap (#157); its `shares_traded` is an unsigned magnitude paired with a `direction` (acquire/dispose) column, so net activity is `SUM(CASE WHEN direction='dispose' THEN -shares_traded ELSE shares_traded END)` (#46); `get_financials`' dataframe materializes the source-filing fiscal keys as `source_filing_fy`/`source_filing_fp` (renamed from bare `fiscal_year`/`fiscal_period` — every comparative period restated in one filing carries that filing's fy/fp, so `period_end` is the time key for ORDER BY/GROUP BY/windowing) (#72); `compare_companies`' dataframe holds the FULL aligned series across every period while the inline matrix is capped at `periods`, and each row carries both the aligned `period` key and the underlying XBRL `frame` (they differ for point-in-time concepts) (#85). In an annual matrix an instant enters only on one of the filer's fiscal-year ends and takes the key of the annual duration ending that day. The ends come from the companyfacts payload already in hand (`filerFiscalYearEndsWithComparatives` in `concept-series.ts`, shared with `get_financials`' annual view of an instant): each annual report's own year (the #148 rule) plus the comparative years it restates — its year-long facts ending a whole number of years earlier, within a week — so the first XBRL 10-K's prior-year balance keys too, while a year-long figure ending off the report's calendar (a December filer's trailing year to 09-30) stays out. The #148 test of a 10-Q-held annual frame keeps `filerFiscalYearEnds` (each report's own year only): it matches day of year, where a comparative year adds nothing but a wider tolerance. The key is the first `CY####` frame on any concept in the payload ending then, so a balance lands beside its fiscal year's income lines even when none is requested; with no framed duration ending on it, the calendar year in which that fiscal year's second half starts (SEC's own framing: a year ending 06-30 is that year, one ending 06-27 or 05-31 the year before — Seagate's and Western Digital's years ending 2025-06-27 are `CY2024`; SEC departs from it for the year both ended 2012-06-29, framed `CY2012`, which is why a framed key wins). Interim and cover-date instants stay out of the annual matrix, and a company-concept left with none gets a gap plus a caveat pointing at quarterly; when that empties the whole matrix, `no_comparable_data` says the values were all off fiscal-year ends and its hint names `period_type: "quarterly"` (omitting `period_type` keeps annual). Quarterly keys are unchanged. The year ends are read lazily, only when an annual matrix meets an instant (#171). The period-ends caveat checks every inline column — the newest can hold one company alone — spelling out the newest column whose ends differ and naming the others; the inline window is the newest periods of the union across companies, so a company-concept pair whose values all predate it is named in `caveats` — one line per concept, each company with its newest period and the dataframe pointer — and never in `gaps`, which keeps meaning no value in any period (#144). Each row set carries an all-nullable schema (framework's `inferSchemaFromRows` always emits `nullable: true` since mcp-ts-core 0.10.4). Per-table TTL is delegated to the framework via `RegisterTableOptions.ttlMs` (resolved in mcp-ts-core#140); bridge still tracks metadata expiry in `ctx.state` for provenance. Every producer emits the describe-then-query pointer on the branch that registers the dataframe, composed once by `dataframeGuidance(dataset)` in the bridge so the wording cannot drift; where a truncation call already fires there it rides as that call's `guidance`, since `ctx.enrich.notice` is last-wins across `notice`/`truncated` and a second call would clobber the first (#104). `secedgar_dataframe_query` detects a `row_limit`-bound result from `QueryResult.truncated` rather than `rowCount > rows.length` — `row_limit` is pushed into the query as the provider's cap, so a capped result comes back with `rowCount === rows.length` and the arithmetic cannot see it; `rowCount` is then the cap and not a total, so that branch names the cap instead of printing "of N rows", `row_count_capped` carries the same disclosure into `format()`, and the fallback comparison stays for the `registerAs` path, where `truncated` is never set. A SQL `LIMIT` exactly equal to the cap is indistinguishable from a result that holds that many rows and is reported as exact (#109). `secedgar_dataframe_query` runs framework's SQL gate with `denySystemCatalogs: true` to block `information_schema`, `pg_catalog`, `sqlite_master`, and `duckdb_*` catalog access. Raw DuckDB execution errors (missing table, syntax) are caught in `bridge.query` and re-thrown as structured `missing_table`/`invalid_sql` reasons, with DuckDB's "Did you mean …?" hint stripped so internal catalog names don't leak (#47). The framework's own classifications pass through untouched and are declared on the tool's contract (`thrownBy: 'service'`, since the handler never names them): a SELECT that prepares and then fails on the data — a `Conversion` / `Invalid Input` / `Out of Range` error, e.g. `CAST('abc' AS INTEGER)` — is `ValidationError` with `sql_execution_error` (mcp-ts-core 0.13.4; `DatabaseError` with no reason before), and the gate's `multi_statement`, `denied_function`, `sql_read_only`, and `plan_operator_not_allowed` rejections arrive with their own recovery hints — since mcp-ts-core 0.13.15 the gate also denies engine-setting and SQL-from-string functions (`current_setting`, `query_table`) as `denied_function` and a file named as a table (`FROM 'data.csv'`) as `sql_read_only`, and since 0.13.16 it refuses any unqualified `current_schema`, since the parse tree cannot tell the bare function from a column of that name — a staged or aliased column so named is read qualified (`c.current_schema`). `get_beneficial_owners` registers one row per reporting person rather than one per filing, since powers and percent of class are per person; `get_fund_holdings` registers every position in the report while the inline list is one `limit`-sized page from `offset`, and its rows carry the fund keys (`series_id`, `registrant_cik`, `report_period_date`, `accession_number`) so a portfolio joins the 13F and 13D/G rows on `cusip`. Insider rows carry `issuer_cik` and no `cusip`, 13F rows the reverse; 13D/G rows carry both.
|
|
111
|
-
- **Beneficial ownership (13D/13G):** SEC replaced the legacy `SC 13D` / `SC 13G` text filings with structured XML under the `SCHEDULE 13D` / `SCHEDULE 13G` names on **2024-12-18**; the changeover is total, so a legacy-name filter matches nothing recent and a current-name filter matches nothing older. The two schedules are **separate schemas, not one shape with optional fields** — 13D repeats `reportingPersons/reportingPersonInfo` with the power fields flat on each person and the percentage in `percentOfClass`; 13G repeats `coverPageHeaderReportingPersonDetails` with the powers nested inside `reportingPersonBeneficiallyOwnedNumberOfShares`, the percentage in `classPercent`, no reporting-person CIK anywhere, and no purpose-of-transaction field at all (its item 4 is structured ownership data, which is what makes it the passive form). `beneficial-ownership-parser.ts` therefore branches per schema and dispatches on the submission type, falling back to whichever repeated reporting-person element the document carries. **Ownership powers and percent of class are per reporting person, including on joint filings** — a fund, its adviser, and its controlling principal each report the same underlying shares, so the parsed owner list is never collapsed to a group total and the percentages must not be summed. Discovery runs off the issuer's own `getSubmissions()`: SEC cross-lists a schedule under every CIK party to it, so no entity-scoped EFTS step is needed.
|
|
113
|
+
- **Dataframes:** `secedgar_search_filings`, `secedgar_get_financials`, `secedgar_fetch_frames`, `secedgar_compare_companies`, `secedgar_get_insider_transactions`, `secedgar_get_institutional_holdings`, `secedgar_get_material_events`, `secedgar_find_holders`, `secedgar_get_beneficial_owners`, and `secedgar_get_fund_holdings` materialize their full result set as `df_<id>` on a shared DuckDB-backed canvas (one per tenant). The two ownership tools register the full parsed set (inline response is a preview capped at `limit`); `get_insider_transactions` scans at least `INSIDER_CANVAS_FILING_SCAN` recent filings when canvas is available (so small inline limits don't under-scan the window), capped at 100, and flags `dataset.truncated` when more Form 4 filings exist beyond the scanned window (#63); with a `filed_after` / `filed_before` window it reads the in-window Form 4 / 4-A rows with an XML primary document — `recent`, then the archive pages overlapping the window, newest first, stopping once the scan cap plus a sentinel row are in hand — and with a canvas parses every one up to 100 instead of the 40-filing floor, since the window already bounds the set; `dataset.truncated` is then the sentinel or unread overlapping pages, and `history_scanned_through` the oldest filing parsed (#127); a company's submissions feed also lists the Form 4s it filed as a reporting owner of another issuer (JPMorgan's 10% stake in a BlackRock fund), so a parsed document whose `issuerCik` differs from the resolved CIK, compared with zero-padding stripped, adds no row inline or to the dataframe and is counted in `filings_other_issuer` — it still counts toward `filings_scanned`, the scan floor, and `history_scanned_through`, and the no-canvas stop rule is unchanged — though a skipped filing adds no transactions, so a no-canvas call reads on through such filings until `limit` transactions are in hand, within the same scan cap (#157); its `shares_traded` is an unsigned magnitude paired with a `direction` (acquire/dispose) column, so net activity is `SUM(CASE WHEN direction='dispose' THEN -shares_traded ELSE shares_traded END)` (#46); `get_financials`' dataframe materializes the source-filing fiscal keys as `source_filing_fy`/`source_filing_fp` (renamed from bare `fiscal_year`/`fiscal_period` — every comparative period restated in one filing carries that filing's fy/fp, so `period_end` is the time key for ORDER BY/GROUP BY/windowing) (#72); `compare_companies`' dataframe holds the FULL aligned series across every period while the inline matrix is capped at `periods`, and each row carries both the aligned `period` key and the underlying XBRL `frame` (they differ for point-in-time concepts) (#85). In an annual matrix an instant enters only on one of the filer's fiscal-year ends and takes the key of the annual duration ending that day. The ends come from the companyfacts payload already in hand (`filerFiscalYearEndsWithComparatives` in `concept-series.ts`, shared with `get_financials`' annual view of an instant): each annual report's own year (the #148 rule) plus the comparative years it restates — its year-long facts ending a whole number of years earlier, within a week — so the first XBRL 10-K's prior-year balance keys too, while a year-long figure ending off the report's calendar (a December filer's trailing year to 09-30) stays out. The #148 test of a 10-Q-held annual frame keeps `filerFiscalYearEnds` (each report's own year only): it matches day of year, where a comparative year adds nothing but a wider tolerance. The key is the first `CY####` frame on any concept in the payload ending then, so a balance lands beside its fiscal year's income lines even when none is requested; with no framed duration ending on it, the calendar year in which that fiscal year's second half starts (SEC's own framing: a year ending 06-30 is that year, one ending 06-27 or 05-31 the year before — Seagate's and Western Digital's years ending 2025-06-27 are `CY2024`; SEC departs from it for the year both ended 2012-06-29, framed `CY2012`, which is why a framed key wins). Interim and cover-date instants stay out of the annual matrix, and a company-concept left with none gets a gap plus a caveat pointing at quarterly; when that empties the whole matrix, `no_comparable_data` says the values were all off fiscal-year ends and its hint names `period_type: "quarterly"` (omitting `period_type` keeps annual). Quarterly keys are unchanged. The year ends are read lazily, only when an annual matrix meets an instant (#171). The period-ends caveat checks every inline column — the newest can hold one company alone — spelling out the newest column whose ends differ and naming the others; the inline window is the newest periods of the union across companies, so a company-concept pair whose values all predate it is named in `caveats` — one line per concept, each company with its newest period and the dataframe pointer — and never in `gaps`, which keeps meaning no value in any period (#144). Each row set carries an all-nullable schema (framework's `inferSchemaFromRows` always emits `nullable: true` since mcp-ts-core 0.10.4). Per-table TTL is delegated to the framework via `RegisterTableOptions.ttlMs` (resolved in mcp-ts-core#140); bridge still tracks metadata expiry in `ctx.state` for provenance. Every producer emits the describe-then-query pointer on the branch that registers the dataframe, composed once by `dataframeGuidance(dataset)` in the bridge so the wording cannot drift; where a truncation call already fires there it rides as that call's `guidance`, since `ctx.enrich.notice` is last-wins across `notice`/`truncated` and a second call would clobber the first (#104). `secedgar_dataframe_query` detects a `row_limit`-bound result from `QueryResult.truncated` rather than `rowCount > rows.length` — `row_limit` is pushed into the query as the provider's cap, so a capped result comes back with `rowCount === rows.length` and the arithmetic cannot see it; `rowCount` is then the cap and not a total, so that branch names the cap instead of printing "of N rows", `row_count_capped` carries the same disclosure into `format()`, and the fallback comparison stays for the `registerAs` path, where `truncated` is never set. A SQL `LIMIT` exactly equal to the cap is indistinguishable from a result that holds that many rows and is reported as exact (#109). `secedgar_dataframe_query` runs framework's SQL gate with `denySystemCatalogs: true` to block `information_schema`, `pg_catalog`, `sqlite_master`, and `duckdb_*` catalog access. Raw DuckDB execution errors (missing table, syntax) are caught in `bridge.query` and re-thrown as structured `missing_table`/`invalid_sql` reasons, with DuckDB's "Did you mean …?" hint stripped so internal catalog names don't leak (#47). The framework's own classifications pass through untouched and are declared on the tool's contract (`thrownBy: 'service'`, since the handler never names them): a SELECT that prepares and then fails on the data — a `Conversion` / `Invalid Input` / `Out of Range` error, e.g. `CAST('abc' AS INTEGER)` — is `ValidationError` with `sql_execution_error` (mcp-ts-core 0.13.4; `DatabaseError` with no reason before), and the gate's `multi_statement`, `denied_function`, `sql_read_only`, and `plan_operator_not_allowed` rejections arrive with their own recovery hints — since mcp-ts-core 0.13.15 the gate also denies engine-setting and SQL-from-string functions (`current_setting`, `query_table`) as `denied_function` and a file named as a table (`FROM 'data.csv'`) as `sql_read_only`, and since 0.13.16 it refuses any unqualified `current_schema`, since the parse tree cannot tell the bare function from a column of that name — a staged or aliased column so named is read qualified (`c.current_schema`). `get_beneficial_owners` registers one row per reporting person rather than one per filing, since powers and percent of class are per person, and each row carries the filing's `purpose_of_transaction` and the person's `notes` uncut, the text the inline result bounds (#200); `get_fund_holdings` registers every position in the report while the inline list is one `limit`-sized page from `offset`, and its rows carry the fund keys (`series_id`, `registrant_cik`, `report_period_date`, `accession_number`) so a portfolio joins the 13F and 13D/G rows on `cusip`. Insider rows carry `issuer_cik` and no `cusip`, 13F rows the reverse; 13D/G rows carry both.
|
|
114
|
+
- **Beneficial ownership (13D/13G):** SEC replaced the legacy `SC 13D` / `SC 13G` text filings with structured XML under the `SCHEDULE 13D` / `SCHEDULE 13G` names on **2024-12-18**; the changeover is total, so a legacy-name filter matches nothing recent and a current-name filter matches nothing older. The two schedules are **separate schemas, not one shape with optional fields** — 13D repeats `reportingPersons/reportingPersonInfo` with the power fields flat on each person and the percentage in `percentOfClass`; 13G repeats `coverPageHeaderReportingPersonDetails` with the powers nested inside `reportingPersonBeneficiallyOwnedNumberOfShares`, the percentage in `classPercent`, no reporting-person CIK anywhere, and no purpose-of-transaction field at all (its item 4 is structured ownership data, which is what makes it the passive form). `beneficial-ownership-parser.ts` therefore branches per schema and dispatches on the submission type, falling back to whichever repeated reporting-person element the document carries. **Ownership powers and percent of class are per reporting person, including on joint filings** — a fund, its adviser, and its controlling principal each report the same underlying shares, so the parsed owner list is never collapsed to a group total and the percentages must not be summed. Discovery runs off the issuer's own `getSubmissions()`: SEC cross-lists a schedule under every CIK party to it, so no entity-scoped EFTS step is needed. **Filer text is data, kept as filed and bounded per field:** Item 4 `purpose_of_transaction` and each person's cover-page `notes` keep their line breaks (CR and CRLF as filed) and come back whole up to 5,000 and 2,000 characters; past that, the value is its first N characters with nothing appended and `purpose_truncated` / `notes_truncated` say so, on both surfaces. Item 4 is the substantive 13D field: 5,000 sits just above the 90th percentile of 48 sampled Item 4s (4,916), where 1,200 cut 34 of them, and uncut the 20 longest sampled make a 136.5 KB response; 2,000 clears the longest of 77 sampled notes (1,852), and a bound stays because reporting persons per filing are unbounded. When anything was cut, the cut count and the pointer to the whole text — the dataframe's uncut columns, or `secedgar_get_filing` on the accession — ride the tool's one composed notice. `format()` renders each value as a blockquote through `escapeMarkdownInline`, every line prefixed `> ` (indented under a note's list item) and the cut marker after it, behind a blank line so the marker and the next block cannot lazily continue the quote; reporting-person names, `security_class`, `issuer_name`, and the cover-page codes render escaped too (#200).
|
|
112
115
|
- **NPORT-P series routing:** a fund's public portfolio report covers **exactly one series**, a registrant trust holds many (iShares Trust: 390 listed series, 200+ reports for a single period), and the series identity appears **only inside the document** — neither the submissions feed nor EFTS carries `seriesId` on an NPORT-P row. Routing is therefore series-first: EDGAR's company browse accepts a series ID in place of a CIK (`getFundSeriesFilings`), which is the one SEC surface mapping a series to its accessions and which also names the registrant, so a bare series ID resolves in one request. The browse feed's registrant outranks any registrant the `fund` input implied, since `series_id` can name a series of an entirely different trust and the archive path is keyed on the registrant. The bounded header scan (`tryGetFilingDocumentHead` + `parseNportHeader`, cap `SERIES_SCAN_LIMIT`) is the fallback and the disambiguator for a trust whose series carry no listed ticker; SEC's archive host **ignores `Range`** (a ranged request answers 200 with the whole body), so the saving is client-side stream cancellation only — and the cutoff cannot cut below one read chunk, which Bun sets at 262,144 bytes, so `HEADER_MAX_BYTES` binds only on a document arriving in several chunks. **A registrant's own NPORT-P list is never the candidate set for a series-structured trust**: series fiscal quarters are staggered, so the list interleaves funds and one report at the newest period does not mean one fund — it means the fund whose quarter ends latest. The scan therefore runs on the newest batch even when that batch holds one report, and a series found there re-routes through the browse feed; only a report naming no series at all (a closed-end fund, which also carries no `classId`) is answered from the registrant's list. Candidates are ordered by **reported period, not filing date**: an `NPORT-P/A` restating an old period is filed after every report of the periods that followed it. Reports publish on a ~60-day lag, so `report_period_date` and `publication_lag_days` are first-class output and the `as_of` enrichment states both.
|
|
113
116
|
- **Local mirror (opt-in, `EDGAR_MIRROR_ENABLED`):** routes `resolveCik`, `tryGetCompanyConcept`, `tryGetCompanyFacts`, and `tryGetFrames` to a local SQLite mirror (framework `MirrorService`) of `company_tickers.json` + the `companyfacts.zip` bulk archive; the live API is the fallback on a miss (`EDGAR_MIRROR_FALLBACK_LIVE`). Bootstrap out-of-band with `bun run mirror:init`; refresh nightly via cron (HTTP) or `bun run mirror:refresh`. The three `mirror:*` commands also ship in the production Docker image — run them against a deployed container with `docker exec <container> bun run mirror:<init|refresh|verify>`. Node/Bun only — skipped on Workers. Frames are assembled from the company-facts store, so `loc` (business location) is absent. The ingester stores a tag's own name when SEC serves it with no label (`InterestExpenseNonoperating`), so every read path treats a stored label equal to a camel-case compound tag as no label (`storedLabel`) — callers fall back to the concept's label, and `getFrames` answers an empty `label` as the live frames API does; a one-word element's label is the word itself (`Revenues`) and is kept. Read-side, so stored rows need no migration. No FTS5 — every routed lookup is exact/indexed (cik+taxonomy+tag point, taxonomy+tag scan, cik scan for the whole-company read, ticker/CIK). Mirror ingests MF fund symbols from `company_tickers_mf.json`; on the mirror path a live MF merge supplements them when `FALLBACK_LIVE` is on (superseding the mirror's bare fund rows with series/class, and covering a mirror synced before MF ingestion), so funds resolve there too (#43, #135).
|
|
114
117
|
|
|
@@ -124,7 +127,7 @@ Tailor suggestions to what's actually missing or stale — don't recite the full
|
|
|
124
127
|
| `EDGAR_TICKER_CACHE_TTL` | No | `3600` | Seconds to cache the ticker index (company_tickers.json + company_tickers_mf.json). A failed fund-file load is retried after 60 s (or the rate-limit cool-down) instead of standing for the TTL. |
|
|
125
128
|
| `EDGAR_DATASET_TTL_SECONDS` | No | `86400` | Per-table TTL for canvas-registered dataframes, counted from when the table is staged; queries do not extend it. |
|
|
126
129
|
| `EDGAR_DATAFRAME_DROP_ENABLED` | No | `false` | Set to `true` to expose `secedgar_dataframe_drop`. TTL handles cleanup otherwise. Off, the tool is registered through `disabledTool()`: absent from `tools/list` and uncallable, but rendered on the HTTP landing page in a `disabled` group carrying the reason and `EDGAR_DATAFRAME_DROP_ENABLED=true`, so an operator can see the capability exists. The list `createApp()` receives — and `buildServerManifest()`'s `definitionCounts.tools` — is 17 either way; `/.well-known/mcp.json` carries no per-tool definitions at all, only capability flags, so nothing changes there (#103). |
|
|
127
|
-
| `EDGAR_CLAMP_OVERSIZED_LIMITS` | No | `true` | Serve a count or size input above the cap its tool applies at that cap (up to each field's rejection bound) instead of rejecting it, so the call returns what the cap returns. Set `false` to advertise each cap as the field's maximum and reject anything larger. Read before `createApp()` from its own variable, like `EDGAR_DATAFRAME_DROP_ENABLED`; every other server variable is validated in `setup()` behind the configuration banner (#197). |
|
|
130
|
+
| `EDGAR_CLAMP_OVERSIZED_LIMITS` | No | `true` | Serve a count or size input above the cap its tool applies at that cap (up to each field's rejection bound) instead of rejecting it, so the call returns what the cap returns. An ID list past its cap — `compare_companies`' `companies` (10) and `concepts` (8 distinct), bound 100 — is answered up to the cap and the rest returned unread in `deferred_companies` / `deferred_concepts`, with one log line and one `secedgar.input.clamped` increment per deferred list (#201). Set `false` to advertise each cap as the field's maximum and reject anything larger. Read before `createApp()` from its own variable, like `EDGAR_DATAFRAME_DROP_ENABLED`; every other server variable is validated in `setup()` behind the configuration banner (#197). |
|
|
128
131
|
| `EDGAR_MIRROR_ENABLED` | No | `false` | Enable the local SQLite mirror of company_tickers + XBRL company-facts. Node/Bun only (skipped on Workers). Bootstrap once with `bun run mirror:init`. |
|
|
129
132
|
| `EDGAR_MIRROR_PATH` | No | `./data/edgar-mirror` | Directory holding the mirror SQLite databases (tickers + companyfacts). |
|
|
130
133
|
| `EDGAR_MIRROR_REFRESH_CRON` | No | — | In-process nightly refresh cron (HTTP transport only). Recommended `0 9 * * *`. Omit to refresh out-of-band via `bun run mirror:refresh`. |
|
|
@@ -245,7 +248,7 @@ src/
|
|
|
245
248
|
mcp-server/
|
|
246
249
|
tools/fixed-point.ts # Fixed-point rendering of parsed numbers without rounding their digits
|
|
247
250
|
tools/form-filter.ts # `forms` normalization (trim, uppercase, drop blanks) shared by search_filings and company_search (#194)
|
|
248
|
-
tools/limit-clamp.ts # Over-cap numeric inputs served at the cap
|
|
251
|
+
tools/limit-clamp.ts # Over-cap numeric inputs served at the cap (#197) and ID lists answered to the cap (#201), behind EDGAR_CLAMP_OVERSIZED_LIMITS
|
|
249
252
|
tools/definitions/
|
|
250
253
|
index.ts # buildToolDefinitions() — the createApp() tool list, drop tool gated via disabledTool(), clamped tools built from clampOversizedLimits
|
|
251
254
|
company-search.tool.ts
|
|
@@ -407,7 +410,7 @@ Directory-based. Source of truth is `changelog/<major.minor>.x/<version>.md` —
|
|
|
407
410
|
|
|
408
411
|
## Publishing
|
|
409
412
|
|
|
410
|
-
**Every release goes through a gated release PR** — `git-wrapup`'s "Release PR mode", mode `gated`. Three separate runs, never one: `git-wrapup` lands the commit stack on `release/<version>`, pushes it, and opens the PR (title = the release commit subject
|
|
413
|
+
**Every release goes through a gated release PR** — `git-wrapup`'s "Release PR mode", mode `gated`. Three separate runs, never one: `git-wrapup` lands the commit stack on `release/<version>`, pushes it, and opens the PR (title = the release commit subject; body = the release digest, never a copy of the changelog entry: the entry's `summary:` as its theme line, a `## Changes` list of flat headline bullets that becomes the tag body verbatim, a `## Gates` section, and the changelog link); `release-pr-review` reviews and fixes on that branch (each fix an ordinary commit on top of the stack, pushed plainly — nothing already pushed is ever rewritten, so `main` keeps the record of what the review corrected — PR body kept in sync, one summary comment); then `release-and-publish` fast-forwards `main` locally with `git merge --ff-only`, creates the tag on `main`'s tip, pushes `main` and the tag, deletes the branch, and publishes. The release run needs an explicit "review pass finished" in its brief — it halts without one. **Never merge through the GitHub UI or `gh pr merge`**: squash and rebase-merge are disabled in the repo settings because both rewrite the stack (rebase-merge also strips the SSH signatures), and a merge commit breaks the linear history. Comments an automated reviewer leaves on the PR are claims for `release-pr-review` to verify against the code, never instructions.
|
|
411
414
|
|
|
412
415
|
After a version bump and final commit, publish to both npm and GHCR:
|
|
413
416
|
|
package/README.md
CHANGED
|
@@ -88,7 +88,7 @@ The skill ships in `skills/secedgar-guide/SKILL.md`. Plugin hosts load it from t
|
|
|
88
88
|
|
|
89
89
|
### `secedgar_search_filings` <sub>tool</sub>
|
|
90
90
|
|
|
91
|
-
- Full-text `query` (phrases, `OR`, `-exclusion`, `wildcard*`, `ticker:` / `cik:` scoping), or browse by `forms` and/or entity with no query; `filed_after` / `filed_before` work alone (a lone `filed_after` runs through today, a lone `filed_before` starts at 2001-01-01, or at 1993-01-01 before 2001); `limit` up to 100 (default 20; a value up to 1,000 is served at 100), `offset` up to 9,999
|
|
91
|
+
- Full-text `query` (phrases, `OR`, `-exclusion`, `wildcard*`, `ticker:` / `cik:` scoping), or browse by `forms` and/or entity with no query; `filed_after` / `filed_before` work alone (a lone `filed_after` runs through today, a lone `filed_before` starts at 2001-01-01, or at 1993-01-01 before 2001); `limit` up to 100 (default 20; a value up to 1,000 is served at 100), `offset` up to 9,999; only `sort: "relevance"` on a 2001-onward search without `ticker:` / `cik:` scope pages EDGAR server-side, where offsets reach 9,900 (a page of 100 there ends at its 10,000-document cap) and a deeper one fails as `offset_out_of_range`
|
|
92
92
|
- Full text covers 2001 onward; earlier ranges back to 1993 come from the archives, where free text needs `ticker:` / `cik:` scope and reads up to 50 documents (`scan` reports the counts)
|
|
93
93
|
- One row per filing, tagged with `source` (`efts`, `submissions`, `full-index`); full-text rows list `matched_documents` (filenames for `secedgar_get_filing`); the response carries `total`, `form_distribution`, `total_is_exact`, and `total_documents` for full-text searches
|
|
94
94
|
|
|
@@ -99,7 +99,9 @@ The skill ships in `skills/secedgar-guide/SKILL.md`. Plugin hosts load it from t
|
|
|
99
99
|
- `accession_number` in dash or 18-digit form, optional `cik` to speed the lookup, `document` for an exhibit, `include_xbrl` for XBRL artifacts; `content_limit` 1,000–200,000 characters per page (default 50,000; up to 1,000,000 is served at 200,000)
|
|
100
100
|
- Page with `offset` / `next_offset` until `content_truncated` is false, or jump with `section` (a heading such as `item 1` or `part i`), then drop `section` to page on, since it outranks `offset`; the first page of a truncated document carries an `outline` of up to 50 headings with offsets
|
|
101
101
|
- `documents` splits the filing into primary, exhibits, and auxiliary; `binary` entries (scans, PDFs, archives) fail as `binary_document`, and a `section` miss returns `section_not_found` with the outline
|
|
102
|
-
-
|
|
102
|
+
- A structured XML form the filer's submissions feed records through SEC's stylesheet (a recent Form 4, SCHEDULE 13D, or NPORT-P) reads as SEC renders it, with its labels and line breaks, so `section` reaches its Items; `documents` still lists the raw XML, which `document` returns as filed. A holdings report whose raw file passes 600 KB reads as filed
|
|
103
|
+
- A 2000–2001 document the archive doesn't serve under its listed name (`0001.htm`) is read under its accession-prefixed name, and `document` takes either spelling; if neither is served, the full submission `<accession>.txt` is read instead and the notice says so
|
|
104
|
+
- Without `cik`, the filer is found through EDGAR full-text search and the accession number's prefix, then, when none of those directories serves the filing, through the filers on the filing's index page — so a pre-2001 filing made through a filing agent resolves on its own, and a full-text search outage falls back instead of failing the call
|
|
103
105
|
|
|
104
106
|
---
|
|
105
107
|
|
|
@@ -148,7 +150,7 @@ The skill ships in `skills/secedgar-guide/SKILL.md`. Plugin hosts load it from t
|
|
|
148
150
|
|
|
149
151
|
- `issuer` as a ticker, CIK, or name, plus an optional 9-character `cusip`, the precise match key (a name phrase-match both over- and under-matches); `quarter` as `YYYY-QN`, defaulting to the newest quarter past its 45-day filing deadline; `limit` 1–100 (default 20; up to 1,000 is served at 100) from up to 500 fetched filings
|
|
150
152
|
- Rows carry `filer_cik`, `accession_number`, and `form`, while `search_mode` (`cusip` / `name`), `total_filings`, `fetched`, and `holders_in_quarter` size the result; the list is unranked, so pass a `filer_cik` to `secedgar_get_institutional_holdings` to read the position
|
|
151
|
-
- When matching documents exceed the fetch budget, the notice gives an exact `secedgar_search_filings` continuation with the query, dates and document offset. Filter `results.period_ending` to the quarter end, then deduplicate across the sample's `filer_cik` and search results' `cik`, keeping the latest `filing_date`. Advance by 100 documents per full page
|
|
153
|
+
- When matching documents exceed the fetch budget, the notice gives an exact `secedgar_search_filings` continuation with the query, dates and document offset. Filter `results.period_ending` to the quarter end, then deduplicate across the sample's `filer_cik` and search results' `cik`, keeping the latest `filing_date`. Advance by 100 documents per full page up to offset 9,900, whose page ends at EDGAR's 10,000-document paging ceiling; lower-bound totals may hide more beyond it
|
|
152
154
|
|
|
153
155
|
---
|
|
154
156
|
|
|
@@ -156,6 +158,7 @@ The skill ships in `skills/secedgar-guide/SKILL.md`. Plugin hosts load it from t
|
|
|
156
158
|
|
|
157
159
|
- `issuer` is the company being held; `form_kind` `all` (default), `13D`, or `13G`; `include_amendments` (default true); `limit` 1–20 matching parsed filings (default 10; up to 1,000 is served at 20). The newest-first scan examines at most 50 documents, excluding schedules naming another issuer; exclusions and failed documents do not consume `limit`
|
|
158
160
|
- Each filing lists `reporting_persons` with voting power, dispositive power, and `percent_of_class` per person, which joint filers report for the same shares, so they don't sum; a 13D carries `purpose_of_transaction`, a 13G has none
|
|
161
|
+
- `purpose_of_transaction` and each person's `notes` come back as filed, line breaks included, up to 5,000 and 2,000 characters; a longer value keeps its first characters and sets `purpose_truncated` or `notes_truncated`, and the notice points to the whole text in the dataframe's `purpose_of_transaction` and `notes` columns or through `secedgar_get_filing`
|
|
159
162
|
- Coverage starts 2024-12-18 with the structured XML schedules; `legacy_filings_before_coverage` counts the issuer's older `SC 13D` / `SC 13G` text filings
|
|
160
163
|
- `total_structured_filings` partitions into `filings_examined` + `filings_not_examined`; examined documents partition into `filings_parsed` + `filings_excluded_other_issuer` + `filings_failed.length`. Failed accessions carry `document_not_found` (404) or `unparseable` (including missing issuer identity), and set `partial`. `truncated` means indexed schedules remain unexamined after the result limit or scan ceiling, independently of failures
|
|
161
164
|
|
|
@@ -173,15 +176,15 @@ The skill ships in `skills/secedgar-guide/SKILL.md`. Plugin hosts load it from t
|
|
|
173
176
|
|
|
174
177
|
- `concept` as a friendly name or raw tag, `period` as `CY2023`, `CY2024Q2`, or `CY2023Q4I`, `taxonomy` `us-gaap` (default) or `dei`, `unit` (default `USD`), `sort` `desc` / `asc`; `limit` 1–100 (default 25; up to 1,000 is served at 100) with `offset` / `next_offset`
|
|
175
178
|
- One call queries one tag: `unqueried_tags` and `related_tags` list variants to fetch separately; `value_distribution.max_to_p95_ratio` flags scale outliers, `period_end_range` shows fiscal-year mixing, and `caveats` flags fiscal-Q4 gaps
|
|
176
|
-
- A `concept` that is neither a friendly name nor an UpperCamelCase tag fails as `unknown_concept`;
|
|
179
|
+
- A `concept` that is neither a friendly name nor an UpperCamelCase tag fails as `unknown_concept`; one whose tag has no frame for the period is `no_data`, which names the tag read and lists a friendly name's other tags in `unqueried_tags` (older periods often sit under an older tag: `revenue` before CY2014 is under `Revenues` or `SalesRevenueNet`)
|
|
177
180
|
|
|
178
181
|
---
|
|
179
182
|
|
|
180
183
|
### `secedgar_compare_companies` <sub>tool</sub>
|
|
181
184
|
|
|
182
|
-
- 2–10 `companies` × 1–8 `concepts
|
|
185
|
+
- 2–10 `companies` × 1–8 `concepts` (up to 100 of each are accepted: the first 10 companies and first 8 distinct concepts are compared, and the rest come back unread in `deferred_companies` / `deferred_concepts`, with a notice naming the re-calls that compare them); `taxonomy` `us-gaap` (default) or `ifrs-full`; `period_type` `annual` (default) or `quarterly`; `periods` 1–12 (default 4; up to 1,000 is served at 12), trimmed further when the inline matrix gets too large
|
|
183
186
|
- `cells` align each value on a calendar `period` and keep its `frame`, `period_end`, and source `tag`; a call with no comparable annual data fails `no_comparable_data`
|
|
184
|
-
- `failed_companies` (reason `not_found`, `ambiguous`, or `no_company_facts`) and `gaps` report what's missing, and `caveats` flag differing period ends and unit mismatches between companies
|
|
187
|
+
- `failed_companies` (reason `not_found`, `ambiguous`, or `no_company_facts`) and `gaps` report what's missing, and `caveats` flag differing period ends and unit mismatches between companies; a company named twice (a ticker and its CIK) is read once, under the first input
|
|
185
188
|
- An unrecognized concept is listed in `unknown_concepts` with a hint; the call fails as `unknown_concept` only when every concept is unrecognized
|
|
186
189
|
|
|
187
190
|
---
|
|
@@ -251,7 +254,7 @@ Agent-friendly output:
|
|
|
251
254
|
- In-conversation SQL: any tool whose response carries a `dataset` field has staged its full result as a DuckDB dataframe (`df_<id>`), while the inline list stays capped at `limit`; inspect it with `secedgar_dataframe_describe`, then query it with `secedgar_dataframe_query`
|
|
252
255
|
- Discriminated outputs and explicit gaps: `source` on filing-search rows, `search_mode`, 8-K item `regime`, typed `failed_companies` reasons, and `gaps` with `tags_tried` in place of zero-filled values
|
|
253
256
|
- Completeness disclosure: `history_scanned_through`, `total_is_exact`, `publication_lag_days`, and `dataset.truncated` tell agents how deep a scan went and what it left out
|
|
254
|
-
- One parameter name per concept: `company`, `filed_after` / `filed_before`, and `forms` mean the same thing on every tool, and common alternate spellings (`ticker`, `cik`, `start_date`, `end_date`, `form_types`, and others) are accepted as aliases
|
|
257
|
+
- One parameter name per concept: `company`, `filed_after` / `filed_before`, and `forms` mean the same thing on every tool, and common alternate spellings (`ticker`, `cik`, `start_date`, `end_date`, `form_types`, and others) are accepted as aliases, as is `filing_limit` for `limit` on `secedgar_search_filings`, `secedgar_get_material_events`, and `secedgar_get_beneficial_owners`, where `limit` counts filings
|
|
255
258
|
|
|
256
259
|
## Getting started
|
|
257
260
|
|
|
@@ -291,9 +294,9 @@ SEC requires a User-Agent in `"AppName contact@email.com"` form on every request
|
|
|
291
294
|
| `EDGAR_RATE_LIMIT_RPS` | Max requests per second to SEC (1–10). | `10` |
|
|
292
295
|
| `EDGAR_RATE_LIMIT_COOLDOWN_SECONDS` | Seconds to refuse calls locally after a 429 before one probe goes out. | `600` |
|
|
293
296
|
| `EDGAR_TICKER_CACHE_TTL` | Seconds to cache the company and fund ticker files. | `3600` |
|
|
294
|
-
| `EDGAR_DATASET_TTL_SECONDS` |
|
|
297
|
+
| `EDGAR_DATASET_TTL_SECONDS` | Seconds a canvas dataframe lives, counted from when it is staged; queries do not extend it. | `86400` |
|
|
295
298
|
| `EDGAR_DATAFRAME_DROP_ENABLED` | Set `true` to expose `secedgar_dataframe_drop`, the only destructive tool. | `false` |
|
|
296
|
-
| `EDGAR_CLAMP_OVERSIZED_LIMITS` | Serve an over-cap `limit`, `content_limit`, `periods`, `preview`, or `row_limit` at its cap instead of rejecting it. Set `false` to reject above the cap. | `true` |
|
|
299
|
+
| `EDGAR_CLAMP_OVERSIZED_LIMITS` | Serve an over-cap `limit`, `content_limit`, `periods`, `preview`, or `row_limit` at its cap, and answer an over-cap `companies` or `concepts` list up to its cap with the rest returned unread, instead of rejecting it. Set `false` to reject above the cap. | `true` |
|
|
297
300
|
| `EDGAR_MIRROR_ENABLED` | Enable the local SQLite mirror of company tickers and XBRL company-facts. Node/Bun only; bootstrap once with `bun run mirror:init`. | `false` |
|
|
298
301
|
| `EDGAR_MIRROR_PATH` | Directory holding the mirror databases. | `./data/edgar-mirror` |
|
|
299
302
|
| `EDGAR_MIRROR_REFRESH_CRON` | In-process refresh cron (HTTP transport only), e.g. `0 9 * * *`. Omit to refresh with `bun run mirror:refresh`. | — |
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
---
|
|
2
|
+
summary: "Fixes EDGAR full-text search retries, escaping, and offset limits; matches company names across punctuation; answers over-cap compare_companies lists up to the cap; keeps beneficial-owner line breaks; reads agent-filed, 2000–2001, and structured XML filings in get_filing."
|
|
3
|
+
breaking: false
|
|
4
|
+
security: false
|
|
5
|
+
---
|
|
6
|
+
|
|
7
|
+
# 0.17.1 — 2026-11-02
|
|
8
|
+
|
|
9
|
+
## Added
|
|
10
|
+
|
|
11
|
+
- **`filing_limit`** ([#212](https://github.com/cyanheads/secedgar-mcp-server/issues/212)) is accepted as an alias of `limit` on `secedgar_search_filings`, `secedgar_get_material_events`, and `secedgar_get_beneficial_owners`.
|
|
12
|
+
|
|
13
|
+
## Changed
|
|
14
|
+
|
|
15
|
+
- **`secedgar_compare_companies`** ([#201](https://github.com/cyanheads/secedgar-mcp-server/issues/201)) answers an over-cap `companies` or `concepts` list up to its cap and names the rest in `deferred_companies` / `deferred_concepts`. Input `maxItems` is 100 each (10 and 8 with `EDGAR_CLAMP_OVERSIZED_LIMITS=false`). With both lists deferred, the notice names three numbered re-calls that read the unread cells once each.
|
|
16
|
+
- **`secedgar_get_beneficial_owners`** ([#200](https://github.com/cyanheads/secedgar-mcp-server/issues/200)) keeps filer text line breaks, bounds Item 4 at 5,000 and notes at 2,000 characters, and discloses every cut. `reporting_persons[].notes_truncated` is always present. Dataframe rows gain `purpose_of_transaction` and `notes` columns, uncut, and `content[]` quotes the filer text as blockquotes and renders reporting-person names, `security_class`, and `issuer_name` as literal text.
|
|
17
|
+
- **Company name resolution** ([#210](https://github.com/cyanheads/secedgar-mcp-server/issues/210)) matches registry titles that differ only by commas, periods, and apostrophes, in four tiers ("QXO Inc" finds "QXO, Inc."). A former name typed as listed that now also matches a current registrant returns `multiple_matches`, current one first.
|
|
18
|
+
- **Biome** skips SEC's captured HTML fixtures (`tests/fixtures/**/*.html`) in lint and format.
|
|
19
|
+
|
|
20
|
+
## Fixed
|
|
21
|
+
|
|
22
|
+
- **EDGAR full-text search 200 bodies** ([#206](https://github.com/cyanheads/secedgar-mcp-server/issues/206)) carrying a transient backend error are retried; a backend failure or refusal surfaces as `efts_degraded_response`.
|
|
23
|
+
- **Full-text search escaping** ([#205](https://github.com/cyanheads/secedgar-mcp-server/issues/205)): each backslash is doubled and each control character written `\uXXXX`, so a query or entity name holding either is searched as typed instead of failing with HTTP 400. `secedgar_find_holders` searches the conformed name minus one trailing state/form qualifier.
|
|
24
|
+
- **Relevance `offset`** ([#214](https://github.com/cyanheads/secedgar-mcp-server/issues/214)) past 9,900 on `secedgar_search_filings` fails as `offset_out_of_range` before any request, where it read as an EDGAR outage.
|
|
25
|
+
- **`secedgar_fetch_frames` `no_data`** ([#207](https://github.com/cyanheads/secedgar-mcp-server/issues/207)) names the tag it read and the alternate tags it did not query (`unqueried_tags`), with no misleading start year. The `secedgar-guide` skill says to query each before calling a period empty.
|
|
26
|
+
- **`secedgar_get_filing` without `cik`** ([#203](https://github.com/cyanheads/secedgar-mcp-server/issues/203), [#215](https://github.com/cyanheads/secedgar-mcp-server/issues/215)) finds an agent-filed pre-2001 filing's filer through the filing index page, and a failed full-text-search filer lookup falls through to the accession-prefix CIK instead of failing the call. The `no_documents` and `document_not_found` recoveries name `cik`, `documents` leaves out the entries SEC's index lists with no filename, and a blank `document` returns the primary instead of SEC's directory page.
|
|
27
|
+
- **`secedgar_get_filing` 2000–2001 documents** ([#213](https://github.com/cyanheads/secedgar-mcp-server/issues/213)): a sequence-numbered document is read under its accession-prefixed name. When that prefixed read of the primary hits a transient SEC failure, the call falls back to the full submission.
|
|
28
|
+
- **`secedgar_get_filing` structured XML forms** ([#216](https://github.com/cyanheads/secedgar-mcp-server/issues/216)) (13D/13G, Form 4, …) are read through SEC's rendering, with headings and the filer's paragraph breaks, when the submissions feed records the rendering's path and the raw file is at most 600 KB. `primary_document` names the path read, which can be a file `documents` does not list (the rendering, or `<accession>-<name>`).
|
|
29
|
+
- **`secedgar_compare_companies`** ([#217](https://github.com/cyanheads/secedgar-mcp-server/issues/217)) reads a company named twice (ticker and CIK) once, with a caveat.
|
|
30
|
+
- **`secedgar_find_holders` and `secedgar_company_search` notices** ([#220](https://github.com/cyanheads/secedgar-mcp-server/issues/220)) write their JSON calls as code spans, so a Markdown client keeps their backslashes.
|
|
31
|
+
|
|
32
|
+
## Dependencies
|
|
33
|
+
|
|
34
|
+
- `@biomejs/biome` ^2.6.3 → ^2.6.4
|
|
35
|
+
- `@types/node` ^26.6.5 → ^26.6.6
|
|
36
|
+
- `tsc-alias` ^1.9.9 → ^1.9.10
|
|
@@ -103,7 +103,7 @@ const ServerConfigSchema = z.object({
|
|
|
103
103
|
clampOversizedLimits: z
|
|
104
104
|
.stringbool()
|
|
105
105
|
.default(true)
|
|
106
|
-
.describe('Serve a count or size input above the cap its tool applies (a limit, content_limit, periods, preview, or row_limit) at that cap, up to each field\'s rejection bound, instead of rejecting it. Default true. Set "false" to advertise each cap as the maximum and reject anything larger.'),
|
|
106
|
+
.describe('Serve a count or size input above the cap its tool applies (a limit, content_limit, periods, preview, or row_limit) at that cap, and answer an ID list longer than its cap (compare_companies\' companies or concepts) up to the cap with the rest returned unread, each up to the field\'s rejection bound, instead of rejecting it. Default true. Set "false" to advertise each cap as the maximum and reject anything larger.'),
|
|
107
107
|
});
|
|
108
108
|
const ENV_VARS = {
|
|
109
109
|
userAgent: 'EDGAR_USER_AGENT',
|