@cyanheads/secedgar-mcp-server 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +22 -16
- package/CLAUDE.md +22 -16
- package/README.md +14 -10
- package/changelog/0.17.x/0.17.1.md +36 -0
- package/changelog/0.18.x/0.18.0.md +16 -0
- package/dist/config/server-config.js +1 -1
- package/dist/config/server-config.js.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/mcp-server/tools/ambiguous-matches.d.ts +20 -0
- package/dist/mcp-server/tools/ambiguous-matches.d.ts.map +1 -0
- package/dist/mcp-server/tools/ambiguous-matches.js +22 -0
- package/dist/mcp-server/tools/ambiguous-matches.js.map +1 -0
- package/dist/mcp-server/tools/definitions/company-search.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/company-search.tool.js +15 -8
- package/dist/mcp-server/tools/definitions/company-search.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/compare-companies.tool.d.ts +8 -4
- package/dist/mcp-server/tools/definitions/compare-companies.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/compare-companies.tool.js +182 -45
- package/dist/mcp-server/tools/definitions/compare-companies.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/dataframe-describe.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/dataframe-describe.tool.js +10 -2
- package/dist/mcp-server/tools/definitions/dataframe-describe.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/dataframe-drop.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/dataframe-drop.tool.js +6 -6
- package/dist/mcp-server/tools/definitions/dataframe-drop.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/dataframe-query.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/dataframe-query.tool.js +10 -10
- package/dist/mcp-server/tools/definitions/dataframe-query.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.d.ts +4 -4
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.js +36 -15
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/find-holders.tool.d.ts +14 -0
- package/dist/mcp-server/tools/definitions/find-holders.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/find-holders.tool.js +80 -16
- package/dist/mcp-server/tools/definitions/find-holders.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.d.ts +2 -0
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.js +147 -77
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-filing.tool.d.ts +4 -4
- package/dist/mcp-server/tools/definitions/get-filing.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-filing.tool.js +321 -90
- package/dist/mcp-server/tools/definitions/get-filing.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-financials.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-financials.tool.js +19 -21
- package/dist/mcp-server/tools/definitions/get-financials.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-fund-holdings.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-fund-holdings.tool.js +11 -19
- package/dist/mcp-server/tools/definitions/get-fund-holdings.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.d.ts +10 -0
- package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.js +23 -9
- package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.d.ts +14 -0
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.js +16 -14
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-material-events.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-material-events.tool.js +9 -5
- package/dist/mcp-server/tools/definitions/get-material-events.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-snapshot.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-snapshot.tool.js +11 -8
- package/dist/mcp-server/tools/definitions/get-snapshot.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/index.d.ts +635 -594
- package/dist/mcp-server/tools/definitions/index.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/search-concepts.tool.js +2 -2
- package/dist/mcp-server/tools/definitions/search-concepts.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/search-filings.tool.d.ts +45 -0
- package/dist/mcp-server/tools/definitions/search-filings.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/search-filings.tool.js +64 -12
- package/dist/mcp-server/tools/definitions/search-filings.tool.js.map +1 -1
- package/dist/mcp-server/tools/escape-inline.d.ts +35 -0
- package/dist/mcp-server/tools/escape-inline.d.ts.map +1 -0
- package/dist/mcp-server/tools/escape-inline.js +51 -0
- package/dist/mcp-server/tools/escape-inline.js.map +1 -0
- package/dist/mcp-server/tools/format-usd.d.ts +22 -0
- package/dist/mcp-server/tools/format-usd.d.ts.map +1 -0
- package/dist/mcp-server/tools/format-usd.js +53 -0
- package/dist/mcp-server/tools/format-usd.js.map +1 -0
- package/dist/mcp-server/tools/limit-clamp.d.ts +50 -18
- package/dist/mcp-server/tools/limit-clamp.d.ts.map +1 -1
- package/dist/mcp-server/tools/limit-clamp.js +51 -15
- package/dist/mcp-server/tools/limit-clamp.js.map +1 -1
- package/dist/services/edgar/edgar-api-service.d.ts +96 -11
- package/dist/services/edgar/edgar-api-service.d.ts.map +1 -1
- package/dist/services/edgar/edgar-api-service.js +333 -50
- package/dist/services/edgar/edgar-api-service.js.map +1 -1
- package/dist/services/edgar/filing-to-text.d.ts +4 -3
- package/dist/services/edgar/filing-to-text.d.ts.map +1 -1
- package/dist/services/edgar/filing-to-text.js +12 -0
- package/dist/services/edgar/filing-to-text.js.map +1 -1
- package/package.json +4 -4
- package/server.json +3 -3
- package/skills/secedgar-guide/SKILL.md +1 -0
|
@@ -161,7 +161,57 @@ function stripCompanySuffix(name) {
|
|
|
161
161
|
return;
|
|
162
162
|
return name.slice(0, cut).replace(/,$/, '').trimEnd();
|
|
163
163
|
}
|
|
164
|
-
|
|
164
|
+
/**
|
|
165
|
+
* Delete every comma, period, and apostrophe (`'` and `’`) from a company name, then
|
|
166
|
+
* collapse each whitespace run to one space and trim the ends — the comparison form the
|
|
167
|
+
* name passes' folded tiers read, so `QXO Inc` meets `QXO, Inc.`, `U.S. Bancorp` meets
|
|
168
|
+
* `US BANCORP`, `Macys` meets `Macy's, Inc.` (#210), and `Forgent Networks Inc` meets the
|
|
169
|
+
* former name `forgent networks inc` (#219). A period is deleted, never turned into a
|
|
170
|
+
* space, so a dotted initialism meets its undotted spelling. Nothing else changes — `&`,
|
|
171
|
+
* `-`, `/`, and `\` stay, and so do spaced initials (`joinSpacedInitials` is a separate
|
|
172
|
+
* step) — and the result is for comparison only: a returned name is never folded.
|
|
173
|
+
*/
|
|
174
|
+
export function foldNamePunctuation(name) {
|
|
175
|
+
return name
|
|
176
|
+
.replace(/[,.'’]/g, '')
|
|
177
|
+
.replace(/\s{2,}|[^\S ]/g, ' ')
|
|
178
|
+
.trim();
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Join each run of two or more one-letter words into one word (`j m smucker co` →
|
|
182
|
+
* `jm smucker co`), so `JM Smucker` meets the conformed name `J M SMUCKER Co` and
|
|
183
|
+
* `WP Carey` meets `W. P. Carey Inc.` once folded (#219). Reads a folded name: words are
|
|
184
|
+
* split on single spaces, and a word is one letter in any script — a digit or a symbol
|
|
185
|
+
* ends the run. A space joins when a one-letter word sits on each side of it; the
|
|
186
|
+
* lookarounds are fixed-width, so the scan stays linear in the name's length.
|
|
187
|
+
*/
|
|
188
|
+
export function joinSpacedInitials(name) {
|
|
189
|
+
return name.replace(/(?<=(?:^| )\p{L}) (?=\p{L}(?: |$))/gu, '');
|
|
190
|
+
}
|
|
191
|
+
/** A letter or digit: the characters a name's word boundary falls between (#218). */
|
|
192
|
+
const NAME_WORD_CHAR = /[\p{L}\p{N}]/u;
|
|
193
|
+
/**
|
|
194
|
+
* Whether index `at` of `text` is a word boundary — a letter or digit on exactly one side,
|
|
195
|
+
* regex `\b` read over every script (#218). Either end of the string counts as a side
|
|
196
|
+
* with neither.
|
|
197
|
+
*/
|
|
198
|
+
function isWordBoundary(text, at) {
|
|
199
|
+
return NAME_WORD_CHAR.test(text.charAt(at - 1)) !== NAME_WORD_CHAR.test(text.charAt(at));
|
|
200
|
+
}
|
|
201
|
+
/**
|
|
202
|
+
* Whether `name` contains `query` starting at a word boundary, so a substring hit never
|
|
203
|
+
* begins inside a word: `embassy bancorp, inc.` does not contain `sy bancorp, inc.` (#218).
|
|
204
|
+
* The query may end mid-word (`berkshire hath`). Every occurrence is tried, since the first
|
|
205
|
+
* can sit mid-word and a later one at a word start.
|
|
206
|
+
*/
|
|
207
|
+
export function containsFromWordStart(name, query) {
|
|
208
|
+
for (let at = name.indexOf(query); at >= 0; at = name.indexOf(query, at + 1)) {
|
|
209
|
+
if (isWordBoundary(name, at))
|
|
210
|
+
return true;
|
|
211
|
+
}
|
|
212
|
+
return false;
|
|
213
|
+
}
|
|
214
|
+
const NAME_TIERS = [0, 1, 2, 3, 4];
|
|
165
215
|
// ---------------------------------------------------------------------------
|
|
166
216
|
// Near-match company suggestions
|
|
167
217
|
// ---------------------------------------------------------------------------
|
|
@@ -235,9 +285,12 @@ class EdgarApiService {
|
|
|
235
285
|
dispose() {
|
|
236
286
|
this.pacer.dispose();
|
|
237
287
|
}
|
|
238
|
-
/**
|
|
239
|
-
|
|
240
|
-
|
|
288
|
+
/**
|
|
289
|
+
* Fetch and parse JSON, throwing on non-OK responses and on a body that is not JSON.
|
|
290
|
+
* `backendFailure` names the 200 bodies that fail an attempt (#206).
|
|
291
|
+
*/
|
|
292
|
+
async fetchJson(url, scope, options) {
|
|
293
|
+
const text = await this.rawFetch(url, true, scope, options);
|
|
241
294
|
if (text === null) {
|
|
242
295
|
throw notFound(`SEC EDGAR API returned 404 for ${url}`, { url, status: 404 });
|
|
243
296
|
}
|
|
@@ -265,8 +318,12 @@ class EdgarApiService {
|
|
|
265
318
|
* Resolve a query (ticker, name, or CIK) to company match(es).
|
|
266
319
|
* - Numeric input → direct CIK lookup
|
|
267
320
|
* - 1-5 uppercase letters → ticker lookup (O(1))
|
|
268
|
-
* -
|
|
269
|
-
*
|
|
321
|
+
* - No letter or digit (`.`, `'`) → no match
|
|
322
|
+
* - Otherwise → name search (exact as written; exact once commas, periods, and
|
|
323
|
+
* apostrophes are folded out and whitespace runs collapse; exact once spaced initials
|
|
324
|
+
* join; prefix of the joined form ending at a word boundary; substring as written
|
|
325
|
+
* starting at one — resolved within the best tier a current name reaches), then the
|
|
326
|
+
* catch-all ticker lookup
|
|
270
327
|
* Returns a single match, an array of multiple matches, or an empty array (no match).
|
|
271
328
|
* On no-result name search, the returned empty array carries `suggestions` on
|
|
272
329
|
* the thrown error at the handler layer — call `suggestCompanies` there.
|
|
@@ -291,19 +348,50 @@ class EdgarApiService {
|
|
|
291
348
|
if (match)
|
|
292
349
|
return match;
|
|
293
350
|
}
|
|
294
|
-
//
|
|
295
|
-
//
|
|
296
|
-
|
|
297
|
-
|
|
351
|
+
// A query with no letter or digit names no company: `.` would fold to an empty string
|
|
352
|
+
// that starts every name (#219).
|
|
353
|
+
if (!NAME_WORD_CHAR.test(trimmed))
|
|
354
|
+
return [];
|
|
355
|
+
// Name search, tiered exact as written → exact once folded → exact once initials join →
|
|
356
|
+
// prefix → substring. The three exact tiers also accept a suffix-normalized equality
|
|
357
|
+
// (#107) and the registry name minus its terminal suffix, so `Apple` names `Apple Inc.`
|
|
358
|
+
// exactly (#155); the query's own suffix is never dropped, so `Toro Co` never meets
|
|
359
|
+
// `TORO CORP.`. The folded tier compares with commas, periods, and apostrophes deleted
|
|
360
|
+
// and whitespace runs collapsed on both sides (#210, #219); the joined tier and the
|
|
361
|
+
// prefix tier read that form with each run of one-letter words joined (`j m smucker co`
|
|
362
|
+
// → `jm smucker co`, #219). Each tier ranks above the looser one after it, so
|
|
363
|
+
// `McGraw Hill, Inc.` never ties a former name equal to it only once folded, and
|
|
364
|
+
// `US Gold` (U.S. GOLD CORP.) never ties McEwen's former `u s gold corp`, equal only once
|
|
365
|
+
// joined. No hit starts or ends inside a word of the name (#218): a prefix ends at a
|
|
366
|
+
// word boundary — `helix acquisition corp ii` is not a prefix of
|
|
367
|
+
// `helix acquisition corp iii` — and a substring, matched as written, starts at one, so
|
|
368
|
+
// `SY Bancorp` never names Embassy Bancorp; a query ending mid-word (`Berkshire Hath`)
|
|
369
|
+
// still matches there when the name as written contains it.
|
|
298
370
|
const lower = trimmed.toLowerCase();
|
|
299
371
|
const lowerNormalized = normalizeCompanySuffix(lower);
|
|
372
|
+
const folded = foldNamePunctuation(lower);
|
|
373
|
+
const foldedNormalized = normalizeCompanySuffix(folded);
|
|
374
|
+
const joined = joinSpacedInitials(folded);
|
|
375
|
+
const joinedNormalized = normalizeCompanySuffix(joined);
|
|
300
376
|
// A suffix-dropped name is a prefix of the full one, so the strip runs on prefix hits only.
|
|
377
|
+
const isExact = (name, form, formNormalized) => name === form ||
|
|
378
|
+
normalizeCompanySuffix(name) === formNormalized ||
|
|
379
|
+
(name.startsWith(form) && stripCompanySuffix(name) === form);
|
|
301
380
|
const tierOf = (name) => {
|
|
302
|
-
if (name
|
|
381
|
+
if (isExact(name, lower, lowerNormalized))
|
|
303
382
|
return 0;
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
383
|
+
const foldedName = foldNamePunctuation(name);
|
|
384
|
+
if (isExact(foldedName, folded, foldedNormalized))
|
|
385
|
+
return 1;
|
|
386
|
+
const joinedName = joinSpacedInitials(foldedName);
|
|
387
|
+
// With no initials joined on either side, this repeats the folded tier's comparison.
|
|
388
|
+
if ((joinedName !== foldedName || joined !== folded) &&
|
|
389
|
+
isExact(joinedName, joined, joinedNormalized)) {
|
|
390
|
+
return 2;
|
|
391
|
+
}
|
|
392
|
+
if (joinedName.startsWith(joined) && isWordBoundary(joinedName, joined.length))
|
|
393
|
+
return 3;
|
|
394
|
+
return containsFromWordStart(name, lower) ? 4 : undefined;
|
|
307
395
|
};
|
|
308
396
|
// Current names are registry rows, which carry a ticker; former names (the committed
|
|
309
397
|
// asset) carry none. Resolution runs within the best tier a current name reaches —
|
|
@@ -311,8 +399,8 @@ class EdgarApiService {
|
|
|
311
399
|
// name joins at that tier or better but never outranks a current name, so a company
|
|
312
400
|
// is never silently swapped for a former holder of its name; former names decide
|
|
313
401
|
// alone only when no current name matches (#155).
|
|
314
|
-
const current = { 0: [], 1: [], 2: [] };
|
|
315
|
-
const former = { 0: [], 1: [], 2: [] };
|
|
402
|
+
const current = { 0: [], 1: [], 2: [], 3: [], 4: [] };
|
|
403
|
+
const former = { 0: [], 1: [], 2: [], 3: [], 4: [] };
|
|
316
404
|
for (const entry of cache.allEntries) {
|
|
317
405
|
if (!entry.name)
|
|
318
406
|
continue;
|
|
@@ -358,12 +446,19 @@ class EdgarApiService {
|
|
|
358
446
|
* a legal name under different CIKs (e.g. "VANGUARD GROUP INC" = a transfer agent and
|
|
359
447
|
* the 13F filer), so callers must disambiguate rather than auto-pick the top hit (#73).
|
|
360
448
|
* Distinct from `searchFilings`: same URL path, different query param (`keysTyped`)
|
|
361
|
-
* and response shape (`_source.entity`, not the filing-document `_source`).
|
|
449
|
+
* and response shape (`_source.entity`, not the filing-document `_source`). A
|
|
450
|
+
* failure of the search backend throws `efts_degraded_response` like a search's,
|
|
451
|
+
* never reads as no match (#206). The name is escaped like a search's `q` (#205).
|
|
362
452
|
*/
|
|
363
453
|
async resolveEntityByName(name, scope) {
|
|
364
454
|
const url = new URL('https://efts.sec.gov/LATEST/search-index');
|
|
365
|
-
url.searchParams.set('keysTyped', name);
|
|
366
|
-
const response = await this.fetchJson(url.toString(), scope
|
|
455
|
+
url.searchParams.set('keysTyped', escapeEftsText(name));
|
|
456
|
+
const response = await this.fetchJson(url.toString(), scope, {
|
|
457
|
+
backendFailure: transientEftsError,
|
|
458
|
+
});
|
|
459
|
+
const backendError = eftsBackendError(response);
|
|
460
|
+
if (backendError)
|
|
461
|
+
throw eftsBackendFailure(url.toString(), backendError);
|
|
367
462
|
const hits = response?.hits?.hits;
|
|
368
463
|
if (!Array.isArray(hits))
|
|
369
464
|
return [];
|
|
@@ -501,9 +596,11 @@ class EdgarApiService {
|
|
|
501
596
|
async searchFilings(params, scope) {
|
|
502
597
|
const url = new URL('https://efts.sec.gov/LATEST/search-index');
|
|
503
598
|
// `q` is optional — EFTS honors `ciks` for pure entity scope with no
|
|
504
|
-
// full-text query, so a bare cik:/ticker: search sends no `q`.
|
|
599
|
+
// full-text query, so a bare cik:/ticker: search sends no `q`. Escaped here, at
|
|
600
|
+
// the one place a query is sent, so a caller re-sending a phrase another tool
|
|
601
|
+
// searched (find_holders' `search_key`) runs the same search (#205).
|
|
505
602
|
if (params.query)
|
|
506
|
-
url.searchParams.set('q', params.query);
|
|
603
|
+
url.searchParams.set('q', escapeEftsText(params.query));
|
|
507
604
|
if (params.forms?.length)
|
|
508
605
|
url.searchParams.set('forms', params.forms.join(','));
|
|
509
606
|
// Server-side entity scope by CIK, independent of the document's name text —
|
|
@@ -517,7 +614,14 @@ class EdgarApiService {
|
|
|
517
614
|
}
|
|
518
615
|
url.searchParams.set('from', String(params.from ?? 0));
|
|
519
616
|
url.searchParams.set('size', String(params.size ?? 20));
|
|
520
|
-
|
|
617
|
+
// A transport failure of the search backend retries on the ladder; any other
|
|
618
|
+
// backend error fails on its one request (#206).
|
|
619
|
+
const response = await this.fetchJson(url.toString(), scope, {
|
|
620
|
+
backendFailure: transientEftsError,
|
|
621
|
+
});
|
|
622
|
+
const backendError = eftsBackendError(response);
|
|
623
|
+
if (backendError)
|
|
624
|
+
throw eftsBackendFailure(url.toString(), backendError);
|
|
521
625
|
// Shape guard: EFTS can return a 2xx whose body omits `hits.total` (degraded
|
|
522
626
|
// payload, or a rejected request echoed as `{ error: ... }` with 200). A genuine
|
|
523
627
|
// zero-hit response is still well-formed (`hits.total.value: 0`, empty
|
|
@@ -526,7 +630,7 @@ class EdgarApiService {
|
|
|
526
630
|
if (!Array.isArray(response?.hits?.hits) || typeof response.hits.total?.value !== 'number') {
|
|
527
631
|
const upstreamError = response?.error;
|
|
528
632
|
throw serviceUnavailable('SEC EDGAR full-text search returned an unexpected response without hits — the service may be degraded.', {
|
|
529
|
-
reason:
|
|
633
|
+
reason: EFTS_DEGRADED_REASON,
|
|
530
634
|
...(typeof upstreamError === 'string' && { upstreamError }),
|
|
531
635
|
recovery: {
|
|
532
636
|
hint: 'Retry the search in a few minutes — EDGAR full-text search returned an incomplete response.',
|
|
@@ -553,9 +657,18 @@ class EdgarApiService {
|
|
|
553
657
|
}
|
|
554
658
|
return [...ciks];
|
|
555
659
|
}
|
|
556
|
-
/**
|
|
557
|
-
|
|
558
|
-
|
|
660
|
+
/**
|
|
661
|
+
* Fetch a filing's document index. Returns `null` if the filing does not exist.
|
|
662
|
+
* Entries listed with no filename are dropped: SEC lists each document of a
|
|
663
|
+
* pre-2001 SGML submission that way, with only its size, and such a document
|
|
664
|
+
* exists only inside the full submission `<accession>.txt` (#203).
|
|
665
|
+
*/
|
|
666
|
+
async tryGetFilingIndex(cik, accessionNumber, scope) {
|
|
667
|
+
const index = await this.tryFetchJson(filingArchiveUrl(cik, accessionNumber, 'index.json'), scope);
|
|
668
|
+
if (index === null)
|
|
669
|
+
return null;
|
|
670
|
+
const named = index.directory.item.filter((item) => item.name.trim() !== '');
|
|
671
|
+
return { ...index, directory: { ...index.directory, item: named } };
|
|
559
672
|
}
|
|
560
673
|
/**
|
|
561
674
|
* Fetch the SEC submission header page (`<accession>-index-headers.html`) and parse
|
|
@@ -567,6 +680,19 @@ class EdgarApiService {
|
|
|
567
680
|
const text = await this.tryFetchText(filingArchiveUrl(cik, accessionNumber, `${accessionNumber}-index-headers.html`), scope);
|
|
568
681
|
return text ? parseFilingHeaders(text) : null;
|
|
569
682
|
}
|
|
683
|
+
/**
|
|
684
|
+
* Fetch a filing's index page (`<accession>-index.html`) and return the CIKs of the
|
|
685
|
+
* parties it names — filer, subject company, filed-by, reporting owner, issuer — in
|
|
686
|
+
* page order, without repeats (see {@link filingPartyCiks}). Returns `null` if the
|
|
687
|
+
* page is absent. SEC serves the page under the accession number's own prefix CIK
|
|
688
|
+
* for any filing that exists, including a filing agent's prefix whose directory has
|
|
689
|
+
* no `index.json`, so it names the filers of an accession full-text search does not
|
|
690
|
+
* cover (before 2001) (#203).
|
|
691
|
+
*/
|
|
692
|
+
async tryGetFilingPartyCiks(cik, accessionNumber, scope) {
|
|
693
|
+
const html = await this.tryFetchText(filingArchiveUrl(cik, accessionNumber, `${accessionNumber}-index.html`), scope);
|
|
694
|
+
return html === null ? null : filingPartyCiks(html);
|
|
695
|
+
}
|
|
570
696
|
/**
|
|
571
697
|
* Fetch the bare SGML submission header (`<accession>.hdr.sgml`, under 1 KB) and read
|
|
572
698
|
* its form, filing date, and period. Returns `null` if the file is absent. The
|
|
@@ -605,7 +731,9 @@ class EdgarApiService {
|
|
|
605
731
|
* retried; stopping at the cutoff is not a failure (#181).
|
|
606
732
|
*/
|
|
607
733
|
tryGetFilingDocumentHead(cik, accessionNumber, document, options, scope) {
|
|
608
|
-
return this.rawFetch(filingArchiveUrl(cik, accessionNumber, document), false, scope,
|
|
734
|
+
return this.rawFetch(filingArchiveUrl(cik, accessionNumber, document), false, scope, {
|
|
735
|
+
cutoff: options,
|
|
736
|
+
});
|
|
609
737
|
}
|
|
610
738
|
/**
|
|
611
739
|
* Search EDGAR submissions for recent filings of specified form types, newest first.
|
|
@@ -707,10 +835,12 @@ class EdgarApiService {
|
|
|
707
835
|
* 404's, a retried 5xx's, a 429's, one that lands as the caller cancels — is
|
|
708
836
|
* cancelled unread (#188); an error the last attempt earns carries a bounded head
|
|
709
837
|
* of its body, read like any other (#202). The body is read inside the ladder
|
|
710
|
-
* (#181): a 5xx, an attempt that got no HTTP response,
|
|
711
|
-
*
|
|
712
|
-
*
|
|
713
|
-
*
|
|
838
|
+
* (#181): a 5xx, an attempt that got no HTTP response, a 2xx whose body failed to
|
|
839
|
+
* read, and — for a request that opts in with `backendFailure` — a 2xx whose whole
|
|
840
|
+
* body relays a transient failure of the search backend (#206) all retry up to
|
|
841
|
+
* `MAX_RETRIES` attempts, and then the last attempt decides — a 5xx throws its
|
|
842
|
+
* status-classified `McpError`, a backend failure `efts_degraded_response`, the
|
|
843
|
+
* other two `sec_unreachable`. Other non-OK statuses throw at once, as does
|
|
714
844
|
* the local refusal while SEC's rate-limit block is active, and the pacer's
|
|
715
845
|
* `RequestCancelled` — those are `McpError`s from `send()`, never retried.
|
|
716
846
|
*
|
|
@@ -720,7 +850,7 @@ class EdgarApiService {
|
|
|
720
850
|
* untouched — a queued request leaves the pacer unsent, and an attempt in flight,
|
|
721
851
|
* a body read, or a backoff sleep ends at once — never retried or classified.
|
|
722
852
|
*/
|
|
723
|
-
async rawFetch(url, acceptJson, scope,
|
|
853
|
+
async rawFetch(url, acceptJson, scope, options = {}) {
|
|
724
854
|
scope.signal.throwIfAborted();
|
|
725
855
|
const headers = { 'User-Agent': getServerConfig().userAgent };
|
|
726
856
|
if (acceptJson)
|
|
@@ -751,17 +881,26 @@ class EdgarApiService {
|
|
|
751
881
|
return null;
|
|
752
882
|
}
|
|
753
883
|
if (response.ok) {
|
|
754
|
-
const
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
884
|
+
const readFrom = Date.now();
|
|
885
|
+
const body = await readBody(response, scope, options.cutoff);
|
|
886
|
+
if (body.kind === 'read') {
|
|
887
|
+
const error = options.backendFailure?.(body.text, url);
|
|
888
|
+
if (error === undefined)
|
|
889
|
+
return body.text;
|
|
890
|
+
last = { kind: 'backend_failed', error, waitedMs: Date.now() - readFrom };
|
|
891
|
+
}
|
|
892
|
+
else {
|
|
893
|
+
last = body;
|
|
894
|
+
}
|
|
758
895
|
}
|
|
759
896
|
}
|
|
760
897
|
const transient = last.kind !== 'answered' || RETRYABLE_STATUSES.has(last.response.status);
|
|
761
898
|
if (!transient) {
|
|
762
899
|
throw await this.ladderFailure(url, last, { attempt, budgetStopped: false }, scope);
|
|
763
900
|
}
|
|
764
|
-
|
|
901
|
+
// A failed body read adds its own wait to the wait for the headers.
|
|
902
|
+
const readWaitMs = last.kind === 'body_failed' || last.kind === 'backend_failed' ? last.waitedMs : 0;
|
|
903
|
+
scope.charge(outcome.waitedMs + readWaitMs);
|
|
765
904
|
const backoffMs = BASE_BACKOFF_MS * 2 ** (attempt - 1);
|
|
766
905
|
if (attempt < MAX_RETRIES && scope.remainingMs > backoffMs) {
|
|
767
906
|
if (last.kind === 'answered')
|
|
@@ -776,7 +915,8 @@ class EdgarApiService {
|
|
|
776
915
|
}
|
|
777
916
|
/**
|
|
778
917
|
* The error a request's last attempt earned: `sec_unreachable` for no response or
|
|
779
|
-
* an unreadable body; for
|
|
918
|
+
* an unreadable body; `efts_degraded_response` for a body relaying a failure of the
|
|
919
|
+
* search backend (#206); for an HTTP answer, its status-classified `McpError`, with
|
|
780
920
|
* the 403 and 429 hints. A 5xx ladder the failure budget stopped keeps its status
|
|
781
921
|
* mapping and `data`, and its message says the budget stopped it (#179).
|
|
782
922
|
*
|
|
@@ -786,6 +926,8 @@ class EdgarApiService {
|
|
|
786
926
|
* a body, the 429's cancelled unread (#188).
|
|
787
927
|
*/
|
|
788
928
|
async ladderFailure(url, last, end, scope) {
|
|
929
|
+
if (last.kind === 'backend_failed')
|
|
930
|
+
return eftsBackendFailure(url, last.error, end);
|
|
789
931
|
if (last.kind !== 'answered')
|
|
790
932
|
return secUnreachable(url, last, end);
|
|
791
933
|
const { response } = last;
|
|
@@ -1242,6 +1384,47 @@ export function parseSeriesFilingFeed(xml) {
|
|
|
1242
1384
|
filings,
|
|
1243
1385
|
};
|
|
1244
1386
|
}
|
|
1387
|
+
/** The query start of the company-browse links an index page gives each party. */
|
|
1388
|
+
const BROWSE_EDGAR_QUERY = 'browse-edgar?';
|
|
1389
|
+
const CIK_PARAM = 'CIK=';
|
|
1390
|
+
const TEN_DIGITS = /^\d{10}$/;
|
|
1391
|
+
/** True for a character that ends an `href` value: whitespace or a control character, a quote, `<`, or `>`. */
|
|
1392
|
+
function endsHref(code) {
|
|
1393
|
+
return code <= 0x20 || code === 0x22 || code === 0x27 || code === 0x3c || code === 0x3e;
|
|
1394
|
+
}
|
|
1395
|
+
/**
|
|
1396
|
+
* The 10-digit CIKs a filing index page (`<accession>-index.html`) names, in page
|
|
1397
|
+
* order without repeats: every party block links its company as
|
|
1398
|
+
* `browse-edgar?CIK=<10 digits>&action=getcompany`, while the page's other
|
|
1399
|
+
* `browse-edgar` links (latest filings, file number, SIC code) carry no `CIK`
|
|
1400
|
+
* parameter (#203). One forward pass: each link's query ends at the first character
|
|
1401
|
+
* that ends an `href`, the next link is searched for from there, and the `CIK`
|
|
1402
|
+
* parameter is looked for inside that query alone, so no character is scanned more
|
|
1403
|
+
* than twice whatever the markup. Exported for direct unit testing.
|
|
1404
|
+
*/
|
|
1405
|
+
export function filingPartyCiks(html) {
|
|
1406
|
+
const ciks = new Set();
|
|
1407
|
+
let link = html.indexOf(BROWSE_EDGAR_QUERY);
|
|
1408
|
+
while (link >= 0) {
|
|
1409
|
+
const start = link + BROWSE_EDGAR_QUERY.length;
|
|
1410
|
+
let end = start;
|
|
1411
|
+
while (end < html.length && !endsHref(html.charCodeAt(end)))
|
|
1412
|
+
end++;
|
|
1413
|
+
const query = html.slice(start, end);
|
|
1414
|
+
for (let at = query.indexOf(CIK_PARAM); at >= 0; at = query.indexOf(CIK_PARAM, at + 1)) {
|
|
1415
|
+
const opensParam = at === 0 || query[at - 1] === '&' || query[at - 1] === ';';
|
|
1416
|
+
const valueEnd = at + CIK_PARAM.length + 10;
|
|
1417
|
+
const value = query.slice(at + CIK_PARAM.length, valueEnd);
|
|
1418
|
+
const nextCode = query.charCodeAt(valueEnd);
|
|
1419
|
+
if (opensParam && TEN_DIGITS.test(value) && !(nextCode >= 0x30 && nextCode <= 0x39)) {
|
|
1420
|
+
ciks.add(value);
|
|
1421
|
+
break;
|
|
1422
|
+
}
|
|
1423
|
+
}
|
|
1424
|
+
link = html.indexOf(BROWSE_EDGAR_QUERY, end);
|
|
1425
|
+
}
|
|
1426
|
+
return [...ciks];
|
|
1427
|
+
}
|
|
1245
1428
|
/**
|
|
1246
1429
|
* Name of the raw XML document inside a filing's archive directory. The submissions feed
|
|
1247
1430
|
* points at the human-readable rendering (`xslSCHEDULE_13D_X02/primary_doc.xml`,
|
|
@@ -1487,18 +1670,28 @@ async function errorBodyHead(response, scope) {
|
|
|
1487
1670
|
const read = await readBody(response, scope, { maxBytes: ERROR_BODY_LIMIT_BYTES + 4 });
|
|
1488
1671
|
if (read.kind === 'body_failed')
|
|
1489
1672
|
scope.charge(read.waitedMs);
|
|
1490
|
-
const
|
|
1491
|
-
if (
|
|
1492
|
-
|
|
1493
|
-
const head = new TextDecoder().decode(bytes.subarray(0, ERROR_BODY_LIMIT_BYTES), {
|
|
1494
|
-
stream: true,
|
|
1495
|
-
});
|
|
1496
|
-
return `${head}…`;
|
|
1497
|
-
}
|
|
1673
|
+
const cut = cutToErrorLimit(read.text);
|
|
1674
|
+
if (cut !== undefined)
|
|
1675
|
+
return cut;
|
|
1498
1676
|
if (read.kind === 'read')
|
|
1499
1677
|
return read.text;
|
|
1500
1678
|
return read.text === '' ? undefined : `${read.text}…`;
|
|
1501
1679
|
}
|
|
1680
|
+
/**
|
|
1681
|
+
* `text`'s first {@link ERROR_BODY_LIMIT_BYTES} bytes — never a partial code point —
|
|
1682
|
+
* and `…`, or `undefined` when the whole of it fits: the bound on every upstream
|
|
1683
|
+
* text an error carries (#202, #206).
|
|
1684
|
+
*/
|
|
1685
|
+
function cutToErrorLimit(text) {
|
|
1686
|
+
const bytes = utf8.encode(text);
|
|
1687
|
+
if (bytes.byteLength <= ERROR_BODY_LIMIT_BYTES)
|
|
1688
|
+
return;
|
|
1689
|
+
// A streaming decode holds back the code point the cut splits instead of emitting U+FFFD.
|
|
1690
|
+
const head = new TextDecoder().decode(bytes.subarray(0, ERROR_BODY_LIMIT_BYTES), {
|
|
1691
|
+
stream: true,
|
|
1692
|
+
});
|
|
1693
|
+
return `${head}…`;
|
|
1694
|
+
}
|
|
1502
1695
|
/**
|
|
1503
1696
|
* Release a body the ladder will not read (#188), so no runtime holds its
|
|
1504
1697
|
* connection for it. A cancel that fails changes nothing: the body is discarded
|
|
@@ -1539,6 +1732,7 @@ async function nextChunk(reader, stalled, capMs, signal) {
|
|
|
1539
1732
|
function cutByBudget(last) {
|
|
1540
1733
|
switch (last.kind) {
|
|
1541
1734
|
case 'answered':
|
|
1735
|
+
case 'backend_failed':
|
|
1542
1736
|
return false;
|
|
1543
1737
|
case 'unanswered':
|
|
1544
1738
|
return last.timedOut && last.capMs < ATTEMPT_HEADERS_TIMEOUT_MS;
|
|
@@ -1631,10 +1825,7 @@ function secUnreachable(url, last, end) {
|
|
|
1631
1825
|
/** The message and recovery hint for each way SEC's answer was unreadable. */
|
|
1632
1826
|
function unreadableText(host, last, end) {
|
|
1633
1827
|
const transportEnding = (rejection) => `ended with ${transportCode(rejection) ?? 'a network error that carried no code'}`;
|
|
1634
|
-
const onAttempt = end
|
|
1635
|
-
? `attempt ${end.attempt} of ${MAX_RETRIES}`
|
|
1636
|
-
: `the last of ${MAX_RETRIES} attempts`;
|
|
1637
|
-
const close = end?.budgetStopped ? `; ${BUDGET_STOP_TEXT}.` : '.';
|
|
1828
|
+
const { close, onAttempt } = ladderEndText(end);
|
|
1638
1829
|
switch (last.kind) {
|
|
1639
1830
|
case 'unanswered':
|
|
1640
1831
|
return {
|
|
@@ -1662,6 +1853,98 @@ function unreadableText(host, last, end) {
|
|
|
1662
1853
|
}
|
|
1663
1854
|
}
|
|
1664
1855
|
}
|
|
1856
|
+
/**
|
|
1857
|
+
* Where a ladder's failure says it ended: on the last of its attempts, or — when the
|
|
1858
|
+
* call's failure budget ended it — on the attempt it reached, naming the budget (#179).
|
|
1859
|
+
*/
|
|
1860
|
+
function ladderEndText(end) {
|
|
1861
|
+
return end?.budgetStopped
|
|
1862
|
+
? { onAttempt: `attempt ${end.attempt} of ${MAX_RETRIES}`, close: `; ${BUDGET_STOP_TEXT}.` }
|
|
1863
|
+
: { onAttempt: `the last of ${MAX_RETRIES} attempts`, close: '.' };
|
|
1864
|
+
}
|
|
1865
|
+
/**
|
|
1866
|
+
* Caller text as it is sent to EDGAR full-text search, the one encoding every `q` and
|
|
1867
|
+
* `keysTyped` value goes through: each backslash doubled and each control character
|
|
1868
|
+
* (Unicode category Cc) written as `\uXXXX`, so a term holding either is searched as
|
|
1869
|
+
* typed. Quotes, operators, and wildcards pass unchanged and keep their search syntax
|
|
1870
|
+
* (#205).
|
|
1871
|
+
*/
|
|
1872
|
+
function escapeEftsText(text) {
|
|
1873
|
+
return text.replace(/[\\\p{Cc}]/gu, (char) => char === '\\' ? '\\\\' : `\\u${char.charCodeAt(0).toString(16).padStart(4, '0')}`);
|
|
1874
|
+
}
|
|
1875
|
+
/** `data.reason` when EDGAR full-text search answers a 200 without its results (#61, #206). */
|
|
1876
|
+
const EFTS_DEGRADED_REASON = 'efts_degraded_response';
|
|
1877
|
+
/**
|
|
1878
|
+
* The `errorType` values naming a failure to reach EDGAR full-text search's OpenSearch
|
|
1879
|
+
* cluster at all — its client's transport errors, which a resend can get past, so a
|
|
1880
|
+
* body naming one fails its attempt on the ladder. Every other class fails on its one
|
|
1881
|
+
* request: a `ResponseError` is the cluster answering with an error, such as the
|
|
1882
|
+
* result window that a `from` past 9,900 overruns (#206).
|
|
1883
|
+
*/
|
|
1884
|
+
const TRANSIENT_EFTS_ERRORS = new Set([
|
|
1885
|
+
'ConnectionError',
|
|
1886
|
+
'TimeoutError',
|
|
1887
|
+
'NoLivingConnectionsError',
|
|
1888
|
+
]);
|
|
1889
|
+
/** The backend error a parsed EDGAR full-text search body relays in place of its results, if it relays one (#206). */
|
|
1890
|
+
function eftsBackendError(body) {
|
|
1891
|
+
if (!isRecord(body) || typeof body.errorType !== 'string')
|
|
1892
|
+
return;
|
|
1893
|
+
return {
|
|
1894
|
+
errorType: body.errorType,
|
|
1895
|
+
errorMessage: typeof body.errorMessage === 'string' ? body.errorMessage : undefined,
|
|
1896
|
+
};
|
|
1897
|
+
}
|
|
1898
|
+
/**
|
|
1899
|
+
* The ladder's check on an EDGAR full-text search body read whole: the transient
|
|
1900
|
+
* backend error it relays, else `undefined` (#206). Only a body holding the key is
|
|
1901
|
+
* parsed here, so a page of hits is not parsed twice; one that does not parse fails
|
|
1902
|
+
* on its one request, as the caller's own parse would fail it (#181).
|
|
1903
|
+
*/
|
|
1904
|
+
function transientEftsError(text, url) {
|
|
1905
|
+
if (!text.includes('"errorType"'))
|
|
1906
|
+
return;
|
|
1907
|
+
const error = eftsBackendError(parseJson(url, text));
|
|
1908
|
+
return error && TRANSIENT_EFTS_ERRORS.has(error.errorType) ? error : undefined;
|
|
1909
|
+
}
|
|
1910
|
+
/**
|
|
1911
|
+
* The failure for a 200 relaying a backend error in place of EDGAR full-text search's
|
|
1912
|
+
* results (#206). With `end`, a transient one the ladder retried until it ended:
|
|
1913
|
+
* `retryable: true`, the message naming the attempt it ended on. Without, any other
|
|
1914
|
+
* error, failed on its one request: `retryable: false`, since resending the request
|
|
1915
|
+
* unchanged draws the same error. The error's class and message ride typed fields,
|
|
1916
|
+
* bounded like an error body and kept out of the message; the body's `trace` (the
|
|
1917
|
+
* backend's stack frames) and `requestId` go nowhere — the framework writes its own
|
|
1918
|
+
* `data.requestId`.
|
|
1919
|
+
*/
|
|
1920
|
+
function eftsBackendFailure(url, error, end) {
|
|
1921
|
+
const host = new URL(url).hostname;
|
|
1922
|
+
const upstream = {
|
|
1923
|
+
upstreamErrorType: cutToErrorLimit(error.errorType) ?? error.errorType,
|
|
1924
|
+
...(error.errorMessage !== undefined && {
|
|
1925
|
+
upstreamErrorMessage: cutToErrorLimit(error.errorMessage) ?? error.errorMessage,
|
|
1926
|
+
}),
|
|
1927
|
+
};
|
|
1928
|
+
if (end === undefined) {
|
|
1929
|
+
return serviceUnavailable(`EDGAR full-text search (${host}) answered with an error from its search backend instead of results.`, {
|
|
1930
|
+
reason: EFTS_DEGRADED_REASON,
|
|
1931
|
+
retryable: false,
|
|
1932
|
+
...upstream,
|
|
1933
|
+
recovery: {
|
|
1934
|
+
hint: `EDGAR full-text search (${host}) refused to run this search, and resending it unchanged fails the same way — upstreamErrorMessage gives the reason to change.`,
|
|
1935
|
+
},
|
|
1936
|
+
});
|
|
1937
|
+
}
|
|
1938
|
+
const { close, onAttempt } = ladderEndText(end);
|
|
1939
|
+
return serviceUnavailable(`EDGAR full-text search (${host}) answered ${onAttempt} with a failure of its search backend instead of results${close}`, {
|
|
1940
|
+
reason: EFTS_DEGRADED_REASON,
|
|
1941
|
+
retryable: true,
|
|
1942
|
+
...upstream,
|
|
1943
|
+
recovery: {
|
|
1944
|
+
hint: `EDGAR full-text search (${host}) answered, but its search backend could not run the search — a temporary failure on SEC's side, not a problem with the request. Retry the same call in a minute.`,
|
|
1945
|
+
},
|
|
1946
|
+
});
|
|
1947
|
+
}
|
|
1665
1948
|
/** An identifier-shaped transport code: `ECONNRESET`, `EAI_AGAIN`, `UND_ERR_CONNECT_TIMEOUT`. */
|
|
1666
1949
|
const TRANSPORT_CODE = /^[A-Za-z][A-Za-z0-9_]*$/;
|
|
1667
1950
|
/**
|