@cyanheads/secedgar-mcp-server 0.17.0 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +18 -15
- package/CLAUDE.md +18 -15
- package/README.md +12 -9
- package/changelog/0.17.x/0.17.1.md +36 -0
- package/dist/config/server-config.js +1 -1
- package/dist/config/server-config.js.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/mcp-server/tools/definitions/company-search.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/company-search.tool.js +6 -4
- package/dist/mcp-server/tools/definitions/company-search.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/compare-companies.tool.d.ts +8 -4
- package/dist/mcp-server/tools/definitions/compare-companies.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/compare-companies.tool.js +169 -39
- package/dist/mcp-server/tools/definitions/compare-companies.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.d.ts +4 -4
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.js +24 -8
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/find-holders.tool.d.ts +14 -0
- package/dist/mcp-server/tools/definitions/find-holders.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/find-holders.tool.js +34 -7
- package/dist/mcp-server/tools/definitions/find-holders.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.d.ts +2 -0
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.js +146 -77
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-filing.tool.d.ts +4 -4
- package/dist/mcp-server/tools/definitions/get-filing.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-filing.tool.js +305 -81
- package/dist/mcp-server/tools/definitions/get-filing.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.d.ts +14 -0
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.js +8 -0
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-material-events.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-material-events.tool.js +3 -1
- package/dist/mcp-server/tools/definitions/get-material-events.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/index.d.ts +453 -423
- package/dist/mcp-server/tools/definitions/index.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/search-filings.tool.d.ts +33 -0
- package/dist/mcp-server/tools/definitions/search-filings.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/search-filings.tool.js +31 -2
- package/dist/mcp-server/tools/definitions/search-filings.tool.js.map +1 -1
- package/dist/mcp-server/tools/limit-clamp.d.ts +50 -18
- package/dist/mcp-server/tools/limit-clamp.d.ts.map +1 -1
- package/dist/mcp-server/tools/limit-clamp.js +51 -15
- package/dist/mcp-server/tools/limit-clamp.js.map +1 -1
- package/dist/services/edgar/edgar-api-service.d.ts +75 -11
- package/dist/services/edgar/edgar-api-service.d.ts.map +1 -1
- package/dist/services/edgar/edgar-api-service.js +273 -50
- package/dist/services/edgar/edgar-api-service.js.map +1 -1
- package/dist/services/edgar/filing-to-text.d.ts +4 -3
- package/dist/services/edgar/filing-to-text.d.ts.map +1 -1
- package/dist/services/edgar/filing-to-text.js +12 -0
- package/dist/services/edgar/filing-to-text.js.map +1 -1
- package/package.json +4 -4
- package/server.json +3 -3
- package/skills/secedgar-guide/SKILL.md +1 -0
|
@@ -161,7 +161,18 @@ function stripCompanySuffix(name) {
|
|
|
161
161
|
return;
|
|
162
162
|
return name.slice(0, cut).replace(/,$/, '').trimEnd();
|
|
163
163
|
}
|
|
164
|
-
|
|
164
|
+
/**
|
|
165
|
+
* Delete every comma, period, and apostrophe (`'` and `’`) from a company name, the
|
|
166
|
+
* comparison form the name passes' folded tiers read, so `QXO Inc` meets `QXO, Inc.`,
|
|
167
|
+
* `U.S. Bancorp` meets `US BANCORP`, and `Macys` meets `Macy's, Inc.` (#210). A period
|
|
168
|
+
* is deleted, never turned into a space, so a dotted initialism meets its undotted
|
|
169
|
+
* spelling. Nothing else changes — whitespace runs, `&`, `-`, `/`, and `\` stay — and the
|
|
170
|
+
* result is for comparison only: a returned name is never folded.
|
|
171
|
+
*/
|
|
172
|
+
export function foldNamePunctuation(name) {
|
|
173
|
+
return name.replace(/[,.'’]/g, '');
|
|
174
|
+
}
|
|
175
|
+
const NAME_TIERS = [0, 1, 2, 3];
|
|
165
176
|
// ---------------------------------------------------------------------------
|
|
166
177
|
// Near-match company suggestions
|
|
167
178
|
// ---------------------------------------------------------------------------
|
|
@@ -235,9 +246,12 @@ class EdgarApiService {
|
|
|
235
246
|
dispose() {
|
|
236
247
|
this.pacer.dispose();
|
|
237
248
|
}
|
|
238
|
-
/**
|
|
239
|
-
|
|
240
|
-
|
|
249
|
+
/**
|
|
250
|
+
* Fetch and parse JSON, throwing on non-OK responses and on a body that is not JSON.
|
|
251
|
+
* `backendFailure` names the 200 bodies that fail an attempt (#206).
|
|
252
|
+
*/
|
|
253
|
+
async fetchJson(url, scope, options) {
|
|
254
|
+
const text = await this.rawFetch(url, true, scope, options);
|
|
241
255
|
if (text === null) {
|
|
242
256
|
throw notFound(`SEC EDGAR API returned 404 for ${url}`, { url, status: 404 });
|
|
243
257
|
}
|
|
@@ -265,8 +279,9 @@ class EdgarApiService {
|
|
|
265
279
|
* Resolve a query (ticker, name, or CIK) to company match(es).
|
|
266
280
|
* - Numeric input → direct CIK lookup
|
|
267
281
|
* - 1-5 uppercase letters → ticker lookup (O(1))
|
|
268
|
-
* - Otherwise → name search (exact
|
|
269
|
-
*
|
|
282
|
+
* - Otherwise → name search (exact as written, exact once commas, periods, and
|
|
283
|
+
* apostrophes are folded out, prefix once folded, substring as written — resolved
|
|
284
|
+
* within the best tier a current name reaches), then the catch-all ticker lookup
|
|
270
285
|
* Returns a single match, an array of multiple matches, or an empty array (no match).
|
|
271
286
|
* On no-result name search, the returned empty array carries `suggestions` on
|
|
272
287
|
* the thrown error at the handler layer — call `suggestCompanies` there.
|
|
@@ -291,19 +306,32 @@ class EdgarApiService {
|
|
|
291
306
|
if (match)
|
|
292
307
|
return match;
|
|
293
308
|
}
|
|
294
|
-
// Name search, tiered exact →
|
|
295
|
-
// suffix-normalized equality (#107) and the registry name
|
|
296
|
-
// so `Apple` names `Apple Inc.` exactly (#155); the query's
|
|
297
|
-
// dropped, so `Toro Co` never meets `TORO CORP.`.
|
|
309
|
+
// Name search, tiered exact → exact once folded → prefix once folded → substring. Both
|
|
310
|
+
// exact tiers also accept a suffix-normalized equality (#107) and the registry name
|
|
311
|
+
// minus its terminal suffix, so `Apple` names `Apple Inc.` exactly (#155); the query's
|
|
312
|
+
// own suffix is never dropped, so `Toro Co` never meets `TORO CORP.`. The folded tiers
|
|
313
|
+
// compare with commas, periods, and apostrophes deleted from both sides (#210). Exact
|
|
314
|
+
// as written ranks first, so `McGraw Hill, Inc.` never ties a former name that equals
|
|
315
|
+
// it only once folded, and substring stays as written, since `S.Y. Bancorp Inc` folded
|
|
316
|
+
// sits inside a folded `Embassy Bancorp, Inc.`.
|
|
298
317
|
const lower = trimmed.toLowerCase();
|
|
299
318
|
const lowerNormalized = normalizeCompanySuffix(lower);
|
|
300
|
-
|
|
319
|
+
const folded = foldNamePunctuation(lower);
|
|
320
|
+
const foldedNormalized = normalizeCompanySuffix(folded);
|
|
321
|
+
// A suffix-dropped name is a prefix of the full one, so the strip runs on prefix hits
|
|
322
|
+
// only. A prefix as written is a prefix once folded, so the folded tiers rank it.
|
|
301
323
|
const tierOf = (name) => {
|
|
302
324
|
if (name === lower || normalizeCompanySuffix(name) === lowerNormalized)
|
|
303
325
|
return 0;
|
|
304
|
-
if (name.startsWith(lower))
|
|
305
|
-
return
|
|
306
|
-
|
|
326
|
+
if (name.startsWith(lower) && stripCompanySuffix(name) === lower)
|
|
327
|
+
return 0;
|
|
328
|
+
const foldedName = foldNamePunctuation(name);
|
|
329
|
+
if (foldedName === folded || normalizeCompanySuffix(foldedName) === foldedNormalized) {
|
|
330
|
+
return 1;
|
|
331
|
+
}
|
|
332
|
+
if (foldedName.startsWith(folded))
|
|
333
|
+
return stripCompanySuffix(foldedName) === folded ? 1 : 2;
|
|
334
|
+
return name.includes(lower) ? 3 : undefined;
|
|
307
335
|
};
|
|
308
336
|
// Current names are registry rows, which carry a ticker; former names (the committed
|
|
309
337
|
// asset) carry none. Resolution runs within the best tier a current name reaches —
|
|
@@ -311,8 +339,8 @@ class EdgarApiService {
|
|
|
311
339
|
// name joins at that tier or better but never outranks a current name, so a company
|
|
312
340
|
// is never silently swapped for a former holder of its name; former names decide
|
|
313
341
|
// alone only when no current name matches (#155).
|
|
314
|
-
const current = { 0: [], 1: [], 2: [] };
|
|
315
|
-
const former = { 0: [], 1: [], 2: [] };
|
|
342
|
+
const current = { 0: [], 1: [], 2: [], 3: [] };
|
|
343
|
+
const former = { 0: [], 1: [], 2: [], 3: [] };
|
|
316
344
|
for (const entry of cache.allEntries) {
|
|
317
345
|
if (!entry.name)
|
|
318
346
|
continue;
|
|
@@ -358,12 +386,19 @@ class EdgarApiService {
|
|
|
358
386
|
* a legal name under different CIKs (e.g. "VANGUARD GROUP INC" = a transfer agent and
|
|
359
387
|
* the 13F filer), so callers must disambiguate rather than auto-pick the top hit (#73).
|
|
360
388
|
* Distinct from `searchFilings`: same URL path, different query param (`keysTyped`)
|
|
361
|
-
* and response shape (`_source.entity`, not the filing-document `_source`).
|
|
389
|
+
* and response shape (`_source.entity`, not the filing-document `_source`). A
|
|
390
|
+
* failure of the search backend throws `efts_degraded_response` like a search's,
|
|
391
|
+
* never reads as no match (#206). The name is escaped like a search's `q` (#205).
|
|
362
392
|
*/
|
|
363
393
|
async resolveEntityByName(name, scope) {
|
|
364
394
|
const url = new URL('https://efts.sec.gov/LATEST/search-index');
|
|
365
|
-
url.searchParams.set('keysTyped', name);
|
|
366
|
-
const response = await this.fetchJson(url.toString(), scope
|
|
395
|
+
url.searchParams.set('keysTyped', escapeEftsText(name));
|
|
396
|
+
const response = await this.fetchJson(url.toString(), scope, {
|
|
397
|
+
backendFailure: transientEftsError,
|
|
398
|
+
});
|
|
399
|
+
const backendError = eftsBackendError(response);
|
|
400
|
+
if (backendError)
|
|
401
|
+
throw eftsBackendFailure(url.toString(), backendError);
|
|
367
402
|
const hits = response?.hits?.hits;
|
|
368
403
|
if (!Array.isArray(hits))
|
|
369
404
|
return [];
|
|
@@ -501,9 +536,11 @@ class EdgarApiService {
|
|
|
501
536
|
async searchFilings(params, scope) {
|
|
502
537
|
const url = new URL('https://efts.sec.gov/LATEST/search-index');
|
|
503
538
|
// `q` is optional — EFTS honors `ciks` for pure entity scope with no
|
|
504
|
-
// full-text query, so a bare cik:/ticker: search sends no `q`.
|
|
539
|
+
// full-text query, so a bare cik:/ticker: search sends no `q`. Escaped here, at
|
|
540
|
+
// the one place a query is sent, so a caller re-sending a phrase another tool
|
|
541
|
+
// searched (find_holders' `search_key`) runs the same search (#205).
|
|
505
542
|
if (params.query)
|
|
506
|
-
url.searchParams.set('q', params.query);
|
|
543
|
+
url.searchParams.set('q', escapeEftsText(params.query));
|
|
507
544
|
if (params.forms?.length)
|
|
508
545
|
url.searchParams.set('forms', params.forms.join(','));
|
|
509
546
|
// Server-side entity scope by CIK, independent of the document's name text —
|
|
@@ -517,7 +554,14 @@ class EdgarApiService {
|
|
|
517
554
|
}
|
|
518
555
|
url.searchParams.set('from', String(params.from ?? 0));
|
|
519
556
|
url.searchParams.set('size', String(params.size ?? 20));
|
|
520
|
-
|
|
557
|
+
// A transport failure of the search backend retries on the ladder; any other
|
|
558
|
+
// backend error fails on its one request (#206).
|
|
559
|
+
const response = await this.fetchJson(url.toString(), scope, {
|
|
560
|
+
backendFailure: transientEftsError,
|
|
561
|
+
});
|
|
562
|
+
const backendError = eftsBackendError(response);
|
|
563
|
+
if (backendError)
|
|
564
|
+
throw eftsBackendFailure(url.toString(), backendError);
|
|
521
565
|
// Shape guard: EFTS can return a 2xx whose body omits `hits.total` (degraded
|
|
522
566
|
// payload, or a rejected request echoed as `{ error: ... }` with 200). A genuine
|
|
523
567
|
// zero-hit response is still well-formed (`hits.total.value: 0`, empty
|
|
@@ -526,7 +570,7 @@ class EdgarApiService {
|
|
|
526
570
|
if (!Array.isArray(response?.hits?.hits) || typeof response.hits.total?.value !== 'number') {
|
|
527
571
|
const upstreamError = response?.error;
|
|
528
572
|
throw serviceUnavailable('SEC EDGAR full-text search returned an unexpected response without hits — the service may be degraded.', {
|
|
529
|
-
reason:
|
|
573
|
+
reason: EFTS_DEGRADED_REASON,
|
|
530
574
|
...(typeof upstreamError === 'string' && { upstreamError }),
|
|
531
575
|
recovery: {
|
|
532
576
|
hint: 'Retry the search in a few minutes — EDGAR full-text search returned an incomplete response.',
|
|
@@ -553,9 +597,18 @@ class EdgarApiService {
|
|
|
553
597
|
}
|
|
554
598
|
return [...ciks];
|
|
555
599
|
}
|
|
556
|
-
/**
|
|
557
|
-
|
|
558
|
-
|
|
600
|
+
/**
|
|
601
|
+
* Fetch a filing's document index. Returns `null` if the filing does not exist.
|
|
602
|
+
* Entries listed with no filename are dropped: SEC lists each document of a
|
|
603
|
+
* pre-2001 SGML submission that way, with only its size, and such a document
|
|
604
|
+
* exists only inside the full submission `<accession>.txt` (#203).
|
|
605
|
+
*/
|
|
606
|
+
async tryGetFilingIndex(cik, accessionNumber, scope) {
|
|
607
|
+
const index = await this.tryFetchJson(filingArchiveUrl(cik, accessionNumber, 'index.json'), scope);
|
|
608
|
+
if (index === null)
|
|
609
|
+
return null;
|
|
610
|
+
const named = index.directory.item.filter((item) => item.name.trim() !== '');
|
|
611
|
+
return { ...index, directory: { ...index.directory, item: named } };
|
|
559
612
|
}
|
|
560
613
|
/**
|
|
561
614
|
* Fetch the SEC submission header page (`<accession>-index-headers.html`) and parse
|
|
@@ -567,6 +620,19 @@ class EdgarApiService {
|
|
|
567
620
|
const text = await this.tryFetchText(filingArchiveUrl(cik, accessionNumber, `${accessionNumber}-index-headers.html`), scope);
|
|
568
621
|
return text ? parseFilingHeaders(text) : null;
|
|
569
622
|
}
|
|
623
|
+
/**
|
|
624
|
+
* Fetch a filing's index page (`<accession>-index.html`) and return the CIKs of the
|
|
625
|
+
* parties it names — filer, subject company, filed-by, reporting owner, issuer — in
|
|
626
|
+
* page order, without repeats (see {@link filingPartyCiks}). Returns `null` if the
|
|
627
|
+
* page is absent. SEC serves the page under the accession number's own prefix CIK
|
|
628
|
+
* for any filing that exists, including a filing agent's prefix whose directory has
|
|
629
|
+
* no `index.json`, so it names the filers of an accession full-text search does not
|
|
630
|
+
* cover (before 2001) (#203).
|
|
631
|
+
*/
|
|
632
|
+
async tryGetFilingPartyCiks(cik, accessionNumber, scope) {
|
|
633
|
+
const html = await this.tryFetchText(filingArchiveUrl(cik, accessionNumber, `${accessionNumber}-index.html`), scope);
|
|
634
|
+
return html === null ? null : filingPartyCiks(html);
|
|
635
|
+
}
|
|
570
636
|
/**
|
|
571
637
|
* Fetch the bare SGML submission header (`<accession>.hdr.sgml`, under 1 KB) and read
|
|
572
638
|
* its form, filing date, and period. Returns `null` if the file is absent. The
|
|
@@ -605,7 +671,9 @@ class EdgarApiService {
|
|
|
605
671
|
* retried; stopping at the cutoff is not a failure (#181).
|
|
606
672
|
*/
|
|
607
673
|
tryGetFilingDocumentHead(cik, accessionNumber, document, options, scope) {
|
|
608
|
-
return this.rawFetch(filingArchiveUrl(cik, accessionNumber, document), false, scope,
|
|
674
|
+
return this.rawFetch(filingArchiveUrl(cik, accessionNumber, document), false, scope, {
|
|
675
|
+
cutoff: options,
|
|
676
|
+
});
|
|
609
677
|
}
|
|
610
678
|
/**
|
|
611
679
|
* Search EDGAR submissions for recent filings of specified form types, newest first.
|
|
@@ -707,10 +775,12 @@ class EdgarApiService {
|
|
|
707
775
|
* 404's, a retried 5xx's, a 429's, one that lands as the caller cancels — is
|
|
708
776
|
* cancelled unread (#188); an error the last attempt earns carries a bounded head
|
|
709
777
|
* of its body, read like any other (#202). The body is read inside the ladder
|
|
710
|
-
* (#181): a 5xx, an attempt that got no HTTP response,
|
|
711
|
-
*
|
|
712
|
-
*
|
|
713
|
-
*
|
|
778
|
+
* (#181): a 5xx, an attempt that got no HTTP response, a 2xx whose body failed to
|
|
779
|
+
* read, and — for a request that opts in with `backendFailure` — a 2xx whose whole
|
|
780
|
+
* body relays a transient failure of the search backend (#206) all retry up to
|
|
781
|
+
* `MAX_RETRIES` attempts, and then the last attempt decides — a 5xx throws its
|
|
782
|
+
* status-classified `McpError`, a backend failure `efts_degraded_response`, the
|
|
783
|
+
* other two `sec_unreachable`. Other non-OK statuses throw at once, as does
|
|
714
784
|
* the local refusal while SEC's rate-limit block is active, and the pacer's
|
|
715
785
|
* `RequestCancelled` — those are `McpError`s from `send()`, never retried.
|
|
716
786
|
*
|
|
@@ -720,7 +790,7 @@ class EdgarApiService {
|
|
|
720
790
|
* untouched — a queued request leaves the pacer unsent, and an attempt in flight,
|
|
721
791
|
* a body read, or a backoff sleep ends at once — never retried or classified.
|
|
722
792
|
*/
|
|
723
|
-
async rawFetch(url, acceptJson, scope,
|
|
793
|
+
async rawFetch(url, acceptJson, scope, options = {}) {
|
|
724
794
|
scope.signal.throwIfAborted();
|
|
725
795
|
const headers = { 'User-Agent': getServerConfig().userAgent };
|
|
726
796
|
if (acceptJson)
|
|
@@ -751,17 +821,26 @@ class EdgarApiService {
|
|
|
751
821
|
return null;
|
|
752
822
|
}
|
|
753
823
|
if (response.ok) {
|
|
754
|
-
const
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
824
|
+
const readFrom = Date.now();
|
|
825
|
+
const body = await readBody(response, scope, options.cutoff);
|
|
826
|
+
if (body.kind === 'read') {
|
|
827
|
+
const error = options.backendFailure?.(body.text, url);
|
|
828
|
+
if (error === undefined)
|
|
829
|
+
return body.text;
|
|
830
|
+
last = { kind: 'backend_failed', error, waitedMs: Date.now() - readFrom };
|
|
831
|
+
}
|
|
832
|
+
else {
|
|
833
|
+
last = body;
|
|
834
|
+
}
|
|
758
835
|
}
|
|
759
836
|
}
|
|
760
837
|
const transient = last.kind !== 'answered' || RETRYABLE_STATUSES.has(last.response.status);
|
|
761
838
|
if (!transient) {
|
|
762
839
|
throw await this.ladderFailure(url, last, { attempt, budgetStopped: false }, scope);
|
|
763
840
|
}
|
|
764
|
-
|
|
841
|
+
// A failed body read adds its own wait to the wait for the headers.
|
|
842
|
+
const readWaitMs = last.kind === 'body_failed' || last.kind === 'backend_failed' ? last.waitedMs : 0;
|
|
843
|
+
scope.charge(outcome.waitedMs + readWaitMs);
|
|
765
844
|
const backoffMs = BASE_BACKOFF_MS * 2 ** (attempt - 1);
|
|
766
845
|
if (attempt < MAX_RETRIES && scope.remainingMs > backoffMs) {
|
|
767
846
|
if (last.kind === 'answered')
|
|
@@ -776,7 +855,8 @@ class EdgarApiService {
|
|
|
776
855
|
}
|
|
777
856
|
/**
|
|
778
857
|
* The error a request's last attempt earned: `sec_unreachable` for no response or
|
|
779
|
-
* an unreadable body; for
|
|
858
|
+
* an unreadable body; `efts_degraded_response` for a body relaying a failure of the
|
|
859
|
+
* search backend (#206); for an HTTP answer, its status-classified `McpError`, with
|
|
780
860
|
* the 403 and 429 hints. A 5xx ladder the failure budget stopped keeps its status
|
|
781
861
|
* mapping and `data`, and its message says the budget stopped it (#179).
|
|
782
862
|
*
|
|
@@ -786,6 +866,8 @@ class EdgarApiService {
|
|
|
786
866
|
* a body, the 429's cancelled unread (#188).
|
|
787
867
|
*/
|
|
788
868
|
async ladderFailure(url, last, end, scope) {
|
|
869
|
+
if (last.kind === 'backend_failed')
|
|
870
|
+
return eftsBackendFailure(url, last.error, end);
|
|
789
871
|
if (last.kind !== 'answered')
|
|
790
872
|
return secUnreachable(url, last, end);
|
|
791
873
|
const { response } = last;
|
|
@@ -1242,6 +1324,47 @@ export function parseSeriesFilingFeed(xml) {
|
|
|
1242
1324
|
filings,
|
|
1243
1325
|
};
|
|
1244
1326
|
}
|
|
1327
|
+
/** The query start of the company-browse links an index page gives each party. */
|
|
1328
|
+
const BROWSE_EDGAR_QUERY = 'browse-edgar?';
|
|
1329
|
+
const CIK_PARAM = 'CIK=';
|
|
1330
|
+
const TEN_DIGITS = /^\d{10}$/;
|
|
1331
|
+
/** True for a character that ends an `href` value: whitespace or a control character, a quote, `<`, or `>`. */
|
|
1332
|
+
function endsHref(code) {
|
|
1333
|
+
return code <= 0x20 || code === 0x22 || code === 0x27 || code === 0x3c || code === 0x3e;
|
|
1334
|
+
}
|
|
1335
|
+
/**
|
|
1336
|
+
* The 10-digit CIKs a filing index page (`<accession>-index.html`) names, in page
|
|
1337
|
+
* order without repeats: every party block links its company as
|
|
1338
|
+
* `browse-edgar?CIK=<10 digits>&action=getcompany`, while the page's other
|
|
1339
|
+
* `browse-edgar` links (latest filings, file number, SIC code) carry no `CIK`
|
|
1340
|
+
* parameter (#203). One forward pass: each link's query ends at the first character
|
|
1341
|
+
* that ends an `href`, the next link is searched for from there, and the `CIK`
|
|
1342
|
+
* parameter is looked for inside that query alone, so no character is scanned more
|
|
1343
|
+
* than twice whatever the markup. Exported for direct unit testing.
|
|
1344
|
+
*/
|
|
1345
|
+
export function filingPartyCiks(html) {
|
|
1346
|
+
const ciks = new Set();
|
|
1347
|
+
let link = html.indexOf(BROWSE_EDGAR_QUERY);
|
|
1348
|
+
while (link >= 0) {
|
|
1349
|
+
const start = link + BROWSE_EDGAR_QUERY.length;
|
|
1350
|
+
let end = start;
|
|
1351
|
+
while (end < html.length && !endsHref(html.charCodeAt(end)))
|
|
1352
|
+
end++;
|
|
1353
|
+
const query = html.slice(start, end);
|
|
1354
|
+
for (let at = query.indexOf(CIK_PARAM); at >= 0; at = query.indexOf(CIK_PARAM, at + 1)) {
|
|
1355
|
+
const opensParam = at === 0 || query[at - 1] === '&' || query[at - 1] === ';';
|
|
1356
|
+
const valueEnd = at + CIK_PARAM.length + 10;
|
|
1357
|
+
const value = query.slice(at + CIK_PARAM.length, valueEnd);
|
|
1358
|
+
const nextCode = query.charCodeAt(valueEnd);
|
|
1359
|
+
if (opensParam && TEN_DIGITS.test(value) && !(nextCode >= 0x30 && nextCode <= 0x39)) {
|
|
1360
|
+
ciks.add(value);
|
|
1361
|
+
break;
|
|
1362
|
+
}
|
|
1363
|
+
}
|
|
1364
|
+
link = html.indexOf(BROWSE_EDGAR_QUERY, end);
|
|
1365
|
+
}
|
|
1366
|
+
return [...ciks];
|
|
1367
|
+
}
|
|
1245
1368
|
/**
|
|
1246
1369
|
* Name of the raw XML document inside a filing's archive directory. The submissions feed
|
|
1247
1370
|
* points at the human-readable rendering (`xslSCHEDULE_13D_X02/primary_doc.xml`,
|
|
@@ -1487,18 +1610,28 @@ async function errorBodyHead(response, scope) {
|
|
|
1487
1610
|
const read = await readBody(response, scope, { maxBytes: ERROR_BODY_LIMIT_BYTES + 4 });
|
|
1488
1611
|
if (read.kind === 'body_failed')
|
|
1489
1612
|
scope.charge(read.waitedMs);
|
|
1490
|
-
const
|
|
1491
|
-
if (
|
|
1492
|
-
|
|
1493
|
-
const head = new TextDecoder().decode(bytes.subarray(0, ERROR_BODY_LIMIT_BYTES), {
|
|
1494
|
-
stream: true,
|
|
1495
|
-
});
|
|
1496
|
-
return `${head}…`;
|
|
1497
|
-
}
|
|
1613
|
+
const cut = cutToErrorLimit(read.text);
|
|
1614
|
+
if (cut !== undefined)
|
|
1615
|
+
return cut;
|
|
1498
1616
|
if (read.kind === 'read')
|
|
1499
1617
|
return read.text;
|
|
1500
1618
|
return read.text === '' ? undefined : `${read.text}…`;
|
|
1501
1619
|
}
|
|
1620
|
+
/**
|
|
1621
|
+
* `text`'s first {@link ERROR_BODY_LIMIT_BYTES} bytes — never a partial code point —
|
|
1622
|
+
* and `…`, or `undefined` when the whole of it fits: the bound on every upstream
|
|
1623
|
+
* text an error carries (#202, #206).
|
|
1624
|
+
*/
|
|
1625
|
+
function cutToErrorLimit(text) {
|
|
1626
|
+
const bytes = utf8.encode(text);
|
|
1627
|
+
if (bytes.byteLength <= ERROR_BODY_LIMIT_BYTES)
|
|
1628
|
+
return;
|
|
1629
|
+
// A streaming decode holds back the code point the cut splits instead of emitting U+FFFD.
|
|
1630
|
+
const head = new TextDecoder().decode(bytes.subarray(0, ERROR_BODY_LIMIT_BYTES), {
|
|
1631
|
+
stream: true,
|
|
1632
|
+
});
|
|
1633
|
+
return `${head}…`;
|
|
1634
|
+
}
|
|
1502
1635
|
/**
|
|
1503
1636
|
* Release a body the ladder will not read (#188), so no runtime holds its
|
|
1504
1637
|
* connection for it. A cancel that fails changes nothing: the body is discarded
|
|
@@ -1539,6 +1672,7 @@ async function nextChunk(reader, stalled, capMs, signal) {
|
|
|
1539
1672
|
function cutByBudget(last) {
|
|
1540
1673
|
switch (last.kind) {
|
|
1541
1674
|
case 'answered':
|
|
1675
|
+
case 'backend_failed':
|
|
1542
1676
|
return false;
|
|
1543
1677
|
case 'unanswered':
|
|
1544
1678
|
return last.timedOut && last.capMs < ATTEMPT_HEADERS_TIMEOUT_MS;
|
|
@@ -1631,10 +1765,7 @@ function secUnreachable(url, last, end) {
|
|
|
1631
1765
|
/** The message and recovery hint for each way SEC's answer was unreadable. */
|
|
1632
1766
|
function unreadableText(host, last, end) {
|
|
1633
1767
|
const transportEnding = (rejection) => `ended with ${transportCode(rejection) ?? 'a network error that carried no code'}`;
|
|
1634
|
-
const onAttempt = end
|
|
1635
|
-
? `attempt ${end.attempt} of ${MAX_RETRIES}`
|
|
1636
|
-
: `the last of ${MAX_RETRIES} attempts`;
|
|
1637
|
-
const close = end?.budgetStopped ? `; ${BUDGET_STOP_TEXT}.` : '.';
|
|
1768
|
+
const { close, onAttempt } = ladderEndText(end);
|
|
1638
1769
|
switch (last.kind) {
|
|
1639
1770
|
case 'unanswered':
|
|
1640
1771
|
return {
|
|
@@ -1662,6 +1793,98 @@ function unreadableText(host, last, end) {
|
|
|
1662
1793
|
}
|
|
1663
1794
|
}
|
|
1664
1795
|
}
|
|
1796
|
+
/**
|
|
1797
|
+
* Where a ladder's failure says it ended: on the last of its attempts, or — when the
|
|
1798
|
+
* call's failure budget ended it — on the attempt it reached, naming the budget (#179).
|
|
1799
|
+
*/
|
|
1800
|
+
function ladderEndText(end) {
|
|
1801
|
+
return end?.budgetStopped
|
|
1802
|
+
? { onAttempt: `attempt ${end.attempt} of ${MAX_RETRIES}`, close: `; ${BUDGET_STOP_TEXT}.` }
|
|
1803
|
+
: { onAttempt: `the last of ${MAX_RETRIES} attempts`, close: '.' };
|
|
1804
|
+
}
|
|
1805
|
+
/**
|
|
1806
|
+
* Caller text as it is sent to EDGAR full-text search, the one encoding every `q` and
|
|
1807
|
+
* `keysTyped` value goes through: each backslash doubled and each control character
|
|
1808
|
+
* (Unicode category Cc) written as `\uXXXX`, so a term holding either is searched as
|
|
1809
|
+
* typed. Quotes, operators, and wildcards pass unchanged and keep their search syntax
|
|
1810
|
+
* (#205).
|
|
1811
|
+
*/
|
|
1812
|
+
function escapeEftsText(text) {
|
|
1813
|
+
return text.replace(/[\\\p{Cc}]/gu, (char) => char === '\\' ? '\\\\' : `\\u${char.charCodeAt(0).toString(16).padStart(4, '0')}`);
|
|
1814
|
+
}
|
|
1815
|
+
/** `data.reason` when EDGAR full-text search answers a 200 without its results (#61, #206). */
|
|
1816
|
+
const EFTS_DEGRADED_REASON = 'efts_degraded_response';
|
|
1817
|
+
/**
|
|
1818
|
+
* The `errorType` values naming a failure to reach EDGAR full-text search's OpenSearch
|
|
1819
|
+
* cluster at all — its client's transport errors, which a resend can get past, so a
|
|
1820
|
+
* body naming one fails its attempt on the ladder. Every other class fails on its one
|
|
1821
|
+
* request: a `ResponseError` is the cluster answering with an error, such as the
|
|
1822
|
+
* result window that a `from` past 9,900 overruns (#206).
|
|
1823
|
+
*/
|
|
1824
|
+
const TRANSIENT_EFTS_ERRORS = new Set([
|
|
1825
|
+
'ConnectionError',
|
|
1826
|
+
'TimeoutError',
|
|
1827
|
+
'NoLivingConnectionsError',
|
|
1828
|
+
]);
|
|
1829
|
+
/** The backend error a parsed EDGAR full-text search body relays in place of its results, if it relays one (#206). */
|
|
1830
|
+
function eftsBackendError(body) {
|
|
1831
|
+
if (!isRecord(body) || typeof body.errorType !== 'string')
|
|
1832
|
+
return;
|
|
1833
|
+
return {
|
|
1834
|
+
errorType: body.errorType,
|
|
1835
|
+
errorMessage: typeof body.errorMessage === 'string' ? body.errorMessage : undefined,
|
|
1836
|
+
};
|
|
1837
|
+
}
|
|
1838
|
+
/**
|
|
1839
|
+
* The ladder's check on an EDGAR full-text search body read whole: the transient
|
|
1840
|
+
* backend error it relays, else `undefined` (#206). Only a body holding the key is
|
|
1841
|
+
* parsed here, so a page of hits is not parsed twice; one that does not parse fails
|
|
1842
|
+
* on its one request, as the caller's own parse would fail it (#181).
|
|
1843
|
+
*/
|
|
1844
|
+
function transientEftsError(text, url) {
|
|
1845
|
+
if (!text.includes('"errorType"'))
|
|
1846
|
+
return;
|
|
1847
|
+
const error = eftsBackendError(parseJson(url, text));
|
|
1848
|
+
return error && TRANSIENT_EFTS_ERRORS.has(error.errorType) ? error : undefined;
|
|
1849
|
+
}
|
|
1850
|
+
/**
|
|
1851
|
+
* The failure for a 200 relaying a backend error in place of EDGAR full-text search's
|
|
1852
|
+
* results (#206). With `end`, a transient one the ladder retried until it ended:
|
|
1853
|
+
* `retryable: true`, the message naming the attempt it ended on. Without, any other
|
|
1854
|
+
* error, failed on its one request: `retryable: false`, since resending the request
|
|
1855
|
+
* unchanged draws the same error. The error's class and message ride typed fields,
|
|
1856
|
+
* bounded like an error body and kept out of the message; the body's `trace` (the
|
|
1857
|
+
* backend's stack frames) and `requestId` go nowhere — the framework writes its own
|
|
1858
|
+
* `data.requestId`.
|
|
1859
|
+
*/
|
|
1860
|
+
function eftsBackendFailure(url, error, end) {
|
|
1861
|
+
const host = new URL(url).hostname;
|
|
1862
|
+
const upstream = {
|
|
1863
|
+
upstreamErrorType: cutToErrorLimit(error.errorType) ?? error.errorType,
|
|
1864
|
+
...(error.errorMessage !== undefined && {
|
|
1865
|
+
upstreamErrorMessage: cutToErrorLimit(error.errorMessage) ?? error.errorMessage,
|
|
1866
|
+
}),
|
|
1867
|
+
};
|
|
1868
|
+
if (end === undefined) {
|
|
1869
|
+
return serviceUnavailable(`EDGAR full-text search (${host}) answered with an error from its search backend instead of results.`, {
|
|
1870
|
+
reason: EFTS_DEGRADED_REASON,
|
|
1871
|
+
retryable: false,
|
|
1872
|
+
...upstream,
|
|
1873
|
+
recovery: {
|
|
1874
|
+
hint: `EDGAR full-text search (${host}) refused to run this search, and resending it unchanged fails the same way — upstreamErrorMessage gives the reason to change.`,
|
|
1875
|
+
},
|
|
1876
|
+
});
|
|
1877
|
+
}
|
|
1878
|
+
const { close, onAttempt } = ladderEndText(end);
|
|
1879
|
+
return serviceUnavailable(`EDGAR full-text search (${host}) answered ${onAttempt} with a failure of its search backend instead of results${close}`, {
|
|
1880
|
+
reason: EFTS_DEGRADED_REASON,
|
|
1881
|
+
retryable: true,
|
|
1882
|
+
...upstream,
|
|
1883
|
+
recovery: {
|
|
1884
|
+
hint: `EDGAR full-text search (${host}) answered, but its search backend could not run the search — a temporary failure on SEC's side, not a problem with the request. Retry the same call in a minute.`,
|
|
1885
|
+
},
|
|
1886
|
+
});
|
|
1887
|
+
}
|
|
1665
1888
|
/** An identifier-shaped transport code: `ECONNRESET`, `EAI_AGAIN`, `UND_ERR_CONNECT_TIMEOUT`. */
|
|
1666
1889
|
const TRANSPORT_CODE = /^[A-Za-z][A-Za-z0-9_]*$/;
|
|
1667
1890
|
/**
|