@cyanheads/secedgar-mcp-server 0.15.13 → 0.15.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/AGENTS.md +8 -6
  2. package/CLAUDE.md +8 -6
  3. package/Dockerfile +3 -3
  4. package/README.md +8 -5
  5. package/changelog/0.15.x/0.15.14.md +31 -0
  6. package/changelog/0.15.x/0.15.15.md +24 -0
  7. package/dist/index.js +7 -5
  8. package/dist/index.js.map +1 -1
  9. package/dist/mcp-server/tools/definitions/company-search.tool.d.ts +2 -2
  10. package/dist/mcp-server/tools/definitions/company-search.tool.d.ts.map +1 -1
  11. package/dist/mcp-server/tools/definitions/company-search.tool.js +8 -10
  12. package/dist/mcp-server/tools/definitions/company-search.tool.js.map +1 -1
  13. package/dist/mcp-server/tools/definitions/compare-companies.tool.d.ts +2 -2
  14. package/dist/mcp-server/tools/definitions/compare-companies.tool.d.ts.map +1 -1
  15. package/dist/mcp-server/tools/definitions/compare-companies.tool.js +6 -4
  16. package/dist/mcp-server/tools/definitions/compare-companies.tool.js.map +1 -1
  17. package/dist/mcp-server/tools/definitions/dataframe-describe.tool.d.ts +1 -1
  18. package/dist/mcp-server/tools/definitions/dataframe-describe.tool.d.ts.map +1 -1
  19. package/dist/mcp-server/tools/definitions/dataframe-describe.tool.js +1 -1
  20. package/dist/mcp-server/tools/definitions/dataframe-describe.tool.js.map +1 -1
  21. package/dist/mcp-server/tools/definitions/dataframe-drop.tool.d.ts +1 -1
  22. package/dist/mcp-server/tools/definitions/dataframe-drop.tool.d.ts.map +1 -1
  23. package/dist/mcp-server/tools/definitions/dataframe-drop.tool.js +1 -1
  24. package/dist/mcp-server/tools/definitions/dataframe-drop.tool.js.map +1 -1
  25. package/dist/mcp-server/tools/definitions/dataframe-query.tool.d.ts +1 -1
  26. package/dist/mcp-server/tools/definitions/dataframe-query.tool.d.ts.map +1 -1
  27. package/dist/mcp-server/tools/definitions/dataframe-query.tool.js +1 -1
  28. package/dist/mcp-server/tools/definitions/dataframe-query.tool.js.map +1 -1
  29. package/dist/mcp-server/tools/definitions/fetch-frames.tool.d.ts +2 -2
  30. package/dist/mcp-server/tools/definitions/fetch-frames.tool.d.ts.map +1 -1
  31. package/dist/mcp-server/tools/definitions/fetch-frames.tool.js +6 -4
  32. package/dist/mcp-server/tools/definitions/fetch-frames.tool.js.map +1 -1
  33. package/dist/mcp-server/tools/definitions/find-holders.tool.d.ts +2 -2
  34. package/dist/mcp-server/tools/definitions/find-holders.tool.d.ts.map +1 -1
  35. package/dist/mcp-server/tools/definitions/find-holders.tool.js +35 -23
  36. package/dist/mcp-server/tools/definitions/find-holders.tool.js.map +1 -1
  37. package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.d.ts +15 -5
  38. package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.d.ts.map +1 -1
  39. package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.js +92 -35
  40. package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.js.map +1 -1
  41. package/dist/mcp-server/tools/definitions/get-filing.tool.d.ts +2 -2
  42. package/dist/mcp-server/tools/definitions/get-filing.tool.d.ts.map +1 -1
  43. package/dist/mcp-server/tools/definitions/get-filing.tool.js +59 -33
  44. package/dist/mcp-server/tools/definitions/get-filing.tool.js.map +1 -1
  45. package/dist/mcp-server/tools/definitions/get-financials.tool.d.ts +2 -2
  46. package/dist/mcp-server/tools/definitions/get-financials.tool.d.ts.map +1 -1
  47. package/dist/mcp-server/tools/definitions/get-financials.tool.js +11 -8
  48. package/dist/mcp-server/tools/definitions/get-financials.tool.js.map +1 -1
  49. package/dist/mcp-server/tools/definitions/get-fund-holdings.tool.d.ts +2 -2
  50. package/dist/mcp-server/tools/definitions/get-fund-holdings.tool.d.ts.map +1 -1
  51. package/dist/mcp-server/tools/definitions/get-fund-holdings.tool.js +27 -27
  52. package/dist/mcp-server/tools/definitions/get-fund-holdings.tool.js.map +1 -1
  53. package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.d.ts +2 -2
  54. package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.d.ts.map +1 -1
  55. package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.js +12 -16
  56. package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.js.map +1 -1
  57. package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.d.ts +21 -2
  58. package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.d.ts.map +1 -1
  59. package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.js +153 -37
  60. package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.js.map +1 -1
  61. package/dist/mcp-server/tools/definitions/get-material-events.tool.d.ts +2 -2
  62. package/dist/mcp-server/tools/definitions/get-material-events.tool.d.ts.map +1 -1
  63. package/dist/mcp-server/tools/definitions/get-material-events.tool.js +8 -10
  64. package/dist/mcp-server/tools/definitions/get-material-events.tool.js.map +1 -1
  65. package/dist/mcp-server/tools/definitions/get-snapshot.tool.d.ts +2 -2
  66. package/dist/mcp-server/tools/definitions/get-snapshot.tool.d.ts.map +1 -1
  67. package/dist/mcp-server/tools/definitions/get-snapshot.tool.js +6 -4
  68. package/dist/mcp-server/tools/definitions/get-snapshot.tool.js.map +1 -1
  69. package/dist/mcp-server/tools/definitions/index.d.ts +597 -572
  70. package/dist/mcp-server/tools/definitions/index.d.ts.map +1 -1
  71. package/dist/mcp-server/tools/definitions/search-filings.tool.d.ts +3 -8
  72. package/dist/mcp-server/tools/definitions/search-filings.tool.d.ts.map +1 -1
  73. package/dist/mcp-server/tools/definitions/search-filings.tool.js +117 -59
  74. package/dist/mcp-server/tools/definitions/search-filings.tool.js.map +1 -1
  75. package/dist/mcp-server/tools/fixed-point.d.ts +7 -0
  76. package/dist/mcp-server/tools/fixed-point.d.ts.map +1 -0
  77. package/dist/mcp-server/tools/fixed-point.js +28 -0
  78. package/dist/mcp-server/tools/fixed-point.js.map +1 -0
  79. package/dist/services/edgar/call-scope.d.ts +41 -0
  80. package/dist/services/edgar/call-scope.d.ts.map +1 -0
  81. package/dist/services/edgar/call-scope.js +53 -0
  82. package/dist/services/edgar/call-scope.js.map +1 -0
  83. package/dist/services/edgar/concept-series.d.ts.map +1 -1
  84. package/dist/services/edgar/concept-series.js +5 -1
  85. package/dist/services/edgar/concept-series.js.map +1 -1
  86. package/dist/services/edgar/edgar-api-service.d.ts +80 -50
  87. package/dist/services/edgar/edgar-api-service.d.ts.map +1 -1
  88. package/dist/services/edgar/edgar-api-service.js +483 -211
  89. package/dist/services/edgar/edgar-api-service.js.map +1 -1
  90. package/dist/services/edgar/filing-to-text.d.ts +53 -14
  91. package/dist/services/edgar/filing-to-text.d.ts.map +1 -1
  92. package/dist/services/edgar/filing-to-text.js +181 -33
  93. package/dist/services/edgar/filing-to-text.js.map +1 -1
  94. package/dist/services/edgar/nport-parser.js +1 -1
  95. package/dist/services/edgar/nport-parser.js.map +1 -1
  96. package/dist/services/edgar/ownership-parser.d.ts +2 -0
  97. package/dist/services/edgar/ownership-parser.d.ts.map +1 -1
  98. package/dist/services/edgar/ownership-parser.js +4 -0
  99. package/dist/services/edgar/ownership-parser.js.map +1 -1
  100. package/dist/services/edgar/same-cik.d.ts +7 -0
  101. package/dist/services/edgar/same-cik.d.ts.map +1 -0
  102. package/dist/services/edgar/same-cik.js +9 -0
  103. package/dist/services/edgar/same-cik.js.map +1 -0
  104. package/dist/services/edgar/submissions-archive.d.ts +6 -3
  105. package/dist/services/edgar/submissions-archive.d.ts.map +1 -1
  106. package/dist/services/edgar/submissions-archive.js +6 -3
  107. package/dist/services/edgar/submissions-archive.js.map +1 -1
  108. package/package.json +6 -6
  109. package/server.json +3 -3
@@ -9,6 +9,7 @@ import { createPacer, httpErrorFromResponse, logger, requestContextService, with
9
9
  import { parseDocument } from 'htmlparser2';
10
10
  import { getServerConfig } from '../../config/server-config.js';
11
11
  import { getEdgarMirror } from './mirror/index.js';
12
+ import { CALL_FAILURE_BUDGET_MS, CallScope } from './call-scope.js';
12
13
  import formerNamesData from './data/former-names.json' with { type: 'json' };
13
14
  import { parseFilingHeaders, parseSubmissionHeader, } from './filing-headers.js';
14
15
  import { trigramSimilarity } from './trigram-similarity.js';
@@ -19,8 +20,8 @@ const BASE_BACKOFF_MS = 1000;
19
20
  * Statuses worth a retry. 429 is deliberately absent: SEC holds a rate-limit block
20
21
  * until the request rate has stayed under the threshold for ten minutes, so the
21
22
  * ~3s retry budget cannot outlast it and every retry restarts the clock the caller
22
- * is waiting on (#112). An attempt that gets no HTTP response at all retries on the
23
- * same ladder (#173).
23
+ * is waiting on (#112). An attempt that gets no HTTP response at all (#173), and
24
+ * one whose body fails to read after its headers (#181), retry on the same ladder.
24
25
  */
25
26
  const RETRYABLE_STATUSES = new Set([500, 502, 503, 504]);
26
27
  /**
@@ -28,19 +29,31 @@ const RETRYABLE_STATUSES = new Set([500, 502, 503, 504]);
28
29
  * it is aborted and retried as a no-response failure (#173). Bun applies no connect
29
30
  * timeout of its own — an unanswered connect waits out the OS TCP timeout, 75 s on
30
31
  * macOS — and neither runtime bounds a server that accepts and never answers, so one
31
- * attempt could outlast the MCP client's 60 s request timeout alone. Three capped
32
- * attempts plus the 1 s and 2 s backoffs come to 48 s. The clock starts once the
33
- * pacer dispatches the request, so queue time never counts, and stops at headers,
34
- * so a multi-megabyte body is never cut off. SEC's slowest reads sampled send
35
- * headers in under 2 s.
32
+ * attempt could outlast the MCP client's 60 s request timeout alone. Each wait is
33
+ * also clamped to what is left of the call's {@link CALL_FAILURE_BUDGET_MS}, which
34
+ * bounds a whole tool call's failure time (#179). The clock starts once the
35
+ * pacer dispatches the request, so queue time never counts, and stops at headers;
36
+ * the body is bounded only by {@link BODY_IDLE_TIMEOUT_MS}, so a multi-megabyte body
37
+ * that keeps arriving is never cut off. SEC's slowest reads sampled send headers in
38
+ * under 2 s.
36
39
  */
37
40
  const ATTEMPT_HEADERS_TIMEOUT_MS = 15_000;
38
41
  /**
39
- * `data.reason` once every attempt got no HTTP response (#173). Declared retryable
40
- * on every tool that reaches SEC; set on `data.retryable` here for the same reason
41
- * as `rate_limited` (#122).
42
+ * How long a response body may go without a byte before the read is abandoned and
43
+ * the attempt retried as a body-read failure (#181). Measured between reads, never
44
+ * across the whole body, so a body still arriving, however slowly, is never cut.
45
+ * Without it a stalled body holds the call until undici's 300 s body timeout on
46
+ * Node, and indefinitely on Bun.
47
+ */
48
+ const BODY_IDLE_TIMEOUT_MS = 15_000;
49
+ /**
50
+ * `data.reason` once SEC gave no complete, readable answer: the last attempt got no
51
+ * HTTP response (#173) or its body failed to read, or a 200 body was not JSON
52
+ * (#181). Declared retryable on every tool that reaches SEC; set on `data.retryable`
53
+ * here for the same reason as `rate_limited` (#122).
42
54
  */
43
55
  const SEC_UNREACHABLE_REASON = 'sec_unreachable';
56
+ const BUDGET_SPENT = { kind: 'budget_spent' };
44
57
  /**
45
58
  * `data.reason` on every SEC rate-limit failure — the upstream 429 and a call
46
59
  * refused locally while the cool-down after one runs — so a caller branches the
@@ -221,31 +234,30 @@ class EdgarApiService {
221
234
  dispose() {
222
235
  this.pacer.dispose();
223
236
  }
224
- /** Fetch and parse JSON, throwing on non-OK responses. */
225
- async fetchJson(url) {
226
- const response = await this.rawFetch(url, true);
227
- if (response.status === 404) {
237
+ /** Fetch and parse JSON, throwing on non-OK responses and on a body that is not JSON. */
238
+ async fetchJson(url, scope) {
239
+ const text = await this.rawFetch(url, true, scope);
240
+ if (text === null) {
228
241
  throw notFound(`SEC EDGAR API returned 404 for ${url}`, { url, status: 404 });
229
242
  }
230
- return response.json();
243
+ return parseJson(url, text);
231
244
  }
232
245
  /** Fetch JSON, returning `null` on 404 and throwing on other non-OK responses. */
233
- async tryFetchJson(url) {
234
- const response = await this.rawFetch(url, true);
235
- return response.status === 404 ? null : response.json();
246
+ async tryFetchJson(url, scope) {
247
+ const text = await this.rawFetch(url, true, scope);
248
+ return text === null ? null : parseJson(url, text);
236
249
  }
237
250
  /** Fetch raw text content (HTML filing documents). */
238
- async fetchText(url) {
239
- const response = await this.rawFetch(url, false);
240
- if (response.status === 404) {
251
+ async fetchText(url, scope) {
252
+ const text = await this.rawFetch(url, false, scope);
253
+ if (text === null) {
241
254
  throw notFound(`SEC EDGAR returned 404 for ${url}`, { url, status: 404 });
242
255
  }
243
- return response.text();
256
+ return text;
244
257
  }
245
258
  /** Fetch raw text content, returning `null` on 404 and throwing on other failures. */
246
- async tryFetchText(url) {
247
- const response = await this.rawFetch(url, false);
248
- return response.status === 404 ? null : response.text();
259
+ tryFetchText(url, scope) {
260
+ return this.rawFetch(url, false, scope);
249
261
  }
250
262
  // --- CIK Resolution ---
251
263
  /**
@@ -258,8 +270,8 @@ class EdgarApiService {
258
270
  * On no-result name search, the returned empty array carries `suggestions` on
259
271
  * the thrown error at the handler layer — call `suggestCompanies` there.
260
272
  */
261
- async resolveCik(query) {
262
- const cache = await this.getTickerCache();
273
+ async resolveCik(query, scope) {
274
+ const cache = await this.getTickerCache(scope);
263
275
  const trimmed = query.trim();
264
276
  // Numeric → CIK
265
277
  if (/^\d+$/.test(trimmed)) {
@@ -347,10 +359,10 @@ class EdgarApiService {
347
359
  * Distinct from `searchFilings`: same URL path, different query param (`keysTyped`)
348
360
  * and response shape (`_source.entity`, not the filing-document `_source`).
349
361
  */
350
- async resolveEntityByName(name) {
362
+ async resolveEntityByName(name, scope) {
351
363
  const url = new URL('https://efts.sec.gov/LATEST/search-index');
352
364
  url.searchParams.set('keysTyped', name);
353
- const response = await this.fetchJson(url.toString());
365
+ const response = await this.fetchJson(url.toString(), scope);
354
366
  const hits = response?.hits?.hits;
355
367
  if (!Array.isArray(hits))
356
368
  return [];
@@ -367,8 +379,8 @@ class EdgarApiService {
367
379
  return matches;
368
380
  }
369
381
  /** Reverse lookup: CIK → ticker symbol. */
370
- async cikToTicker(cik) {
371
- const cache = await this.getTickerCache();
382
+ async cikToTicker(cik, scope) {
383
+ const cache = await this.getTickerCache(scope);
372
384
  return cache.byCik.get(cik.padStart(10, '0'))?.ticker;
373
385
  }
374
386
  /**
@@ -376,8 +388,8 @@ class EdgarApiService {
376
388
  * company_tickers_mf.json, which lists a series only once one of its share classes has a
377
389
  * ticker — a series with no listed class resolves to `undefined` rather than a wrong trust.
378
390
  */
379
- async resolveFundSeries(seriesId) {
380
- const cache = await this.getTickerCache();
391
+ async resolveFundSeries(seriesId, scope) {
392
+ const cache = await this.getTickerCache(scope);
381
393
  return cache.bySeriesId.get(seriesId.toUpperCase());
382
394
  }
383
395
  /**
@@ -387,8 +399,8 @@ class EdgarApiService {
387
399
  * class only — a trust whose series carry no ticker returns an empty list, which is not
388
400
  * the same as having no series.
389
401
  */
390
- async listFundSeries(cik) {
391
- const cache = await this.getTickerCache();
402
+ async listFundSeries(cik, scope) {
403
+ const cache = await this.getTickerCache(scope);
392
404
  const padded = cik.padStart(10, '0');
393
405
  const out = [];
394
406
  for (const [seriesId, match] of cache.bySeriesId) {
@@ -406,7 +418,7 @@ class EdgarApiService {
406
418
  * response also names the registrant, so a bare series ID resolves without a second call.
407
419
  * Entries come back newest-filed first and include amendments of the form.
408
420
  */
409
- async getFundSeriesFilings(seriesId, formType, count) {
421
+ async getFundSeriesFilings(seriesId, formType, count, scope) {
410
422
  const url = new URL('https://www.sec.gov/cgi-bin/browse-edgar');
411
423
  url.searchParams.set('action', 'getcompany');
412
424
  url.searchParams.set('CIK', seriesId);
@@ -414,11 +426,11 @@ class EdgarApiService {
414
426
  url.searchParams.set('owner', 'include');
415
427
  url.searchParams.set('count', String(count));
416
428
  url.searchParams.set('output', 'atom');
417
- return parseSeriesFilingFeed(await this.fetchText(url.toString()));
429
+ return parseSeriesFilingFeed(await this.fetchText(url.toString(), scope));
418
430
  }
419
431
  /** Return the current in-memory entry list (used by the handler for trigram suggestions). */
420
- async getAllEntries() {
421
- const cache = await this.getTickerCache();
432
+ async getAllEntries(scope) {
433
+ const cache = await this.getTickerCache(scope);
422
434
  return cache.allEntries;
423
435
  }
424
436
  // --- SEC API Methods ---
@@ -427,14 +439,15 @@ class EdgarApiService {
427
439
  * the last year or 1,000 filings, whichever holds more — + the `files[]`
428
440
  * archive-page manifest). Cached per CIK within
429
441
  * the ticker-cache TTL — the doc is large and re-read on every archive-paging
430
- * scan (#78). A 404 throws (uncached) so a bad CIK still surfaces.
442
+ * scan (#78). A 404 throws (uncached) so a bad CIK still surfaces. Like every
443
+ * cache here, written only by a completed read — a cancelled one caches nothing.
431
444
  */
432
- async getSubmissions(cik) {
445
+ async getSubmissions(cik, scope) {
433
446
  const padded = cik.padStart(10, '0');
434
447
  const cached = this.submissionsCache.get(padded);
435
448
  if (cached && this.isFresh(cached.at))
436
449
  return cached.data;
437
- const data = await this.fetchJson(`https://data.sec.gov/submissions/CIK${padded}.json`);
450
+ const data = await this.fetchJson(`https://data.sec.gov/submissions/CIK${padded}.json`, scope);
438
451
  this.submissionsCache.set(padded, { at: Date.now(), data });
439
452
  return data;
440
453
  }
@@ -445,11 +458,11 @@ class EdgarApiService {
445
458
  * field-compatible with `FilingsRecent`. Cached per page within the ticker-cache
446
459
  * TTL (pages are large and effectively immutable once archived).
447
460
  */
448
- async fetchArchivePage(name) {
461
+ async fetchArchivePage(name, scope) {
449
462
  const cached = this.archivePageCache.get(name);
450
463
  if (cached && this.isFresh(cached.at))
451
464
  return cached.data;
452
- const data = await this.fetchJson(`https://data.sec.gov/submissions/${name}`);
465
+ const data = await this.fetchJson(`https://data.sec.gov/submissions/${name}`, scope);
453
466
  this.archivePageCache.set(name, { at: Date.now(), data });
454
467
  return data;
455
468
  }
@@ -462,12 +475,12 @@ class EdgarApiService {
462
475
  * bound how many quarters they scan and filter client-side (#77). Cached per
463
476
  * quarter within the ticker-cache TTL — an archived quarter is immutable.
464
477
  */
465
- async fetchFullIndexQuarter(year, quarter) {
478
+ async fetchFullIndexQuarter(year, quarter, scope) {
466
479
  const key = `${year}Q${quarter}`;
467
480
  const cached = this.fullIndexCache.get(key);
468
481
  if (cached && this.isFresh(cached.at))
469
482
  return cached.data;
470
- const text = await this.fetchText(`https://www.sec.gov/Archives/edgar/full-index/${year}/QTR${quarter}/master.idx`);
483
+ const text = await this.fetchText(`https://www.sec.gov/Archives/edgar/full-index/${year}/QTR${quarter}/master.idx`, scope);
471
484
  const data = parseMasterIndex(text);
472
485
  this.fullIndexCache.set(key, { at: Date.now(), data });
473
486
  return data;
@@ -476,7 +489,7 @@ class EdgarApiService {
476
489
  isFresh(at) {
477
490
  return Date.now() - at < getServerConfig().tickerCacheTtl * 1000;
478
491
  }
479
- async searchFilings(params) {
492
+ async searchFilings(params, scope) {
480
493
  const url = new URL('https://efts.sec.gov/LATEST/search-index');
481
494
  // `q` is optional — EFTS honors `ciks` for pure entity scope with no
482
495
  // full-text query, so a bare cik:/ticker: search sends no `q`.
@@ -495,7 +508,7 @@ class EdgarApiService {
495
508
  }
496
509
  url.searchParams.set('from', String(params.from ?? 0));
497
510
  url.searchParams.set('size', String(params.size ?? 20));
498
- const response = await this.fetchJson(url.toString());
511
+ const response = await this.fetchJson(url.toString(), scope);
499
512
  // Shape guard: EFTS can return a 2xx whose body omits `hits.total` (degraded
500
513
  // payload, or a rejected request echoed as `{ error: ... }` with 200). A genuine
501
514
  // zero-hit response is still well-formed (`hits.total.value: 0`, empty
@@ -517,8 +530,8 @@ class EdgarApiService {
517
530
  * Resolve likely company CIKs for a filing accession number using SEC search metadata.
518
531
  * Returns zero or more padded 10-digit CIKs in SEC-provided order.
519
532
  */
520
- async findFilingCiks(accessionNumber) {
521
- const response = await this.searchFilings({ query: accessionNumber, size: 10 });
533
+ async findFilingCiks(accessionNumber, scope) {
534
+ const response = await this.searchFilings({ query: accessionNumber, size: 10 }, scope);
522
535
  const normalizedAccession = accessionNumber.replace(/[^0-9]/g, '');
523
536
  const ciks = new Set();
524
537
  for (const hit of response.hits.hits) {
@@ -532,8 +545,8 @@ class EdgarApiService {
532
545
  return [...ciks];
533
546
  }
534
547
  /** Fetch a filing's document index. Returns `null` if the filing does not exist. */
535
- tryGetFilingIndex(cik, accessionNumber) {
536
- return this.tryFetchJson(filingArchiveUrl(cik, accessionNumber, 'index.json'));
548
+ tryGetFilingIndex(cik, accessionNumber, scope) {
549
+ return this.tryFetchJson(filingArchiveUrl(cik, accessionNumber, 'index.json'), scope);
537
550
  }
538
551
  /**
539
552
  * Fetch the SEC submission header page (`<accession>-index-headers.html`) and parse
@@ -541,8 +554,8 @@ class EdgarApiService {
541
554
  * and period. Returns `null` if the page is absent. The page exposes canonical SEC
542
555
  * document TYPE values (e.g. "EX-21.1") that the directory listing JSON does not.
543
556
  */
544
- async tryGetFilingHeaders(cik, accessionNumber) {
545
- const text = await this.tryFetchText(filingArchiveUrl(cik, accessionNumber, `${accessionNumber}-index-headers.html`));
557
+ async tryGetFilingHeaders(cik, accessionNumber, scope) {
558
+ const text = await this.tryFetchText(filingArchiveUrl(cik, accessionNumber, `${accessionNumber}-index-headers.html`), scope);
546
559
  return text ? parseFilingHeaders(text) : null;
547
560
  }
548
561
  /**
@@ -551,18 +564,18 @@ class EdgarApiService {
551
564
  * fallback for filings whose index-headers page is missing — common among filings
552
565
  * made before 2014; it lists no documents.
553
566
  */
554
- async tryGetSubmissionHeader(cik, accessionNumber) {
555
- const text = await this.tryFetchText(filingArchiveUrl(cik, accessionNumber, `${accessionNumber}.hdr.sgml`));
567
+ async tryGetSubmissionHeader(cik, accessionNumber, scope) {
568
+ const text = await this.tryFetchText(filingArchiveUrl(cik, accessionNumber, `${accessionNumber}.hdr.sgml`), scope);
556
569
  return text ? parseSubmissionHeader(text) : null;
557
570
  }
558
- getFilingDocument(cik, accessionNumber, document) {
559
- return this.fetchText(filingArchiveUrl(cik, accessionNumber, document));
571
+ getFilingDocument(cik, accessionNumber, document, scope) {
572
+ return this.fetchText(filingArchiveUrl(cik, accessionNumber, document), scope);
560
573
  }
561
574
  /**
562
575
  * Fetch a filing document, returning `null` when the archive path exists but this document does not.
563
576
  */
564
- tryGetFilingDocument(cik, accessionNumber, document) {
565
- return this.tryFetchText(filingArchiveUrl(cik, accessionNumber, document));
577
+ tryGetFilingDocument(cik, accessionNumber, document, scope) {
578
+ return this.tryFetchText(filingArchiveUrl(cik, accessionNumber, document), scope);
566
579
  }
567
580
  /**
568
581
  * Fetch the leading bytes of a filing document, stopping as soon as `stopAt` appears in
@@ -578,45 +591,20 @@ class EdgarApiService {
578
591
  * appears 1.5 KB in still costs 256 KB, and a `maxBytes` below that costs the same. Treat
579
592
  * both as "one chunk, not the file" rather than as a byte budget; `maxBytes` binds only on
580
593
  * a document long enough to arrive in several chunks, which is what stops a document
581
- * missing `stopAt` entirely from being read to the end. A runtime that hands back no
582
- * readable body (a stubbed fetch, for one) falls through to reading it whole, which costs
583
- * bandwidth but returns the same text.
594
+ * missing `stopAt` entirely from being read to the end. The head is read inside the
595
+ * retry ladder like any other body, so a read that breaks off before the cutoff is
596
+ * retried; stopping at the cutoff is not a failure (#181).
584
597
  */
585
- async tryGetFilingDocumentHead(cik, accessionNumber, document, options) {
586
- const response = await this.rawFetch(filingArchiveUrl(cik, accessionNumber, document), false);
587
- if (response.status === 404)
588
- return null;
589
- if (!response.body)
590
- return response.text();
591
- const reader = response.body.getReader();
592
- const decoder = new TextDecoder();
593
- let text = '';
594
- let read = 0;
595
- try {
596
- while (read < options.maxBytes) {
597
- const { done, value } = await reader.read();
598
- if (done)
599
- break;
600
- read += value.byteLength;
601
- text += decoder.decode(value, { stream: true });
602
- if (text.includes(options.stopAt))
603
- break;
604
- }
605
- }
606
- finally {
607
- await reader.cancel().catch(() => {
608
- /* the body is being discarded; a cancel race has nothing left to report */
609
- });
610
- }
611
- return text;
598
+ tryGetFilingDocumentHead(cik, accessionNumber, document, options, scope) {
599
+ return this.rawFetch(filingArchiveUrl(cik, accessionNumber, document), false, scope, options);
612
600
  }
613
601
  /**
614
602
  * Search EDGAR submissions for recent filings of specified form types, newest first.
615
603
  * Returns up to `limit` matches from the submissions recent-filings window. Each carries
616
604
  * `reportDate` (the period-of-report end date) for callers that target a specific period.
617
605
  */
618
- async getRecentFilingsByForm(cik, formTypes, limit) {
619
- const submissions = await this.getSubmissions(cik);
606
+ async getRecentFilingsByForm(cik, formTypes, limit, scope) {
607
+ const submissions = await this.getSubmissions(cik, scope);
620
608
  const recent = submissions.filings.recent;
621
609
  const results = [];
622
610
  for (let i = 0; i < recent.form.length && results.length < limit; i++) {
@@ -640,9 +628,9 @@ class EdgarApiService {
640
628
  * lookup; the live API is the fallback. A concept unit that arrives as
641
629
  * anything but an array is dropped (#141).
642
630
  */
643
- async tryGetCompanyFacts(cik) {
631
+ async tryGetCompanyFacts(cik, scope) {
644
632
  const padded = cik.padStart(10, '0');
645
- const facts = await this.mirrorOrLive((m) => m.companyFactsReady(), (m) => m.getCompanyFacts(cik), () => this.tryFetchJson(`https://data.sec.gov/api/xbrl/companyfacts/CIK${padded}.json`));
633
+ const facts = await this.mirrorOrLive((m) => m.companyFactsReady(), (m) => m.getCompanyFacts(cik), () => this.tryFetchJson(`https://data.sec.gov/api/xbrl/companyfacts/CIK${padded}.json`, scope));
646
634
  for (const namespace of Object.values(facts?.facts ?? {})) {
647
635
  for (const concept of Object.values(namespace))
648
636
  concept.units = arrayUnits(concept.units);
@@ -656,9 +644,9 @@ class EdgarApiService {
656
644
  * A payload whose every unit arrives malformed comes back with no units — the
657
645
  * tag is named but carries nothing (#141).
658
646
  */
659
- async tryGetCompanyConcept(cik, taxonomy, tag) {
647
+ async tryGetCompanyConcept(cik, taxonomy, tag, scope) {
660
648
  const padded = cik.padStart(10, '0');
661
- const concept = await this.mirrorOrLive((m) => m.companyFactsReady(), (m) => m.getCompanyConcept(cik, taxonomy, tag), () => this.tryFetchJson(`https://data.sec.gov/api/xbrl/companyconcept/CIK${padded}/${taxonomy}/${tag}.json`));
649
+ const concept = await this.mirrorOrLive((m) => m.companyFactsReady(), (m) => m.getCompanyConcept(cik, taxonomy, tag), () => this.tryFetchJson(`https://data.sec.gov/api/xbrl/companyconcept/CIK${padded}/${taxonomy}/${tag}.json`, scope));
662
650
  if (concept)
663
651
  concept.units = arrayUnits(concept.units);
664
652
  return concept;
@@ -671,8 +659,8 @@ class EdgarApiService {
671
659
  * fall back to the live API until the mirror is complete. The live API is also
672
660
  * the fallback on a genuine miss.
673
661
  */
674
- tryGetFrames(taxonomy, tag, unit, period) {
675
- return this.mirrorOrLive((m) => m.companyFactsComplete(), (m) => m.getFrames(taxonomy, tag, unit, period), () => this.tryFetchJson(`https://data.sec.gov/api/xbrl/frames/${taxonomy}/${tag}/${unit}/${period}.json`));
662
+ tryGetFrames(taxonomy, tag, unit, period, scope) {
663
+ return this.mirrorOrLive((m) => m.companyFactsComplete(), (m) => m.getFrames(taxonomy, tag, unit, period), () => this.tryFetchJson(`https://data.sec.gov/api/xbrl/frames/${taxonomy}/${tag}/${unit}/${period}.json`, scope));
676
664
  }
677
665
  /**
678
666
  * Route a company-facts query through the local mirror when ready, with live-API
@@ -705,63 +693,115 @@ class EdgarApiService {
705
693
  }
706
694
  // --- Internals ---
707
695
  /**
708
- * Paced fetch with retry/backoff. Returns the response on 2xx or 404. A 5xx, and
709
- * an attempt that got no HTTP response, retry up to `MAX_RETRIES` attempts; then
710
- * the 5xx throws its status-classified `McpError` and the no-response case throws
711
- * `sec_unreachable`. Other non-OK statuses throw at once, as does the local
712
- * refusal while SEC's rate-limit block is active, and the pacer's
696
+ * Paced fetch with retry/backoff. Returns the body's text on 2xx — only its head
697
+ * when `cutoff` is given — and `null` on 404, whose body is left unread. The body
698
+ * is read inside the ladder (#181): a 5xx, an attempt that got no HTTP response,
699
+ * and a 2xx whose body failed to read all retry up to `MAX_RETRIES` attempts, and
700
+ * then the last attempt decides — a 5xx throws its status-classified `McpError`,
701
+ * the other two `sec_unreachable`. Other non-OK statuses throw at once, as does
702
+ * the local refusal while SEC's rate-limit block is active, and the pacer's
713
703
  * `RequestCancelled` — those are `McpError`s from `send()`, never retried.
704
+ *
705
+ * Every failed attempt and every backoff is charged to the call's failure budget,
706
+ * and the ladder ends early once the budget cannot cover another backoff and
707
+ * attempt (#179). Once the caller's signal has fired, whatever rejects is rethrown
708
+ * untouched — a queued request leaves the pacer unsent, and an attempt in flight,
709
+ * a body read, or a backoff sleep ends at once — never retried or classified.
714
710
  */
715
- async rawFetch(url, acceptJson) {
711
+ async rawFetch(url, acceptJson, scope, cutoff) {
712
+ scope.signal.throwIfAborted();
716
713
  const headers = { 'User-Agent': getServerConfig().userAgent };
717
714
  if (acceptJson)
718
715
  headers.Accept = 'application/json';
719
- for (let attempt = 0;; attempt++) {
720
- const outcome = await this.send(url, headers);
721
- if (outcome.kind === 'answered' && (outcome.response.ok || outcome.response.status === 404)) {
722
- return outcome.response;
716
+ /** The previous attempt's failure, which a request the budget stops before its next attempt ends with. */
717
+ let failed;
718
+ for (let attempt = 1;; attempt++) {
719
+ const outcome = scope.remainingMs > 0 ? await this.send(url, headers, scope) : BUDGET_SPENT;
720
+ // An answer that lands as the caller cancels is not read.
721
+ scope.signal.throwIfAborted();
722
+ if (outcome.kind === 'budget_spent') {
723
+ // Spent before this attempt could go out — by an earlier request, or by a
724
+ // concurrent one while this retry waited for its slot.
725
+ if (failed === undefined)
726
+ throw secUnreachable(url, outcome);
727
+ throw await this.ladderFailure(url, failed, { attempt: attempt - 1, budgetStopped: true });
723
728
  }
724
- const transient = outcome.kind === 'unanswered' || RETRYABLE_STATUSES.has(outcome.response.status);
725
- if (transient && attempt < MAX_RETRIES - 1) {
726
- await sleep(BASE_BACKOFF_MS * 2 ** attempt);
727
- continue;
728
- }
729
- if (outcome.kind === 'unanswered')
730
- throw secUnreachable(url, outcome);
731
- const { response } = outcome;
732
- const data = { url };
733
- if (response.status === 403) {
734
- const host = new URL(url).hostname;
735
- data.recovery = {
736
- hint: `${host} may be blocking requests. Check EDGAR_USER_AGENT format ("AppName contact@email.com") or retry later.`,
737
- };
729
+ let last = outcome;
730
+ if (outcome.kind === 'answered') {
731
+ const { response } = outcome;
732
+ if (response.status === 404)
733
+ return null;
734
+ if (response.ok) {
735
+ const body = await readBody(response, scope, cutoff);
736
+ if (body.kind === 'read')
737
+ return body.text;
738
+ last = body;
739
+ }
738
740
  }
739
- else if (response.status === 429) {
740
- // send() closed the gate on this response, so the wait is the full cool-down.
741
- const retryAfter = this.cooldownSecondsRemaining();
742
- data.reason = RATE_LIMITED_REASON;
743
- data.retryable = true;
744
- data.retryAfter = retryAfter;
745
- data.recovery = { hint: rateLimitHint(retryAfter) };
741
+ const transient = last.kind !== 'answered' || RETRYABLE_STATUSES.has(last.response.status);
742
+ if (!transient)
743
+ throw await this.ladderFailure(url, last, { attempt, budgetStopped: false });
744
+ scope.charge(outcome.waitedMs + (last.kind === 'body_failed' ? last.waitedMs : 0));
745
+ const backoffMs = BASE_BACKOFF_MS * 2 ** (attempt - 1);
746
+ if (attempt < MAX_RETRIES && scope.remainingMs > backoffMs) {
747
+ scope.charge(backoffMs);
748
+ await sleep(backoffMs, scope.signal);
749
+ failed = last;
750
+ continue;
746
751
  }
747
- throw await httpErrorFromResponse(response, {
748
- service: 'SEC EDGAR',
749
- data,
750
- // SEC answers a rate-limit block with a full HTML page. The recovery hint
751
- // above carries everything actionable, so the markup is not forwarded (#112).
752
- captureBody: response.status !== 429,
752
+ throw await this.ladderFailure(url, last, {
753
+ attempt,
754
+ budgetStopped: attempt < MAX_RETRIES || cutByBudget(last),
753
755
  });
754
756
  }
755
757
  }
756
758
  /**
757
- * One attempt through the block gate and the pacer. A 429 closes the gate
758
- * before the response is handed back, so no call admitted after it sends. A
759
- * `fetch` rejection comes back as an unanswered attempt, and so does a probe
760
- * this call waited on that got no HTTP response; the gate's refusal and the
761
- * pacer's `RequestCancelled` throw.
759
+ * The error a request's last attempt earned: `sec_unreachable` for no response or
760
+ * an unreadable body; for an HTTP answer, its status-classified `McpError`, with
761
+ * the 403 and 429 hints. A 5xx ladder the failure budget stopped keeps its status
762
+ * mapping and `data`, and its message says the budget stopped it (#179).
763
+ */
764
+ async ladderFailure(url, last, end) {
765
+ if (last.kind !== 'answered')
766
+ return secUnreachable(url, last, end);
767
+ const { response } = last;
768
+ const data = { url };
769
+ if (response.status === 403) {
770
+ const host = new URL(url).hostname;
771
+ data.recovery = {
772
+ hint: `${host} may be blocking requests. Check EDGAR_USER_AGENT format ("AppName contact@email.com") or retry later.`,
773
+ };
774
+ }
775
+ else if (response.status === 429) {
776
+ // send() closed the gate on this response, so the wait is the full cool-down.
777
+ const retryAfter = this.cooldownSecondsRemaining();
778
+ data.reason = RATE_LIMITED_REASON;
779
+ data.retryable = true;
780
+ data.retryAfter = retryAfter;
781
+ data.recovery = { hint: rateLimitHint(retryAfter) };
782
+ }
783
+ const error = await httpErrorFromResponse(response, {
784
+ service: 'SEC EDGAR',
785
+ data,
786
+ // SEC answers a rate-limit block with a full HTML page. The recovery hint
787
+ // above carries everything actionable, so the markup is not forwarded (#112).
788
+ captureBody: response.status !== 429,
789
+ });
790
+ if (!end.budgetStopped)
791
+ return error;
792
+ return new McpError(error.code, `SEC EDGAR returned HTTP ${response.status} on attempt ${end.attempt} of ${MAX_RETRIES}; ${BUDGET_STOP_TEXT}.`, error.data, { cause: error });
793
+ }
794
+ /**
795
+ * One attempt through the block gate and the pacer, under the caller's signal. A
796
+ * 429 closes the gate before the response is handed back, so no call admitted
797
+ * after it sends. A `fetch` rejection comes back as an unanswered attempt, and so
798
+ * does a probe this call waited on that got no HTTP response; an attempt the spent
799
+ * failure budget kept from going out comes back as `budget_spent`. The gate's
800
+ * refusal and the pacer's `RequestCancelled` throw, and so does the caller's
801
+ * cancel, with its own reason — then a probe settles as having got no answer.
762
802
  */
763
- async send(url, headers) {
764
- const admission = await this.admit();
803
+ async send(url, headers, scope) {
804
+ const admission = await this.admit(scope);
765
805
  if ('kind' in admission)
766
806
  return admission;
767
807
  let outcome;
@@ -771,8 +811,8 @@ class EdgarApiService {
771
811
  // slot, and a call queued before the block must not go out inside it.
772
812
  if (this.gate !== admission.gate)
773
813
  throw this.cooldownRefusal();
774
- return fetchUntilHeaders(url, headers);
775
- });
814
+ return fetchUntilHeaders(url, headers, scope);
815
+ }, { signal: scope.signal });
776
816
  if (outcome.kind === 'answered' && outcome.response.status === 429) {
777
817
  this.gate = { state: 'closed', reopensAt: Date.now() + this.cooldownMs };
778
818
  }
@@ -787,18 +827,21 @@ class EdgarApiService {
787
827
  * cool-down runs. The first call after the cool-down becomes the probe; every
788
828
  * call behind it waits for the probe to resolve and then reads the gate again —
789
829
  * unless the probe got no HTTP response, which each waiter takes as its own
790
- * unanswered attempt. Otherwise calls queued behind a silent SEC would probe one
791
- * at a time, 15 s each, and the last of N would wait about 45·N s (#173).
830
+ * unanswered attempt, its wait on the probe charged to its own budget. Otherwise
831
+ * calls queued behind a silent SEC would probe one at a time, 15 s each, and the
832
+ * last of N would wait about 45·N s (#173). A waiter whose call is cancelled
833
+ * leaves with the caller's reason and leaves the gate as it found it (#179).
792
834
  */
793
- async admit() {
835
+ async admit(scope) {
794
836
  for (;;) {
795
837
  const gate = this.gate;
796
838
  if (gate.state === 'open')
797
839
  return { gate };
798
840
  if (gate.state === 'probing') {
799
- const unanswered = await gate.settled;
841
+ const waitedFrom = Date.now();
842
+ const unanswered = await untilAborted(gate.settled, scope.signal);
800
843
  if (unanswered)
801
- return unanswered;
844
+ return { ...unanswered, waitedMs: Date.now() - waitedFrom };
802
845
  continue;
803
846
  }
804
847
  if (gate.reopensAt > Date.now())
@@ -811,9 +854,13 @@ class EdgarApiService {
811
854
  releaseProbe: (outcome) => {
812
855
  /**
813
856
  * A probe 429 has already closed the gate for a full cool-down. Any other
814
- * reply means SEC is serving again; no reply at all (a network error, or
815
- * no headers within the per-attempt cap) proves nothing about the block,
816
- * so the gate stays shut for the next call to probe in its place.
857
+ * reply means SEC is serving again, even when its body then fails to read —
858
+ * the probe settles at headers, and only its own call retries the body
859
+ * (#181). No reply at all (a network error, or no headers within the
860
+ * per-attempt cap) proves nothing about the block, so the gate stays shut
861
+ * for the next call to probe in its place. Nor does a probe whose own call
862
+ * was cancelled, or never sent for a spent budget: it settles with nothing,
863
+ * so no waiter counts it as one of its own attempts (#179).
817
864
  */
818
865
  if (this.gate === probe) {
819
866
  this.gate =
@@ -840,7 +887,7 @@ class EdgarApiService {
840
887
  const remainingMs = gate.state === 'closed' ? gate.reopensAt - Date.now() : 0;
841
888
  return Math.max(1, Math.ceil(remainingMs / 1000));
842
889
  }
843
- getTickerCache() {
890
+ getTickerCache(scope) {
844
891
  const cache = this.tickerCache;
845
892
  if (cache && this.isFresh(cache.loadedAt)) {
846
893
  if (cache.fundRetryAt === undefined || Date.now() < cache.fundRetryAt) {
@@ -848,30 +895,39 @@ class EdgarApiService {
848
895
  }
849
896
  // The live fund slice failed and its retry window has passed: refetch it
850
897
  // onto the cached base, which keeps its own load time and TTL (#119).
851
- return this.loadTickerCacheOnce(() => this.reloadFundSlice(cache));
898
+ return this.loadTickerCacheOnce(scope, (load) => this.reloadFundSlice(cache, load));
852
899
  }
853
- return this.loadTickerCacheOnce(() => this.loadTickerCache());
900
+ return this.loadTickerCacheOnce(scope, (load) => this.loadTickerCache(load));
854
901
  }
855
902
  /**
856
903
  * Singleflight: concurrent callers (e.g. fetch-frames enriching ~5k reporters
857
904
  * in parallel) share one in-flight load instead of each queuing its own SEC
858
- * fetch through the pacer.
905
+ * fetch through the pacer. The load runs under a scope of its own — its own
906
+ * failure budget, and no caller's signal — so one waiter's cancel ends only its
907
+ * own wait while the load completes and caches for the rest; each waiter is
908
+ * charged the failure time the load spent while it waited (#179). A call already
909
+ * cancelled starts no load.
859
910
  */
860
- loadTickerCacheOnce(load) {
861
- this.tickerCacheLoad ??= load().finally(() => {
862
- this.tickerCacheLoad = undefined;
863
- });
864
- return this.tickerCacheLoad;
911
+ async loadTickerCacheOnce(waiter, load) {
912
+ waiter.signal.throwIfAborted();
913
+ if (!this.tickerCacheLoad) {
914
+ const scope = CallScope.unowned();
915
+ const promise = load(scope).finally(() => {
916
+ this.tickerCacheLoad = undefined;
917
+ });
918
+ this.tickerCacheLoad = { promise, scope, waits: new Map() };
919
+ }
920
+ return await awaitShared(this.tickerCacheLoad, waiter);
865
921
  }
866
922
  /** Load the whole ticker index: its base, then the live fund slice when the base calls for one. */
867
- async loadTickerCache() {
868
- const base = await this.loadTickerBase();
869
- const funds = base.liveFunds ? await this.loadMfTickers() : NO_LIVE_FUNDS;
923
+ async loadTickerCache(scope) {
924
+ const base = await this.loadTickerBase(scope);
925
+ const funds = base.liveFunds ? await this.loadMfTickers(scope) : NO_LIVE_FUNDS;
870
926
  return this.buildTickerCache(base.rows, funds, Date.now());
871
927
  }
872
928
  /** Refetch a failed live fund slice and rebuild the index on the cached base. */
873
- async reloadFundSlice(cache) {
874
- return this.buildTickerCache(cache.baseRows, await this.loadMfTickers(), cache.loadedAt);
929
+ async reloadFundSlice(cache, scope) {
930
+ return this.buildTickerCache(cache.baseRows, await this.loadMfTickers(scope), cache.loadedAt);
875
931
  }
876
932
  /**
877
933
  * Everything the ticker index holds but the live fund slice, preferring the local
@@ -883,7 +939,7 @@ class EdgarApiService {
883
939
  * since the mirror stores a fund symbol as a bare ticker → CIK row. Strict mirror
884
940
  * mode reads nothing live, so the mirror's fund rows are its only fund tickers.
885
941
  */
886
- async loadTickerBase() {
942
+ async loadTickerBase(scope) {
887
943
  const mirror = getEdgarMirror();
888
944
  if (mirror && (await mirror.tickersReady())) {
889
945
  const rows = await mirror.getTickerRows();
@@ -897,7 +953,7 @@ class EdgarApiService {
897
953
  else if (mirror && !getServerConfig().mirrorFallbackLive) {
898
954
  throw serviceUnavailable('EDGAR mirror enabled but the ticker layer is not synced; run `bun run mirror:init`', { layer: 'tickers' });
899
955
  }
900
- const raw = await this.fetchJson('https://www.sec.gov/files/company_tickers.json');
956
+ const raw = await this.fetchJson('https://www.sec.gov/files/company_tickers.json', scope);
901
957
  return {
902
958
  liveFunds: true,
903
959
  rows: Object.values(raw).map((entry) => ({
@@ -912,17 +968,20 @@ class EdgarApiService {
912
968
  * Fetch company_tickers_mf.json (ETFs and mutual funds) as fund rows carrying
913
969
  * series and class IDs. Never throws: operating-company resolution must not
914
970
  * depend on the fund file. A failed load — any HTTP failure, no HTTP response
915
- * at all (#173), a malformed body, or the block gate's local refusal — returns
916
- * no rows and a `retryAt`, so the index without funds stands for the retry
971
+ * at all (#173), a body that fails to read or is not JSON (#181), a malformed
972
+ * body, or the block gate's local refusal — returns no rows and a `retryAt`, so
973
+ * the index without funds stands for the retry
917
974
  * window rather than the whole TTL (#119). The window is at least the error's
918
975
  * `retryAfter`, so a load refused by the rate-limit gate is not retried until
919
- * the cool-down ends.
976
+ * the cool-down ends. It runs only inside the shared ticker load, whose scope no
977
+ * caller can cancel, so a failure here is always SEC's — a spent failure budget
978
+ * included (#179).
920
979
  */
921
- async loadMfTickers() {
980
+ async loadMfTickers(scope) {
922
981
  let retryAfterMs = 0;
923
982
  let cause = { reason: 'malformed_body' };
924
983
  try {
925
- const rows = parseMfTickerFile(await this.fetchJson(MF_TICKERS_URL));
984
+ const rows = parseMfTickerFile(await this.fetchJson(MF_TICKERS_URL, scope));
926
985
  if (rows)
927
986
  return { retryAt: undefined, rows };
928
987
  }
@@ -1224,64 +1283,277 @@ export function pickPreferredTicker(a, b) {
1224
1283
  return aHyphen ? b : a;
1225
1284
  return a;
1226
1285
  }
1227
- function sleep(ms) {
1228
- return new Promise((resolve) => setTimeout(resolve, ms));
1286
+ /** Sleep `ms`, or until `signal` fires — then reject with its reason. */
1287
+ function sleep(ms, signal) {
1288
+ return new Promise((resolve, reject) => {
1289
+ const onAbort = () => {
1290
+ clearTimeout(timer);
1291
+ reject(signal.reason);
1292
+ };
1293
+ const timer = setTimeout(() => {
1294
+ signal.removeEventListener('abort', onAbort);
1295
+ resolve();
1296
+ }, ms);
1297
+ signal.addEventListener('abort', onAbort, { once: true });
1298
+ if (signal.aborted)
1299
+ onAbort();
1300
+ });
1301
+ }
1302
+ /**
1303
+ * `promise`, or the rejection `signal` fires with, whichever comes first. A wait on
1304
+ * work this call does not own — the block-gate probe, the shared ticker load —
1305
+ * leaves that work running when the caller cancels.
1306
+ */
1307
+ function untilAborted(promise, signal) {
1308
+ return new Promise((resolve, reject) => {
1309
+ const onAbort = () => reject(signal.reason);
1310
+ signal.addEventListener('abort', onAbort, { once: true });
1311
+ promise.then((value) => {
1312
+ signal.removeEventListener('abort', onAbort);
1313
+ resolve(value);
1314
+ }, (error) => {
1315
+ signal.removeEventListener('abort', onAbort);
1316
+ reject(error);
1317
+ });
1318
+ if (signal.aborted)
1319
+ onAbort();
1320
+ });
1321
+ }
1322
+ /**
1323
+ * Wait on a shared load under the waiter's own signal, charging the waiter the
1324
+ * failure time the load spent while it waited — no more, so a call that joins a
1325
+ * load late is not billed for the failures before it asked (#179). One wait per
1326
+ * call: a call already waiting joins that wait.
1327
+ */
1328
+ function awaitShared(shared, waiter) {
1329
+ let wait = shared.waits.get(waiter);
1330
+ if (!wait) {
1331
+ const spentBefore = shared.scope.spentMs;
1332
+ wait = untilAborted(shared.promise, waiter.signal).finally(() => {
1333
+ waiter.charge(shared.scope.spentMs - spentBefore);
1334
+ });
1335
+ shared.waits.set(waiter, wait);
1336
+ }
1337
+ return wait;
1229
1338
  }
1230
1339
  /**
1231
- * One `fetch`, aborted when no response headers arrive within
1232
- * {@link ATTEMPT_HEADERS_TIMEOUT_MS}. The timer is cleared once `fetch` settles, so
1233
- * the body read is never bounded by it. `fetch` rejects with whatever reason the
1234
- * signal carries, so the abort is recognized by the reason's identity. A
1235
- * controller and `setTimeout` rather than `AbortSignal.timeout()`, as the
1236
- * framework's `fetchWithTimeout` does: the latter can fail on a realm mismatch
1237
- * under Bun.
1340
+ * One `fetch` under the caller's signal, aborted when no response headers arrive
1341
+ * within the lesser of {@link ATTEMPT_HEADERS_TIMEOUT_MS} and the call's failure
1342
+ * budget left (#179) — or, with no budget left, not sent at all. The timer is
1343
+ * cleared once `fetch` settles, so the body read is never bounded by it. `fetch`
1344
+ * rejects with whatever reason the signal carries, so the timeout is recognized by
1345
+ * the reason's identity; once the caller's signal has fired, the rejection is
1346
+ * rethrown as it came. A controller and `setTimeout` rather than
1347
+ * `AbortSignal.timeout()` or `AbortSignal.any()`, as the framework's
1348
+ * `fetchWithTimeout` does: those can fail on a realm mismatch under Bun.
1238
1349
  */
1239
- async function fetchUntilHeaders(url, headers) {
1350
+ async function fetchUntilHeaders(url, headers, scope) {
1351
+ scope.signal.throwIfAborted();
1352
+ const capMs = Math.min(ATTEMPT_HEADERS_TIMEOUT_MS, scope.remainingMs);
1353
+ if (capMs <= 0)
1354
+ return BUDGET_SPENT;
1240
1355
  const controller = new AbortController();
1241
- const deadline = new DOMException(`No response headers within ${ATTEMPT_HEADERS_TIMEOUT_MS}ms.`, 'TimeoutError');
1242
- const timer = setTimeout(() => controller.abort(deadline), ATTEMPT_HEADERS_TIMEOUT_MS);
1356
+ const deadline = new DOMException(`No response headers within ${capMs}ms.`, 'TimeoutError');
1357
+ const timer = setTimeout(() => controller.abort(deadline), capMs);
1358
+ const cancel = () => controller.abort(scope.signal.reason);
1359
+ scope.signal.addEventListener('abort', cancel, { once: true });
1360
+ const dispatchedAt = Date.now();
1243
1361
  try {
1244
1362
  const response = await globalThis.fetch(url, { headers, signal: controller.signal });
1245
- return { kind: 'answered', response };
1363
+ return { kind: 'answered', response, waitedMs: Date.now() - dispatchedAt };
1364
+ }
1365
+ catch (rejection) {
1366
+ if (scope.signal.aborted)
1367
+ throw rejection;
1368
+ return {
1369
+ kind: 'unanswered',
1370
+ capMs,
1371
+ rejection,
1372
+ timedOut: controller.signal.reason === deadline,
1373
+ waitedMs: Date.now() - dispatchedAt,
1374
+ };
1375
+ }
1376
+ finally {
1377
+ clearTimeout(timer);
1378
+ scope.signal.removeEventListener('abort', cancel);
1379
+ }
1380
+ }
1381
+ /**
1382
+ * Read one attempt's body as text — whole, or up to `cutoff` — failing the read once
1383
+ * no bytes arrive for the lesser of {@link BODY_IDLE_TIMEOUT_MS} and the call's
1384
+ * failure budget left (#179). The one place an attempt's body is read (#181). The
1385
+ * body is cancelled once the read stops, so an early stop at the cutoff, a stall,
1386
+ * and the caller's cancel all release the connection without reading the rest; a
1387
+ * cancel that fails is not a body failure, since the body is being discarded either
1388
+ * way. Both cutoff checks run between reads, so neither can cut below one read
1389
+ * chunk. Once the caller's signal has fired, the read's rejection is rethrown as
1390
+ * it came, never a body failure.
1391
+ */
1392
+ async function readBody(response, scope, cutoff) {
1393
+ if (!response.body)
1394
+ return { kind: 'read', text: '' };
1395
+ const stalled = new DOMException("No body bytes within the read's idle limit.", 'TimeoutError');
1396
+ const reader = response.body.getReader();
1397
+ const decoder = new TextDecoder();
1398
+ let text = '';
1399
+ let bytes = 0;
1400
+ let capMs = 0;
1401
+ let readFrom = 0;
1402
+ try {
1403
+ while (cutoff === undefined || bytes < cutoff.maxBytes) {
1404
+ capMs = Math.min(BODY_IDLE_TIMEOUT_MS, scope.remainingMs);
1405
+ readFrom = Date.now();
1406
+ const { done, value } = await nextChunk(reader, stalled, capMs, scope.signal);
1407
+ if (done) {
1408
+ text += decoder.decode();
1409
+ break;
1410
+ }
1411
+ bytes += value.byteLength;
1412
+ text += decoder.decode(value, { stream: true });
1413
+ if (cutoff !== undefined && text.includes(cutoff.stopAt))
1414
+ break;
1415
+ }
1416
+ return { kind: 'read', text };
1246
1417
  }
1247
1418
  catch (rejection) {
1248
- return { kind: 'unanswered', rejection, timedOut: controller.signal.reason === deadline };
1419
+ if (scope.signal.aborted)
1420
+ throw rejection;
1421
+ // A stall is this read's own idle marker, recognized by identity.
1422
+ return {
1423
+ kind: 'body_failed',
1424
+ capMs,
1425
+ rejection,
1426
+ stalled: rejection === stalled,
1427
+ waitedMs: Date.now() - readFrom,
1428
+ };
1429
+ }
1430
+ finally {
1431
+ await reader.cancel().catch(() => {
1432
+ /* the body is being discarded; a cancel racing its end has nothing to report */
1433
+ });
1434
+ }
1435
+ }
1436
+ /**
1437
+ * The body's next read, rejecting with `stalled` once `capMs` pass with no bytes,
1438
+ * and with the signal's reason once it fires.
1439
+ */
1440
+ async function nextChunk(reader, stalled, capMs, signal) {
1441
+ // Checked before the read: a chunk already buffered would otherwise win the race.
1442
+ signal.throwIfAborted();
1443
+ let timer;
1444
+ let onAbort;
1445
+ const idle = new Promise((_resolve, reject) => {
1446
+ timer = setTimeout(() => reject(stalled), capMs);
1447
+ onAbort = () => reject(signal.reason);
1448
+ signal.addEventListener('abort', onAbort, { once: true });
1449
+ });
1450
+ try {
1451
+ return await Promise.race([reader.read(), idle]);
1249
1452
  }
1250
1453
  finally {
1251
1454
  clearTimeout(timer);
1455
+ if (onAbort)
1456
+ signal.removeEventListener('abort', onAbort);
1457
+ }
1458
+ }
1459
+ /**
1460
+ * True when the failure budget cut the attempt's last wait short of its own cap —
1461
+ * headers or body bytes that might still have come (#179).
1462
+ */
1463
+ function cutByBudget(last) {
1464
+ switch (last.kind) {
1465
+ case 'answered':
1466
+ return false;
1467
+ case 'unanswered':
1468
+ return last.timedOut && last.capMs < ATTEMPT_HEADERS_TIMEOUT_MS;
1469
+ case 'body_failed':
1470
+ return last.stalled && last.capMs < BODY_IDLE_TIMEOUT_MS;
1471
+ default: {
1472
+ const unhandled = last;
1473
+ return unhandled;
1474
+ }
1475
+ }
1476
+ }
1477
+ /** How a failure the call's failure budget ended says so, in its message (#179). */
1478
+ const BUDGET_STOP_TEXT = `retries stopped at this call's ${CALL_FAILURE_BUDGET_MS / 1000} s budget for SEC failures`;
1479
+ /** A wait in seconds, to one decimal where it is not whole: `15`, `9.1`. */
1480
+ function seconds(ms) {
1481
+ return String(Math.round(ms / 100) / 10);
1482
+ }
1483
+ /**
1484
+ * Parse a 200's body as JSON. Every SEC route read as JSON serves JSON, so a body
1485
+ * that does not parse — an HTML error page served as a 200 — fails as
1486
+ * `sec_unreachable`, unretried since the body arrived whole; the parser's text,
1487
+ * which quotes the body, rides `cause` only (#181).
1488
+ */
1489
+ function parseJson(url, text) {
1490
+ try {
1491
+ return JSON.parse(text);
1492
+ }
1493
+ catch (rejection) {
1494
+ throw secUnreachable(url, { kind: 'not_json', rejection });
1252
1495
  }
1253
1496
  }
1254
1497
  /**
1255
- * The failure once the last attempt got no HTTP response (#173). The earlier
1256
- * attempts may have drawn a 5xx: a 503, a 502, then a reset ends here too, so
1257
- * the message claims only the last. Message and `data` are composed here from
1258
- * the host, the attempt count, and the last attempt's transport code or
1259
- * timeout: the runtime's own text is developer-facing (Bun's advice to pass
1260
- * `verbose: true`, its "Is the computer able to access the url?"), and Bun's
1261
- * rejection carries the full request URL on `path`. The rejection rides
1262
- * `cause`, which the log records and the caller never sees.
1498
+ * The failure once SEC gave no complete, readable answer. For the two ladder cases
1499
+ * — no HTTP response (#173), a body that failed to read (#181) — the earlier
1500
+ * attempts may have drawn a 5xx or failed the other way, so the message claims only
1501
+ * the last. Message and `data` are composed here from the host, the attempt count,
1502
+ * and the last attempt's transport code or timeout: the runtime's own text is
1503
+ * developer-facing (Bun's advice to pass `verbose: true`, its "Is the computer able
1504
+ * to access the url?"), and Bun carries the full request URL on `path` and, for
1505
+ * some body failures, in its message. The rejection rides `cause`, which the log
1506
+ * records and the caller never sees. A ladder the call's failure budget ended
1507
+ * names the attempt it ended on and the budget (#179).
1263
1508
  */
1264
- function secUnreachable(url, last) {
1509
+ function secUnreachable(url, last, end) {
1265
1510
  const host = new URL(url).hostname;
1266
- const ending = last.timedOut
1267
- ? `got no response headers within ${ATTEMPT_HEADERS_TIMEOUT_MS / 1000} s`
1268
- : `ended with ${transportCode(last.rejection) ?? 'a network error that carried no code'}`;
1269
- return serviceUnavailable(`No HTTP response from SEC EDGAR (${host}) on the last of ${MAX_RETRIES} attempts, which ${ending}.`, {
1270
- reason: SEC_UNREACHABLE_REASON,
1271
- retryable: true,
1272
- url,
1273
- recovery: {
1274
- hint: `SEC EDGAR (${host}) sent no HTTP response to the last attempt — a network failure between this server and SEC, not a problem with the request. Retry the same call in a minute.`,
1275
- },
1276
- }, { cause: last.rejection });
1511
+ const { message, hint } = unreadableText(host, last, end);
1512
+ return serviceUnavailable(message, { reason: SEC_UNREACHABLE_REASON, retryable: true, url, recovery: { hint } }, last.kind === 'budget_spent' ? undefined : { cause: last.rejection });
1513
+ }
1514
+ /** The message and recovery hint for each way SEC's answer was unreadable. */
1515
+ function unreadableText(host, last, end) {
1516
+ const transportEnding = (rejection) => `ended with ${transportCode(rejection) ?? 'a network error that carried no code'}`;
1517
+ const onAttempt = end?.budgetStopped
1518
+ ? `attempt ${end.attempt} of ${MAX_RETRIES}`
1519
+ : `the last of ${MAX_RETRIES} attempts`;
1520
+ const close = end?.budgetStopped ? `; ${BUDGET_STOP_TEXT}.` : '.';
1521
+ switch (last.kind) {
1522
+ case 'unanswered':
1523
+ return {
1524
+ message: `No HTTP response from SEC EDGAR (${host}) on ${onAttempt}, which ${last.timedOut ? `got no response headers within ${seconds(last.capMs)} s` : transportEnding(last.rejection)}${close}`,
1525
+ hint: `SEC EDGAR (${host}) sent no HTTP response to the last attempt — a network failure between this server and SEC, not a problem with the request. Retry the same call in a minute.`,
1526
+ };
1527
+ case 'body_failed':
1528
+ return {
1529
+ message: `The response body from SEC EDGAR (${host}) failed to read on ${onAttempt}, which ${last.stalled ? `got no body bytes for ${seconds(last.capMs)} s` : transportEnding(last.rejection)}${close}`,
1530
+ hint: `SEC EDGAR (${host}) answered the last attempt, but its response body broke off or stalled before it finished — a network failure between this server and SEC, not a problem with the request. Retry the same call in a minute.`,
1531
+ };
1532
+ case 'budget_spent':
1533
+ return {
1534
+ message: `No request sent to SEC EDGAR (${host}): ${BUDGET_STOP_TEXT}, which its earlier requests spent.`,
1535
+ hint: `SEC EDGAR (${host}) failed this call's earlier requests until its budget for SEC failures ran out — a network failure between this server and SEC, not a problem with the request. Retry the same call in a minute.`,
1536
+ };
1537
+ case 'not_json':
1538
+ return {
1539
+ message: `SEC EDGAR (${host}) answered, but its response body is not the JSON this request expects.`,
1540
+ hint: `SEC EDGAR (${host}) answered with something other than the JSON it serves for this request, such as a temporary error page — not a problem with the request. Retry the same call in a minute.`,
1541
+ };
1542
+ default: {
1543
+ const unhandled = last;
1544
+ return unhandled;
1545
+ }
1546
+ }
1277
1547
  }
1278
1548
  /** An identifier-shaped transport code: `ECONNRESET`, `EAI_AGAIN`, `UND_ERR_CONNECT_TIMEOUT`. */
1279
1549
  const TRANSPORT_CODE = /^[A-Za-z][A-Za-z0-9_]*$/;
1280
1550
  /**
1281
- * The transport code on a `fetch` rejection. Bun sets it on the rejection itself
1282
- * (`code: 'ECONNRESET'`, no `cause`); Node's undici rejects with
1283
- * `TypeError('fetch failed')` and sets it on `cause`. Read from the field, never
1284
- * the message, whose wording differs by runtime and version.
1551
+ * The transport code on a `fetch` or body-read rejection. Bun sets it on the
1552
+ * rejection itself (`code: 'ECONNRESET'`, `'InvalidHTTPResponse'`, `'ZlibError'`,
1553
+ * no `cause`); Node's undici rejects with `TypeError('fetch failed')` or, mid-body,
1554
+ * `TypeError('terminated')` and sets it on `cause` — when it has one: its
1555
+ * chunk-framing `HTTPParserError` carries none. Read from the field, never the
1556
+ * message, whose wording differs by runtime and version.
1285
1557
  */
1286
1558
  function transportCode(rejection) {
1287
1559
  const codeOf = (value) => {