@cyanheads/secedgar-mcp-server 0.15.7 → 0.15.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +18 -10
- package/CLAUDE.md +18 -10
- package/README.md +112 -162
- package/changelog/0.15.x/0.15.8.md +29 -0
- package/dist/config/server-config.d.ts.map +1 -1
- package/dist/config/server-config.js +1 -1
- package/dist/config/server-config.js.map +1 -1
- package/dist/mcp-server/tools/definitions/company-search.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/company-search.tool.js +15 -26
- package/dist/mcp-server/tools/definitions/company-search.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/dataframe-query.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/dataframe-query.tool.js +7 -1
- package/dist/mcp-server/tools/definitions/dataframe-query.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.js +1 -1
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-filing.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-filing.tool.js +16 -11
- package/dist/mcp-server/tools/definitions/get-filing.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.d.ts +5 -2
- package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.js +151 -28
- package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.d.ts +1 -1
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.js +89 -41
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-material-events.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-material-events.tool.js +29 -33
- package/dist/mcp-server/tools/definitions/get-material-events.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/index.d.ts +6 -3
- package/dist/mcp-server/tools/definitions/index.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/search-filings.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/search-filings.tool.js +12 -20
- package/dist/mcp-server/tools/definitions/search-filings.tool.js.map +1 -1
- package/dist/services/edgar/edgar-api-service.d.ts +53 -41
- package/dist/services/edgar/edgar-api-service.d.ts.map +1 -1
- package/dist/services/edgar/edgar-api-service.js +213 -144
- package/dist/services/edgar/edgar-api-service.js.map +1 -1
- package/dist/services/edgar/filing-headers.d.ts +37 -9
- package/dist/services/edgar/filing-headers.d.ts.map +1 -1
- package/dist/services/edgar/filing-headers.js +40 -11
- package/dist/services/edgar/filing-headers.js.map +1 -1
- package/dist/services/edgar/filing-to-text.d.ts.map +1 -1
- package/dist/services/edgar/filing-to-text.js +24 -2
- package/dist/services/edgar/filing-to-text.js.map +1 -1
- package/dist/services/edgar/mirror/tickers-sync.d.ts +4 -2
- package/dist/services/edgar/mirror/tickers-sync.d.ts.map +1 -1
- package/dist/services/edgar/mirror/tickers-sync.js +4 -2
- package/dist/services/edgar/mirror/tickers-sync.js.map +1 -1
- package/dist/services/edgar/mirror/types.d.ts +6 -2
- package/dist/services/edgar/mirror/types.d.ts.map +1 -1
- package/dist/services/edgar/ownership-parser.d.ts +16 -0
- package/dist/services/edgar/ownership-parser.d.ts.map +1 -1
- package/dist/services/edgar/ownership-parser.js +19 -0
- package/dist/services/edgar/ownership-parser.js.map +1 -1
- package/dist/services/edgar/submissions-archive.d.ts +78 -0
- package/dist/services/edgar/submissions-archive.d.ts.map +1 -0
- package/dist/services/edgar/submissions-archive.js +87 -0
- package/dist/services/edgar/submissions-archive.js.map +1 -0
- package/package.json +1 -1
- package/server.json +3 -3
|
@@ -4,13 +4,13 @@
|
|
|
4
4
|
* with backoff, CIK resolution, and ticker/entity caching.
|
|
5
5
|
* @module services/edgar/edgar-api-service
|
|
6
6
|
*/
|
|
7
|
-
import { notFound, rateLimited, serviceUnavailable
|
|
8
|
-
import { createPacer, httpErrorFromResponse } from '@cyanheads/mcp-ts-core/utils';
|
|
7
|
+
import { McpError, notFound, rateLimited, serviceUnavailable } from '@cyanheads/mcp-ts-core/errors';
|
|
8
|
+
import { createPacer, httpErrorFromResponse, logger, requestContextService, withExtra, } from '@cyanheads/mcp-ts-core/utils';
|
|
9
9
|
import { parseDocument } from 'htmlparser2';
|
|
10
10
|
import { getServerConfig } from '../../config/server-config.js';
|
|
11
11
|
import { getEdgarMirror } from './mirror/index.js';
|
|
12
12
|
import formerNamesData from './data/former-names.json' with { type: 'json' };
|
|
13
|
-
import { parseFilingHeaders } from './filing-headers.js';
|
|
13
|
+
import { parseFilingHeaders, parseSubmissionHeader, } from './filing-headers.js';
|
|
14
14
|
import { childText, findTag, findTags } from './xml-nodes.js';
|
|
15
15
|
const MAX_RETRIES = 3;
|
|
16
16
|
const BASE_BACKOFF_MS = 1000;
|
|
@@ -24,7 +24,9 @@ const RETRYABLE_STATUSES = new Set([500, 502, 503, 504]);
|
|
|
24
24
|
/**
|
|
25
25
|
* `data.reason` on every SEC rate-limit failure — the upstream 429 and a call
|
|
26
26
|
* refused locally while the cool-down after one runs — so a caller branches the
|
|
27
|
-
* same way on either (#116).
|
|
27
|
+
* same way on either (#116). Both also set `data.retryable: true`: the tools
|
|
28
|
+
* declare the reason retryable, but only `ctx.fail` copies that onto `data`, so
|
|
29
|
+
* a service-thrown error has to carry it itself (#122).
|
|
28
30
|
*/
|
|
29
31
|
const RATE_LIMITED_REASON = 'rate_limited';
|
|
30
32
|
const OPEN_GATE = { state: 'open' };
|
|
@@ -38,10 +40,19 @@ function rateLimitHint(retryAfterSeconds) {
|
|
|
38
40
|
}
|
|
39
41
|
/** URL for SEC's mutual-fund ticker file (ETFs and open-end funds). */
|
|
40
42
|
const MF_TICKERS_URL = 'https://www.sec.gov/files/company_tickers_mf.json';
|
|
43
|
+
/**
|
|
44
|
+
* How long a failed load of the fund-ticker file stands before one resolution
|
|
45
|
+
* refetches it (#119). A window rather than every resolution: against a
|
|
46
|
+
* persistent 5xx, a per-lookup retry would add three requests and ~3s of backoff
|
|
47
|
+
* to each one. After a rate-limit failure the window stretches to the cool-down.
|
|
48
|
+
*/
|
|
49
|
+
const FUND_SLICE_RETRY_MS = 60_000;
|
|
41
50
|
/** Trigram similarity threshold — minimum Dice score to include a candidate suggestion. */
|
|
42
51
|
const TRIGRAM_THRESHOLD = 0.3;
|
|
43
52
|
/** Maximum number of near-match suggestions to include. */
|
|
44
53
|
const TRIGRAM_TOP_N = 3;
|
|
54
|
+
/** The fund slice where nothing is read live: strict mirror mode. */
|
|
55
|
+
const NO_LIVE_FUNDS = { retryAt: undefined, rows: [] };
|
|
45
56
|
// ---------------------------------------------------------------------------
|
|
46
57
|
// Corporate-suffix normalization
|
|
47
58
|
// ---------------------------------------------------------------------------
|
|
@@ -375,8 +386,9 @@ class EdgarApiService {
|
|
|
375
386
|
}
|
|
376
387
|
// --- SEC API Methods ---
|
|
377
388
|
/**
|
|
378
|
-
* Fetch a filer's submissions document (entity metadata + the
|
|
379
|
-
*
|
|
389
|
+
* Fetch a filer's submissions document (entity metadata + the `recent` window —
|
|
390
|
+
* the last year or 1,000 filings, whichever holds more — + the `files[]`
|
|
391
|
+
* archive-page manifest). Cached per CIK within
|
|
380
392
|
* the ticker-cache TTL — the doc is large and re-read on every archive-paging
|
|
381
393
|
* scan (#78). A 404 throws (uncached) so a bad CIK still surfaces.
|
|
382
394
|
*/
|
|
@@ -392,7 +404,7 @@ class EdgarApiService {
|
|
|
392
404
|
/**
|
|
393
405
|
* Fetch a submissions archive page (`filings.files[].name`, e.g.
|
|
394
406
|
* `CIK0000320193-submissions-001.json`) — the older filings that don't fit the
|
|
395
|
-
*
|
|
407
|
+
* `recent` window. Read through `SubmissionsArchiveWalk`. The page body is a flat parallel-array object
|
|
396
408
|
* field-compatible with `FilingsRecent`. Cached per page within the ticker-cache
|
|
397
409
|
* TTL (pages are large and effectively immutable once archived).
|
|
398
410
|
*/
|
|
@@ -489,10 +501,10 @@ class EdgarApiService {
|
|
|
489
501
|
return this.tryFetchJson(`https://www.sec.gov/Archives/edgar/data/${padded}/${noDashes}/index.json`);
|
|
490
502
|
}
|
|
491
503
|
/**
|
|
492
|
-
* Fetch the SEC submission header (`<accession>-index-headers.html`) and parse
|
|
493
|
-
* it into
|
|
494
|
-
*
|
|
495
|
-
* that the directory listing JSON does not.
|
|
504
|
+
* Fetch the SEC submission header page (`<accession>-index-headers.html`) and parse
|
|
505
|
+
* it into the `filename → metadata` map plus the submission's own form, filing date,
|
|
506
|
+
* and period. Returns `null` if the page is absent. The page exposes canonical SEC
|
|
507
|
+
* document TYPE values (e.g. "EX-21.1") that the directory listing JSON does not.
|
|
496
508
|
*/
|
|
497
509
|
async tryGetFilingHeaders(cik, accessionNumber) {
|
|
498
510
|
const padded = cik.padStart(10, '0');
|
|
@@ -500,6 +512,18 @@ class EdgarApiService {
|
|
|
500
512
|
const text = await this.tryFetchText(`https://www.sec.gov/Archives/edgar/data/${padded}/${noDashes}/${accessionNumber}-index-headers.html`);
|
|
501
513
|
return text ? parseFilingHeaders(text) : null;
|
|
502
514
|
}
|
|
515
|
+
/**
|
|
516
|
+
* Fetch the bare SGML submission header (`<accession>.hdr.sgml`, under 1 KB) and read
|
|
517
|
+
* its form, filing date, and period. Returns `null` if the file is absent. The
|
|
518
|
+
* fallback for filings whose index-headers page is missing — common among filings
|
|
519
|
+
* made before 2014; it lists no documents.
|
|
520
|
+
*/
|
|
521
|
+
async tryGetSubmissionHeader(cik, accessionNumber) {
|
|
522
|
+
const padded = cik.padStart(10, '0');
|
|
523
|
+
const noDashes = accessionNumber.replace(/-/g, '');
|
|
524
|
+
const text = await this.tryFetchText(`https://www.sec.gov/Archives/edgar/data/${padded}/${noDashes}/${accessionNumber}.hdr.sgml`);
|
|
525
|
+
return text ? parseSubmissionHeader(text) : null;
|
|
526
|
+
}
|
|
503
527
|
getFilingDocument(cik, accessionNumber, document) {
|
|
504
528
|
const padded = cik.padStart(10, '0');
|
|
505
529
|
const noDashes = accessionNumber.replace(/-/g, '');
|
|
@@ -672,6 +696,7 @@ class EdgarApiService {
|
|
|
672
696
|
// send() closed the gate on this response, so the wait is the full cool-down.
|
|
673
697
|
const retryAfter = this.cooldownSecondsRemaining();
|
|
674
698
|
data.reason = RATE_LIMITED_REASON;
|
|
699
|
+
data.retryable = true;
|
|
675
700
|
data.retryAfter = retryAfter;
|
|
676
701
|
data.recovery = { hint: rateLimitHint(retryAfter) };
|
|
677
702
|
}
|
|
@@ -750,7 +775,12 @@ class EdgarApiService {
|
|
|
750
775
|
/** The error for a call made while the cool-down runs — nothing was sent. */
|
|
751
776
|
cooldownRefusal() {
|
|
752
777
|
const retryAfter = this.cooldownSecondsRemaining();
|
|
753
|
-
return rateLimited(`SEC EDGAR request not sent: SEC is rate-limiting this server's IP, and requests are held for another ${retryAfter}s so the block can clear.`, {
|
|
778
|
+
return rateLimited(`SEC EDGAR request not sent: SEC is rate-limiting this server's IP, and requests are held for another ${retryAfter}s so the block can clear.`, {
|
|
779
|
+
reason: RATE_LIMITED_REASON,
|
|
780
|
+
retryable: true,
|
|
781
|
+
retryAfter,
|
|
782
|
+
recovery: { hint: rateLimitHint(retryAfter) },
|
|
783
|
+
});
|
|
754
784
|
}
|
|
755
785
|
/** Whole seconds until the gate reopens — never below 1, since a caller told to wait 0s is refused again. */
|
|
756
786
|
cooldownSecondsRemaining() {
|
|
@@ -759,157 +789,215 @@ class EdgarApiService {
|
|
|
759
789
|
return Math.max(1, Math.ceil(remainingMs / 1000));
|
|
760
790
|
}
|
|
761
791
|
getTickerCache() {
|
|
762
|
-
const
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
792
|
+
const cache = this.tickerCache;
|
|
793
|
+
if (cache && this.isFresh(cache.loadedAt)) {
|
|
794
|
+
if (cache.fundRetryAt === undefined || Date.now() < cache.fundRetryAt) {
|
|
795
|
+
return Promise.resolve(cache);
|
|
796
|
+
}
|
|
797
|
+
// The live fund slice failed and its retry window has passed: refetch it
|
|
798
|
+
// onto the cached base, which keeps its own load time and TTL (#119).
|
|
799
|
+
return this.loadTickerCacheOnce(() => this.reloadFundSlice(cache));
|
|
766
800
|
}
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
801
|
+
return this.loadTickerCacheOnce(() => this.loadTickerCache());
|
|
802
|
+
}
|
|
803
|
+
/**
|
|
804
|
+
* Singleflight: concurrent callers (e.g. fetch-frames enriching ~5k reporters
|
|
805
|
+
* in parallel) share one in-flight load instead of each queuing its own SEC
|
|
806
|
+
* fetch through the pacer.
|
|
807
|
+
*/
|
|
808
|
+
loadTickerCacheOnce(load) {
|
|
809
|
+
this.tickerCacheLoad ??= load().finally(() => {
|
|
771
810
|
this.tickerCacheLoad = undefined;
|
|
772
811
|
});
|
|
773
812
|
return this.tickerCacheLoad;
|
|
774
813
|
}
|
|
814
|
+
/** Load the whole ticker index: its base, then the live fund slice when the base calls for one. */
|
|
815
|
+
async loadTickerCache() {
|
|
816
|
+
const base = await this.loadTickerBase();
|
|
817
|
+
const funds = base.liveFunds ? await this.loadMfTickers() : NO_LIVE_FUNDS;
|
|
818
|
+
return this.buildTickerCache(base.rows, funds, Date.now());
|
|
819
|
+
}
|
|
820
|
+
/** Refetch a failed live fund slice and rebuild the index on the cached base. */
|
|
821
|
+
async reloadFundSlice(cache) {
|
|
822
|
+
return this.buildTickerCache(cache.baseRows, await this.loadMfTickers(), cache.loadedAt);
|
|
823
|
+
}
|
|
775
824
|
/**
|
|
776
|
-
*
|
|
777
|
-
*
|
|
778
|
-
* when the mirror is off
|
|
779
|
-
*
|
|
780
|
-
*
|
|
781
|
-
*
|
|
782
|
-
* the
|
|
783
|
-
*
|
|
784
|
-
* Mirror path (#43): when `mirrorFallbackLive` is true, a live MF fetch is merged
|
|
785
|
-
* into the mirror-served equity base so fund tickers resolve even if the mirror
|
|
786
|
-
* was synced before MF ingestion was added. When `mirrorFallbackLive` is false
|
|
787
|
-
* (strict offline), the mirror's own MF rows are the sole source.
|
|
825
|
+
* Everything the ticker index holds but the live fund slice, preferring the local
|
|
826
|
+
* mirror when it is enabled and synced; company_tickers.json is the cold-start /
|
|
827
|
+
* not-ready fallback and the only source when the mirror is off. `liveFunds` says
|
|
828
|
+
* whether company_tickers_mf.json supplements it: always on the live path, and on
|
|
829
|
+
* the mirror path under `mirrorFallbackLive`, where it covers a mirror synced
|
|
830
|
+
* before fund ingestion (#43) and is the only source of series and class IDs,
|
|
831
|
+
* since the mirror stores a fund symbol as a bare ticker → CIK row. Strict mirror
|
|
832
|
+
* mode reads nothing live, so the mirror's fund rows are its only fund tickers.
|
|
788
833
|
*/
|
|
789
|
-
async
|
|
834
|
+
async loadTickerBase() {
|
|
790
835
|
const mirror = getEdgarMirror();
|
|
791
836
|
if (mirror && (await mirror.tickersReady())) {
|
|
792
837
|
const rows = await mirror.getTickerRows();
|
|
793
838
|
if (rows.length > 0) {
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
// ingestion. Failure is non-fatal — equity resolution still works.
|
|
799
|
-
if (config.mirrorFallbackLive) {
|
|
800
|
-
const mfEntries = await this.loadMfTickers();
|
|
801
|
-
allEntries.push(...mfEntries);
|
|
802
|
-
}
|
|
803
|
-
return this.buildTickerCache(allEntries, buildFormerNameEntries());
|
|
839
|
+
return {
|
|
840
|
+
liveFunds: getServerConfig().mirrorFallbackLive,
|
|
841
|
+
rows: rows.map(mirrorIndexRow),
|
|
842
|
+
};
|
|
804
843
|
}
|
|
805
844
|
}
|
|
806
845
|
else if (mirror && !getServerConfig().mirrorFallbackLive) {
|
|
807
846
|
throw serviceUnavailable('EDGAR mirror enabled but the ticker layer is not synced; run `bun run mirror:init`', { layer: 'tickers' });
|
|
808
847
|
}
|
|
809
|
-
// Fetch operating-company tickers (company_tickers.json)
|
|
810
848
|
const raw = await this.fetchJson('https://www.sec.gov/files/company_tickers.json');
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
// Merge the committed former-names asset.
|
|
821
|
-
const formerEntries = buildFormerNameEntries();
|
|
822
|
-
return this.buildTickerCache(entries, formerEntries);
|
|
849
|
+
return {
|
|
850
|
+
liveFunds: true,
|
|
851
|
+
rows: Object.values(raw).map((entry) => ({
|
|
852
|
+
cik: String(entry.cik_str).padStart(10, '0'),
|
|
853
|
+
kind: 'equity',
|
|
854
|
+
name: entry.title,
|
|
855
|
+
ticker: entry.ticker,
|
|
856
|
+
})),
|
|
857
|
+
};
|
|
823
858
|
}
|
|
824
859
|
/**
|
|
825
|
-
* Fetch
|
|
826
|
-
*
|
|
827
|
-
*
|
|
828
|
-
*
|
|
829
|
-
*
|
|
860
|
+
* Fetch company_tickers_mf.json (ETFs and mutual funds) as fund rows carrying
|
|
861
|
+
* series and class IDs. Never throws: operating-company resolution must not
|
|
862
|
+
* depend on the fund file. A failed load — any HTTP failure, a malformed body,
|
|
863
|
+
* or the block gate's local refusal — returns no rows and a `retryAt`, so the
|
|
864
|
+
* index without funds stands for the retry window rather than the whole TTL
|
|
865
|
+
* (#119). The window is at least the error's `retryAfter`, so a load refused
|
|
866
|
+
* by the rate-limit gate is not retried until the cool-down ends.
|
|
830
867
|
*/
|
|
831
868
|
async loadMfTickers() {
|
|
869
|
+
let retryAfterMs = 0;
|
|
870
|
+
let cause = { reason: 'malformed_body' };
|
|
832
871
|
try {
|
|
833
|
-
const
|
|
834
|
-
if (
|
|
835
|
-
return
|
|
836
|
-
const fieldIdx = {
|
|
837
|
-
cik: mfRaw.fields.indexOf('cik'),
|
|
838
|
-
seriesId: mfRaw.fields.indexOf('seriesId'),
|
|
839
|
-
classId: mfRaw.fields.indexOf('classId'),
|
|
840
|
-
symbol: mfRaw.fields.indexOf('symbol'),
|
|
841
|
-
};
|
|
842
|
-
if (fieldIdx.cik < 0 || fieldIdx.symbol < 0)
|
|
843
|
-
return [];
|
|
844
|
-
return mfRaw.data
|
|
845
|
-
.filter((row) => row[fieldIdx.symbol])
|
|
846
|
-
.map((row) => ({
|
|
847
|
-
cik: String(row[fieldIdx.cik]).padStart(10, '0'),
|
|
848
|
-
name: '',
|
|
849
|
-
ticker: String(row[fieldIdx.symbol]),
|
|
850
|
-
seriesId: fieldIdx.seriesId >= 0 ? String(row[fieldIdx.seriesId]) : '',
|
|
851
|
-
classId: fieldIdx.classId >= 0 ? String(row[fieldIdx.classId]) : '',
|
|
852
|
-
}));
|
|
872
|
+
const rows = parseMfTickerFile(await this.fetchJson(MF_TICKERS_URL));
|
|
873
|
+
if (rows)
|
|
874
|
+
return { retryAt: undefined, rows };
|
|
853
875
|
}
|
|
854
|
-
catch {
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
876
|
+
catch (error) {
|
|
877
|
+
const data = error instanceof McpError ? error.data : undefined;
|
|
878
|
+
if (typeof data?.retryAfter === 'number')
|
|
879
|
+
retryAfterMs = data.retryAfter * 1000;
|
|
880
|
+
cause = {
|
|
881
|
+
...(typeof data?.status === 'number' && { status: data.status }),
|
|
882
|
+
...(typeof data?.reason === 'string' && { reason: data.reason }),
|
|
883
|
+
error: error instanceof Error ? error.message : String(error),
|
|
884
|
+
};
|
|
860
885
|
}
|
|
886
|
+
const retryMs = Math.max(FUND_SLICE_RETRY_MS, retryAfterMs);
|
|
887
|
+
// Background index maintenance with no request of its own to attach to, so the
|
|
888
|
+
// global logger carries it (#43).
|
|
889
|
+
logger.warning('Fund-ticker file (company_tickers_mf.json) failed to load; ETF and mutual-fund tickers stay unresolved until it is retried.', withExtra(requestContextService.createRequestContext({ operation: 'loadMfTickers' }), {
|
|
890
|
+
url: MF_TICKERS_URL,
|
|
891
|
+
retryInSeconds: Math.ceil(retryMs / 1000),
|
|
892
|
+
...cause,
|
|
893
|
+
}));
|
|
894
|
+
return { retryAt: Date.now() + retryMs, rows: [] };
|
|
861
895
|
}
|
|
862
896
|
/**
|
|
863
|
-
* Build the in-memory
|
|
864
|
-
*
|
|
865
|
-
* a registrant trust (
|
|
866
|
-
*
|
|
897
|
+
* Build the in-memory index. Equity rows fill `byTicker`, `byCik`, and — when they
|
|
898
|
+
* carry a name — `allEntries` for name search. Fund rows fill `byTicker` only,
|
|
899
|
+
* never `byCik`, because a registrant trust (CIK 36405 = Vanguard Index Funds)
|
|
900
|
+
* holds many series; live fund rows also fill `bySeriesId`. On a shared symbol the
|
|
901
|
+
* first to claim it wins, in the order equity, live fund, mirror fund: a fund
|
|
902
|
+
* symbol never overrides an operating-company ticker on the rare cross-file
|
|
903
|
+
* collision (SPCX), and a live fund entry, carrying series and class, supersedes
|
|
904
|
+
* the mirror's bare row for the same symbol (#135). Former names from the
|
|
905
|
+
* committed asset go into `allEntries` only.
|
|
867
906
|
*/
|
|
868
|
-
buildTickerCache(
|
|
907
|
+
buildTickerCache(baseRows, funds, loadedAt) {
|
|
869
908
|
const byTicker = new Map();
|
|
870
909
|
const byCik = new Map();
|
|
871
910
|
const bySeriesId = new Map();
|
|
872
911
|
const allEntries = [];
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
912
|
+
const mirrorFunds = [];
|
|
913
|
+
for (const row of baseRows) {
|
|
914
|
+
if (row.kind === 'fund') {
|
|
915
|
+
mirrorFunds.push(row);
|
|
916
|
+
continue;
|
|
917
|
+
}
|
|
876
918
|
const match = {
|
|
877
|
-
cik:
|
|
878
|
-
ticker:
|
|
879
|
-
...(
|
|
880
|
-
|
|
881
|
-
|
|
919
|
+
cik: row.cik,
|
|
920
|
+
ticker: row.ticker,
|
|
921
|
+
...(row.name ? { name: row.name } : {}),
|
|
922
|
+
};
|
|
923
|
+
byTicker.set(row.ticker.toUpperCase(), match);
|
|
924
|
+
const existing = byCik.get(match.cik);
|
|
925
|
+
byCik.set(match.cik, existing ? pickPreferredTicker(existing, match) : match);
|
|
926
|
+
if (match.name)
|
|
927
|
+
allEntries.push(match);
|
|
928
|
+
}
|
|
929
|
+
for (const row of [...funds.rows, ...mirrorFunds]) {
|
|
930
|
+
const match = {
|
|
931
|
+
cik: row.cik,
|
|
932
|
+
ticker: row.ticker,
|
|
933
|
+
...(row.seriesId ? { seriesId: row.seriesId } : {}),
|
|
934
|
+
...(row.classId ? { classId: row.classId } : {}),
|
|
882
935
|
};
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
const tickerKey = entry.ticker.toUpperCase();
|
|
886
|
-
if (!isMf || !byTicker.has(tickerKey)) {
|
|
936
|
+
const tickerKey = row.ticker.toUpperCase();
|
|
937
|
+
if (!byTicker.has(tickerKey))
|
|
887
938
|
byTicker.set(tickerKey, match);
|
|
888
|
-
|
|
889
|
-
// A series has one entry per listed share class, so the first class registers the
|
|
939
|
+
// A series has one row per listed share class, so the first class registers the
|
|
890
940
|
// series; later classes of the same series would only re-point it at the same trust.
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
// MF entries must not overwrite byCik — the trust CIK is 1:many with fund series.
|
|
895
|
-
if (!isMf) {
|
|
896
|
-
const existing = byCik.get(match.cik);
|
|
897
|
-
byCik.set(match.cik, existing ? pickPreferredTicker(existing, match) : match);
|
|
898
|
-
}
|
|
899
|
-
// Only push to allEntries if the entry has a name (for name search).
|
|
900
|
-
// MF entries have no name, so they're ticker-only.
|
|
901
|
-
if (match.name) {
|
|
902
|
-
allEntries.push(match);
|
|
903
|
-
}
|
|
941
|
+
const seriesKey = row.seriesId?.toUpperCase();
|
|
942
|
+
if (seriesKey && !bySeriesId.has(seriesKey))
|
|
943
|
+
bySeriesId.set(seriesKey, match);
|
|
904
944
|
}
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
allEntries.push({ cik: fn.cik, name: fn.name });
|
|
945
|
+
for (const former of buildFormerNameEntries()) {
|
|
946
|
+
allEntries.push({ cik: former.cik, name: former.name });
|
|
908
947
|
}
|
|
909
|
-
this.tickerCache = {
|
|
948
|
+
this.tickerCache = {
|
|
949
|
+
allEntries,
|
|
950
|
+
baseRows,
|
|
951
|
+
byCik,
|
|
952
|
+
bySeriesId,
|
|
953
|
+
byTicker,
|
|
954
|
+
fundRetryAt: funds.retryAt,
|
|
955
|
+
loadedAt,
|
|
956
|
+
};
|
|
910
957
|
return this.tickerCache;
|
|
911
958
|
}
|
|
912
959
|
}
|
|
960
|
+
/**
|
|
961
|
+
* Classify a mirror ticker row. The mirror schema has no series or class column:
|
|
962
|
+
* tickers-sync stores a fund symbol from company_tickers_mf.json with an empty
|
|
963
|
+
* name, and every operating-company row with its registrant title, so the empty
|
|
964
|
+
* name is what marks a fund (#135).
|
|
965
|
+
*/
|
|
966
|
+
function mirrorIndexRow(row) {
|
|
967
|
+
return row.name
|
|
968
|
+
? { cik: row.cik, kind: 'equity', name: row.name, ticker: row.ticker }
|
|
969
|
+
: { cik: row.cik, kind: 'fund', ticker: row.ticker };
|
|
970
|
+
}
|
|
971
|
+
/**
|
|
972
|
+
* Parse company_tickers_mf.json — columnar, with a `fields` header naming each
|
|
973
|
+
* column — into fund rows. Returns `undefined` when the body lacks that shape or
|
|
974
|
+
* the `cik` and `symbol` columns, which the caller treats as a failed load.
|
|
975
|
+
*/
|
|
976
|
+
function parseMfTickerFile(raw) {
|
|
977
|
+
if (!Array.isArray(raw?.fields) || !Array.isArray(raw?.data))
|
|
978
|
+
return;
|
|
979
|
+
const column = {
|
|
980
|
+
cik: raw.fields.indexOf('cik'),
|
|
981
|
+
classId: raw.fields.indexOf('classId'),
|
|
982
|
+
seriesId: raw.fields.indexOf('seriesId'),
|
|
983
|
+
symbol: raw.fields.indexOf('symbol'),
|
|
984
|
+
};
|
|
985
|
+
if (column.cik < 0 || column.symbol < 0)
|
|
986
|
+
return;
|
|
987
|
+
return raw.data
|
|
988
|
+
.filter((row) => row[column.symbol])
|
|
989
|
+
.map((row) => {
|
|
990
|
+
const seriesId = column.seriesId >= 0 ? String(row[column.seriesId] ?? '') : '';
|
|
991
|
+
const classId = column.classId >= 0 ? String(row[column.classId] ?? '') : '';
|
|
992
|
+
return {
|
|
993
|
+
cik: String(row[column.cik]).padStart(10, '0'),
|
|
994
|
+
kind: 'fund',
|
|
995
|
+
ticker: String(row[column.symbol]),
|
|
996
|
+
...(seriesId ? { seriesId } : {}),
|
|
997
|
+
...(classId ? { classId } : {}),
|
|
998
|
+
};
|
|
999
|
+
});
|
|
1000
|
+
}
|
|
913
1001
|
/**
|
|
914
1002
|
* Build former-name entries from the committed static asset.
|
|
915
1003
|
* Each tuple is [lowercasedName, zeroPaddedCIK].
|
|
@@ -930,25 +1018,6 @@ export function cleanDisplayName(displayName) {
|
|
|
930
1018
|
.replace(/\s+/g, ' ')
|
|
931
1019
|
.trim();
|
|
932
1020
|
}
|
|
933
|
-
/**
|
|
934
|
-
* Range-based selection of submissions archive-page manifest entries (`filings.files[]`),
|
|
935
|
-
* returned newest-first (by `filingTo` descending). A page's [filingFrom, filingTo]
|
|
936
|
-
* window is kept unless it lies entirely before `filedAfter` or entirely after
|
|
937
|
-
* `filedBefore`; with no bounds, every page is returned. Routes company_search's
|
|
938
|
-
* older-filings scan directly to the pages covering a requested date range, or walks
|
|
939
|
-
* all pages newest-first when a form filter under-fills the recent window (#78).
|
|
940
|
-
*/
|
|
941
|
-
export function selectArchivePages(files, filedAfter, filedBefore) {
|
|
942
|
-
return files
|
|
943
|
-
.filter((page) => {
|
|
944
|
-
if (filedAfter && page.filingTo < filedAfter)
|
|
945
|
-
return false;
|
|
946
|
-
if (filedBefore && page.filingFrom > filedBefore)
|
|
947
|
-
return false;
|
|
948
|
-
return true;
|
|
949
|
-
})
|
|
950
|
-
.sort((a, b) => b.filingTo.localeCompare(a.filingTo));
|
|
951
|
-
}
|
|
952
1021
|
/**
|
|
953
1022
|
* Parse a quarterly EDGAR `master.idx` into filing rows. The file is a short
|
|
954
1023
|
* metadata preamble, then a `CIK|Company Name|Form Type|Date Filed|Filename`
|
|
@@ -1025,8 +1094,8 @@ export function rawDocumentName(primaryDocument) {
|
|
|
1025
1094
|
}
|
|
1026
1095
|
/**
|
|
1027
1096
|
* Enumerate the calendar quarters overlapping the inclusive [startDate, endDate]
|
|
1028
|
-
* range (both YYYY-MM-DD), returned NEWEST-first
|
|
1029
|
-
* — a capped scan then keeps the most recent quarters, consistent with the
|
|
1097
|
+
* range (both YYYY-MM-DD), returned NEWEST-first like the archive-page walk's
|
|
1098
|
+
* default order — a capped scan then keeps the most recent quarters, consistent with the
|
|
1030
1099
|
* default filing-date-descending sort. Routes search_filings' pre-2001
|
|
1031
1100
|
* full-index browse to the `master.idx` files it must fetch (#77).
|
|
1032
1101
|
*/
|