@cyanheads/secedgar-mcp-server 0.18.1 → 0.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +36 -24
- package/CLAUDE.md +36 -24
- package/README.md +16 -12
- package/changelog/0.19.x/0.19.0.md +47 -0
- package/changelog/0.19.x/0.19.1.md +35 -0
- package/changelog/template.md +10 -3
- package/dist/config/server-config.js +2 -2
- package/dist/config/server-config.js.map +1 -1
- package/dist/index.js +27 -9
- package/dist/index.js.map +1 -1
- package/dist/mcp-server/tools/dataset-schema.d.ts +14 -0
- package/dist/mcp-server/tools/dataset-schema.d.ts.map +1 -0
- package/dist/mcp-server/tools/dataset-schema.js +16 -0
- package/dist/mcp-server/tools/dataset-schema.js.map +1 -0
- package/dist/mcp-server/tools/definitions/company-search.tool.d.ts +44 -14
- package/dist/mcp-server/tools/definitions/company-search.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/company-search.tool.js +197 -62
- package/dist/mcp-server/tools/definitions/company-search.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/compare-companies.tool.d.ts +26 -12
- package/dist/mcp-server/tools/definitions/compare-companies.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/compare-companies.tool.js +41 -39
- package/dist/mcp-server/tools/definitions/compare-companies.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/dataframe-describe.tool.d.ts +12 -3
- package/dist/mcp-server/tools/definitions/dataframe-describe.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/dataframe-describe.tool.js +23 -9
- package/dist/mcp-server/tools/definitions/dataframe-describe.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/dataframe-drop.tool.js +2 -2
- package/dist/mcp-server/tools/definitions/dataframe-drop.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/dataframe-query.tool.d.ts +16 -16
- package/dist/mcp-server/tools/definitions/dataframe-query.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/dataframe-query.tool.js +17 -17
- package/dist/mcp-server/tools/definitions/dataframe-query.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.d.ts +26 -12
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.js +15 -37
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/find-holders.tool.d.ts +36 -20
- package/dist/mcp-server/tools/definitions/find-holders.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/find-holders.tool.js +26 -41
- package/dist/mcp-server/tools/definitions/find-holders.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.d.ts +28 -12
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.js +30 -44
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-filing.tool.d.ts +50 -14
- package/dist/mcp-server/tools/definitions/get-filing.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-filing.tool.js +193 -59
- package/dist/mcp-server/tools/definitions/get-filing.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-financials.tool.d.ts +14 -6
- package/dist/mcp-server/tools/definitions/get-financials.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-financials.tool.js +38 -41
- package/dist/mcp-server/tools/definitions/get-financials.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-fund-holdings.tool.d.ts +28 -12
- package/dist/mcp-server/tools/definitions/get-fund-holdings.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-fund-holdings.tool.js +22 -35
- package/dist/mcp-server/tools/definitions/get-fund-holdings.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.d.ts +28 -12
- package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.js +22 -31
- package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.d.ts +36 -20
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.js +22 -39
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-material-events.tool.d.ts +28 -12
- package/dist/mcp-server/tools/definitions/get-material-events.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-material-events.tool.js +23 -32
- package/dist/mcp-server/tools/definitions/get-material-events.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-snapshot.tool.d.ts +16 -7
- package/dist/mcp-server/tools/definitions/get-snapshot.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-snapshot.tool.js +24 -32
- package/dist/mcp-server/tools/definitions/get-snapshot.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/index.d.ts +232 -101
- package/dist/mcp-server/tools/definitions/index.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/search-concepts.tool.js +3 -3
- package/dist/mcp-server/tools/definitions/search-concepts.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/search-filings.tool.d.ts +46 -20
- package/dist/mcp-server/tools/definitions/search-filings.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/search-filings.tool.js +126 -68
- package/dist/mcp-server/tools/definitions/search-filings.tool.js.map +1 -1
- package/dist/mcp-server/tools/resolved-as.d.ts +10 -0
- package/dist/mcp-server/tools/resolved-as.d.ts.map +1 -0
- package/dist/mcp-server/tools/resolved-as.js +13 -0
- package/dist/mcp-server/tools/resolved-as.js.map +1 -0
- package/dist/mcp-server/tools/sec-upstream-errors.d.ts +44 -0
- package/dist/mcp-server/tools/sec-upstream-errors.d.ts.map +1 -0
- package/dist/mcp-server/tools/sec-upstream-errors.js +48 -0
- package/dist/mcp-server/tools/sec-upstream-errors.js.map +1 -0
- package/dist/services/canvas-bridge/canvas-bridge.d.ts +98 -45
- package/dist/services/canvas-bridge/canvas-bridge.d.ts.map +1 -1
- package/dist/services/canvas-bridge/canvas-bridge.js +137 -126
- package/dist/services/canvas-bridge/canvas-bridge.js.map +1 -1
- package/dist/services/canvas-bridge/sql-gate-extras.d.ts +4 -17
- package/dist/services/canvas-bridge/sql-gate-extras.d.ts.map +1 -1
- package/dist/services/canvas-bridge/sql-gate-extras.js +4 -22
- package/dist/services/canvas-bridge/sql-gate-extras.js.map +1 -1
- package/dist/services/edgar/concept-map.d.ts +10 -6
- package/dist/services/edgar/concept-map.d.ts.map +1 -1
- package/dist/services/edgar/concept-map.js +90 -22
- package/dist/services/edgar/concept-map.js.map +1 -1
- package/dist/services/edgar/data/former-tickers.json +24159 -0
- package/dist/services/edgar/edgar-api-service.d.ts +250 -62
- package/dist/services/edgar/edgar-api-service.d.ts.map +1 -1
- package/dist/services/edgar/edgar-api-service.js +808 -189
- package/dist/services/edgar/edgar-api-service.js.map +1 -1
- package/dist/services/edgar/filing-to-text.d.ts +71 -2
- package/dist/services/edgar/filing-to-text.d.ts.map +1 -1
- package/dist/services/edgar/filing-to-text.js +533 -5
- package/dist/services/edgar/filing-to-text.js.map +1 -1
- package/dist/services/edgar/former-tickers.d.ts +47 -0
- package/dist/services/edgar/former-tickers.d.ts.map +1 -0
- package/dist/services/edgar/former-tickers.js +77 -0
- package/dist/services/edgar/former-tickers.js.map +1 -0
- package/dist/services/edgar/listing-codes.d.ts +47 -0
- package/dist/services/edgar/listing-codes.d.ts.map +1 -0
- package/dist/services/edgar/listing-codes.js +110 -0
- package/dist/services/edgar/listing-codes.js.map +1 -0
- package/dist/services/edgar/mirror/edgar-mirror.d.ts +11 -1
- package/dist/services/edgar/mirror/edgar-mirror.d.ts.map +1 -1
- package/dist/services/edgar/mirror/edgar-mirror.js +19 -1
- package/dist/services/edgar/mirror/edgar-mirror.js.map +1 -1
- package/dist/services/edgar/mirror/tickers-sync.d.ts.map +1 -1
- package/dist/services/edgar/mirror/tickers-sync.js +1 -3
- package/dist/services/edgar/mirror/tickers-sync.js.map +1 -1
- package/dist/services/edgar/mirror/types.d.ts +3 -1
- package/dist/services/edgar/mirror/types.d.ts.map +1 -1
- package/dist/services/edgar/mirror/types.js +3 -1
- package/dist/services/edgar/mirror/types.js.map +1 -1
- package/dist/services/edgar/trigram-similarity.d.ts +5 -3
- package/dist/services/edgar/trigram-similarity.d.ts.map +1 -1
- package/dist/services/edgar/trigram-similarity.js +18 -17
- package/dist/services/edgar/trigram-similarity.js.map +1 -1
- package/dist/services/edgar/types.d.ts +20 -7
- package/dist/services/edgar/types.d.ts.map +1 -1
- package/package.json +6 -5
- package/server.json +3 -3
- package/skills/secedgar-guide/SKILL.md +3 -3
- package/dist/mcp-server/tools/escape-inline.d.ts +0 -35
- package/dist/mcp-server/tools/escape-inline.d.ts.map +0 -1
- package/dist/mcp-server/tools/escape-inline.js +0 -51
- package/dist/mcp-server/tools/escape-inline.js.map +0 -1
|
@@ -227,6 +227,346 @@ function stripInlineXbrl(html) {
|
|
|
227
227
|
cleaned = cleaned.replace(/<\/?ix:[^>]*>/gi, '');
|
|
228
228
|
return cleaned;
|
|
229
229
|
}
|
|
230
|
+
/**
|
|
231
|
+
* The sentinels that frame a marker's index in the input `convert()` receives
|
|
232
|
+
* (#239): two noncharacters, distinct so the strip pass reads the decimal index
|
|
233
|
+
* between them. The parser passes them through as text and no case transform
|
|
234
|
+
* changes them or the digits, so each lands in the output where its text does.
|
|
235
|
+
*/
|
|
236
|
+
const SENTINEL_OPEN = '';
|
|
237
|
+
const SENTINEL_CLOSE = '';
|
|
238
|
+
/**
|
|
239
|
+
* html-to-text's default `maxInputLength`, which {@link CONVERT_OPTIONS} keeps:
|
|
240
|
+
* longer input is cut silently, so marked input past it would convert
|
|
241
|
+
* differently from the input it marks.
|
|
242
|
+
*/
|
|
243
|
+
const MAX_CONVERT_INPUT_CHARS = 16_777_216;
|
|
244
|
+
/**
|
|
245
|
+
* The named references the parser decodes to one of its whitespace characters:
|
|
246
|
+
* a tab, a line feed, and five spellings of U+200B. Named references are
|
|
247
|
+
* case-sensitive, so `&tab;` stays text.
|
|
248
|
+
*/
|
|
249
|
+
const WHITESPACE_NAMED_REFERENCES = [
|
|
250
|
+
'	',
|
|
251
|
+
'
',
|
|
252
|
+
'​',
|
|
253
|
+
'​',
|
|
254
|
+
'​',
|
|
255
|
+
'​',
|
|
256
|
+
'​',
|
|
257
|
+
];
|
|
258
|
+
/**
|
|
259
|
+
* Whether a code unit is one of html-to-text's whitespace characters
|
|
260
|
+
* (`whitespaceCharacters`: space, tab, CR, LF, form feed, U+200B), which it
|
|
261
|
+
* collapses and trims. NBSP is not one: the parser keeps it as text.
|
|
262
|
+
*/
|
|
263
|
+
function isConvertWhitespace(code) {
|
|
264
|
+
return (code === 0x20 ||
|
|
265
|
+
code === 0x09 ||
|
|
266
|
+
code === 0x0a ||
|
|
267
|
+
code === 0x0c ||
|
|
268
|
+
code === 0x0d ||
|
|
269
|
+
code === 0x200b);
|
|
270
|
+
}
|
|
271
|
+
/** Whether a code unit is an ASCII letter — the first character of a tag name. */
|
|
272
|
+
function isAsciiLetter(code) {
|
|
273
|
+
return (code >= 0x41 && code <= 0x5a) || (code >= 0x61 && code <= 0x7a);
|
|
274
|
+
}
|
|
275
|
+
/** Whether a code unit is HTML's ASCII whitespace, which ends a tag or attribute name and an unquoted value. */
|
|
276
|
+
function isHtmlSpace(code) {
|
|
277
|
+
return code === 0x20 || code === 0x09 || code === 0x0a || code === 0x0c || code === 0x0d;
|
|
278
|
+
}
|
|
279
|
+
/**
|
|
280
|
+
* The code point a numeric character reference at `at` (`&#…`) decodes to, and
|
|
281
|
+
* the index just past it, or undefined when `at` opens none. It takes every
|
|
282
|
+
* digit that follows and its `;` is optional, as the parser reads it: ` ` is
|
|
283
|
+
* a space, and `›` is U+009B, not a tab and a `b`. A value past U+10FFFF,
|
|
284
|
+
* which the parser replaces, is clamped there.
|
|
285
|
+
*/
|
|
286
|
+
function numericReference(html, at) {
|
|
287
|
+
const hex = (html.charCodeAt(at + 2) | 0x20) === 0x78; /* x or X */
|
|
288
|
+
const digits = at + (hex ? 3 : 2);
|
|
289
|
+
let end = digits;
|
|
290
|
+
let value = 0;
|
|
291
|
+
for (; end < html.length; end++) {
|
|
292
|
+
const digit = Number.parseInt(html.charAt(end), hex ? 16 : 10);
|
|
293
|
+
if (Number.isNaN(digit))
|
|
294
|
+
break;
|
|
295
|
+
value = Math.min(value * (hex ? 16 : 10) + digit, 0x110000);
|
|
296
|
+
}
|
|
297
|
+
if (end === digits)
|
|
298
|
+
return;
|
|
299
|
+
return { end: html.charCodeAt(end) === 0x3b /* ; */ ? end + 1 : end, value };
|
|
300
|
+
}
|
|
301
|
+
/**
|
|
302
|
+
* The index just past a character reference at `at` that the parser decodes to
|
|
303
|
+
* one of its whitespace characters, or -1 when the reference at `at` decodes to
|
|
304
|
+
* anything else.
|
|
305
|
+
*/
|
|
306
|
+
function whitespaceReferenceEnd(html, at) {
|
|
307
|
+
if (html.charCodeAt(at + 1) !== 0x23 /* # */) {
|
|
308
|
+
const named = WHITESPACE_NAMED_REFERENCES.find((name) => html.startsWith(name, at));
|
|
309
|
+
return named === undefined ? -1 : at + named.length;
|
|
310
|
+
}
|
|
311
|
+
const reference = numericReference(html, at);
|
|
312
|
+
return reference && isConvertWhitespace(reference.value) ? reference.end : -1;
|
|
313
|
+
}
|
|
314
|
+
/**
|
|
315
|
+
* Whether `input` holds a sentinel, literal or as a numeric reference the parser
|
|
316
|
+
* decodes to one (``, ``). One forward pass over the `&#` openers.
|
|
317
|
+
*/
|
|
318
|
+
function holdsSentinel(input) {
|
|
319
|
+
if (input.includes(SENTINEL_OPEN) || input.includes(SENTINEL_CLOSE))
|
|
320
|
+
return true;
|
|
321
|
+
for (let at = input.indexOf('&#'); at !== -1;) {
|
|
322
|
+
const reference = numericReference(input, at);
|
|
323
|
+
if (reference?.value === 0xfdd0 || reference?.value === 0xfdd1)
|
|
324
|
+
return true;
|
|
325
|
+
at = input.indexOf('&#', reference?.end ?? at + 2);
|
|
326
|
+
}
|
|
327
|
+
return false;
|
|
328
|
+
}
|
|
329
|
+
/** The index of the first `</name` at or after `from`, any case, or -1 — a `script` or `style` body's end. */
|
|
330
|
+
function rawTextEnd(html, name, from) {
|
|
331
|
+
for (let at = html.indexOf('</', from); at !== -1; at = html.indexOf('</', at + 2)) {
|
|
332
|
+
if (html.slice(at + 2, at + 2 + name.length).toLowerCase() === name)
|
|
333
|
+
return at;
|
|
334
|
+
}
|
|
335
|
+
return -1;
|
|
336
|
+
}
|
|
337
|
+
/**
|
|
338
|
+
* The `href`, `id`, and `name` attributes of a start tag whose attributes run
|
|
339
|
+
* from `from` to `to` (the closing `>`), read by hand: names are
|
|
340
|
+
* case-insensitive, values quoted with `"` or `'` or unquoted, and a quote left
|
|
341
|
+
* open ends at `to`. Every index is read once, so the pass stays linear.
|
|
342
|
+
*/
|
|
343
|
+
function readTagAnchors(html, tag, from, to) {
|
|
344
|
+
const anchors = {};
|
|
345
|
+
const endsName = (code) => isHtmlSpace(code) || code === 0x2f /* / */ || code === 0x3d; /* = */
|
|
346
|
+
for (let at = from; at < to;) {
|
|
347
|
+
if (endsName(html.charCodeAt(at))) {
|
|
348
|
+
at++;
|
|
349
|
+
continue;
|
|
350
|
+
}
|
|
351
|
+
const nameStart = at;
|
|
352
|
+
while (at < to && !endsName(html.charCodeAt(at)))
|
|
353
|
+
at++;
|
|
354
|
+
const attribute = html.slice(nameStart, at).toLowerCase();
|
|
355
|
+
while (at < to && isHtmlSpace(html.charCodeAt(at)))
|
|
356
|
+
at++;
|
|
357
|
+
if (at >= to || html.charCodeAt(at) !== 0x3d /* = */)
|
|
358
|
+
continue;
|
|
359
|
+
at++;
|
|
360
|
+
while (at < to && isHtmlSpace(html.charCodeAt(at)))
|
|
361
|
+
at++;
|
|
362
|
+
const quote = html.charAt(at);
|
|
363
|
+
let value;
|
|
364
|
+
if (quote === '"' || quote === "'") {
|
|
365
|
+
let close = at + 1;
|
|
366
|
+
while (close < to && html.charAt(close) !== quote)
|
|
367
|
+
close++;
|
|
368
|
+
value = html.slice(at + 1, close);
|
|
369
|
+
at = close + 1;
|
|
370
|
+
}
|
|
371
|
+
else {
|
|
372
|
+
const valueStart = at;
|
|
373
|
+
while (at < to && !isHtmlSpace(html.charCodeAt(at)))
|
|
374
|
+
at++;
|
|
375
|
+
value = html.slice(valueStart, at);
|
|
376
|
+
}
|
|
377
|
+
if (attribute === 'id')
|
|
378
|
+
anchors.id ??= value;
|
|
379
|
+
else if (tag === 'a' && attribute === 'name')
|
|
380
|
+
anchors.name ??= value;
|
|
381
|
+
else if (tag === 'a' && attribute === 'href') {
|
|
382
|
+
const href = value.trim();
|
|
383
|
+
if (href.length > 1 && href.startsWith('#'))
|
|
384
|
+
anchors.href ??= href.slice(1);
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
return anchors;
|
|
388
|
+
}
|
|
389
|
+
/**
|
|
390
|
+
* Every fragment link and fragment target in `html`, each placed at the first
|
|
391
|
+
* character the parser renders as text after its start tag: tags, comments,
|
|
392
|
+
* `script` and `style` bodies, and its whitespace — literal or a reference — are
|
|
393
|
+
* passed over, so a mark there joins the next word instead of standing alone. A
|
|
394
|
+
* mark right after the tag would leave a blank line where an empty target `div`
|
|
395
|
+
* sits, and a doubled space after an empty link before a `​` cell.
|
|
396
|
+
*
|
|
397
|
+
* One forward pass: a `<` opening a tag (a letter, `/`, `!`, or `?` after it)
|
|
398
|
+
* reads to the next `>`, and one with no `>` after it ends the scan, since the
|
|
399
|
+
* parser renders nothing past it either; a `script` or `style` body runs to its
|
|
400
|
+
* closer, and one with no closer ends the scan too. Anchors waiting for text are
|
|
401
|
+
* held until the next text character, which places them all at once; while none
|
|
402
|
+
* waits, the scan jumps from text to the next `<`.
|
|
403
|
+
*/
|
|
404
|
+
function scanMarkupAnchors(html) {
|
|
405
|
+
const anchors = [];
|
|
406
|
+
let waiting = 0;
|
|
407
|
+
const place = (at) => {
|
|
408
|
+
for (let k = anchors.length - waiting; k < anchors.length; k++) {
|
|
409
|
+
const anchor = anchors[k];
|
|
410
|
+
if (anchor)
|
|
411
|
+
anchor.at = at;
|
|
412
|
+
}
|
|
413
|
+
waiting = 0;
|
|
414
|
+
};
|
|
415
|
+
for (let at = 0; at < html.length;) {
|
|
416
|
+
const code = html.charCodeAt(at);
|
|
417
|
+
if (code === 0x3c /* < */) {
|
|
418
|
+
const next = html.charCodeAt(at + 1);
|
|
419
|
+
const opensTag = isAsciiLetter(next);
|
|
420
|
+
if (opensTag || next === 0x2f /* / */ || next === 0x21 /* ! */ || next === 0x3f /* ? */) {
|
|
421
|
+
if (next === 0x21 && html.startsWith('--', at + 2)) {
|
|
422
|
+
// `<!-->` and `<!--->` close at once, as the parser reads them.
|
|
423
|
+
const close = html.indexOf('-->', at + 2);
|
|
424
|
+
if (close === -1)
|
|
425
|
+
break;
|
|
426
|
+
at = close + 3;
|
|
427
|
+
continue;
|
|
428
|
+
}
|
|
429
|
+
const end = html.indexOf('>', at + 1);
|
|
430
|
+
if (end === -1)
|
|
431
|
+
break;
|
|
432
|
+
if (!opensTag) {
|
|
433
|
+
at = end + 1;
|
|
434
|
+
continue;
|
|
435
|
+
}
|
|
436
|
+
let nameEnd = at + 1;
|
|
437
|
+
while (nameEnd < end &&
|
|
438
|
+
!isHtmlSpace(html.charCodeAt(nameEnd)) &&
|
|
439
|
+
html.charCodeAt(nameEnd) !== 0x2f /* / */) {
|
|
440
|
+
nameEnd++;
|
|
441
|
+
}
|
|
442
|
+
const tag = html.slice(at + 1, nameEnd).toLowerCase();
|
|
443
|
+
const found = readTagAnchors(html, tag, nameEnd, end);
|
|
444
|
+
for (const kind of ['href', 'id', 'name']) {
|
|
445
|
+
const fragment = found[kind];
|
|
446
|
+
if (fragment === undefined)
|
|
447
|
+
continue;
|
|
448
|
+
anchors.push({ at: -1, fragment, kind });
|
|
449
|
+
waiting++;
|
|
450
|
+
}
|
|
451
|
+
at = end + 1;
|
|
452
|
+
if (tag === 'script' || tag === 'style') {
|
|
453
|
+
at = rawTextEnd(html, tag, at);
|
|
454
|
+
if (at === -1)
|
|
455
|
+
break;
|
|
456
|
+
}
|
|
457
|
+
continue;
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
else if (waiting === 0) {
|
|
461
|
+
const lt = html.indexOf('<', at + 1);
|
|
462
|
+
if (lt === -1)
|
|
463
|
+
break;
|
|
464
|
+
at = lt;
|
|
465
|
+
continue;
|
|
466
|
+
}
|
|
467
|
+
else if (isConvertWhitespace(code)) {
|
|
468
|
+
at++;
|
|
469
|
+
continue;
|
|
470
|
+
}
|
|
471
|
+
else if (code === 0x26 /* & */) {
|
|
472
|
+
const end = whitespaceReferenceEnd(html, at);
|
|
473
|
+
if (end !== -1) {
|
|
474
|
+
at = end;
|
|
475
|
+
continue;
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
// A text character: a `<` that opens no tag, an `&` that is no whitespace, anything else.
|
|
479
|
+
if (waiting > 0)
|
|
480
|
+
place(at);
|
|
481
|
+
at++;
|
|
482
|
+
}
|
|
483
|
+
return anchors;
|
|
484
|
+
}
|
|
485
|
+
/**
|
|
486
|
+
* Mark `input` — the string `convert()` receives — with a sentinel before each
|
|
487
|
+
* in-document link's and each linked target's first text character (#239).
|
|
488
|
+
* `undefined` when there is nothing to mark, or when marking could change the
|
|
489
|
+
* conversion: the input already carries a sentinel, literal or as a reference,
|
|
490
|
+
* or the marked input would pass {@link MAX_CONVERT_INPUT_CHARS}.
|
|
491
|
+
*/
|
|
492
|
+
function markInDocumentLinks(input) {
|
|
493
|
+
if (holdsSentinel(input))
|
|
494
|
+
return;
|
|
495
|
+
const anchors = scanMarkupAnchors(input);
|
|
496
|
+
const byId = new Map();
|
|
497
|
+
const byName = new Map();
|
|
498
|
+
for (const anchor of anchors) {
|
|
499
|
+
if (anchor.kind === 'id' && !byId.has(anchor.fragment))
|
|
500
|
+
byId.set(anchor.fragment, anchor);
|
|
501
|
+
if (anchor.kind === 'name' && !byName.has(anchor.fragment))
|
|
502
|
+
byName.set(anchor.fragment, anchor);
|
|
503
|
+
}
|
|
504
|
+
const markerOf = new Map();
|
|
505
|
+
const pairs = [];
|
|
506
|
+
for (const anchor of anchors) {
|
|
507
|
+
if (anchor.kind !== 'href' || anchor.at === -1)
|
|
508
|
+
continue;
|
|
509
|
+
const target = byId.get(anchor.fragment) ?? byName.get(anchor.fragment);
|
|
510
|
+
if (target === undefined || target.at === -1)
|
|
511
|
+
continue;
|
|
512
|
+
pairs.push([anchor, target]);
|
|
513
|
+
markerOf.set(anchor, -1);
|
|
514
|
+
markerOf.set(target, -1);
|
|
515
|
+
}
|
|
516
|
+
if (pairs.length === 0)
|
|
517
|
+
return;
|
|
518
|
+
// Anchors come out of the scan in input order with nondecreasing positions,
|
|
519
|
+
// so the marked string is built in one forward pass.
|
|
520
|
+
const parts = [];
|
|
521
|
+
let copied = 0;
|
|
522
|
+
let markers = 0;
|
|
523
|
+
let length = input.length;
|
|
524
|
+
for (const anchor of anchors) {
|
|
525
|
+
if (!markerOf.has(anchor))
|
|
526
|
+
continue;
|
|
527
|
+
const sentinel = `${SENTINEL_OPEN}${markers}${SENTINEL_CLOSE}`;
|
|
528
|
+
markerOf.set(anchor, markers++);
|
|
529
|
+
length += sentinel.length;
|
|
530
|
+
if (length > MAX_CONVERT_INPUT_CHARS)
|
|
531
|
+
return;
|
|
532
|
+
parts.push(input.slice(copied, anchor.at), sentinel);
|
|
533
|
+
copied = anchor.at;
|
|
534
|
+
}
|
|
535
|
+
parts.push(input.slice(copied));
|
|
536
|
+
return {
|
|
537
|
+
links: pairs.map(([link, target]) => ({
|
|
538
|
+
marker: markerOf.get(link) ?? -1,
|
|
539
|
+
target: markerOf.get(target) ?? -1,
|
|
540
|
+
})),
|
|
541
|
+
marked: parts.join(''),
|
|
542
|
+
markers,
|
|
543
|
+
};
|
|
544
|
+
}
|
|
545
|
+
/**
|
|
546
|
+
* The converted text with every sentinel removed, and each marker's offset in
|
|
547
|
+
* that text (-1 for a marker whose text the parser never rendered, such as one
|
|
548
|
+
* inside an attribute value). One forward pass. Every sentinel in the output is
|
|
549
|
+
* one {@link markInDocumentLinks} placed — it refuses input that already holds
|
|
550
|
+
* one — so an opener without its closer is a broken invariant, not filing text.
|
|
551
|
+
*/
|
|
552
|
+
function stripSentinels(converted, markers) {
|
|
553
|
+
const offsets = new Int32Array(markers).fill(-1);
|
|
554
|
+
const parts = [];
|
|
555
|
+
let length = 0;
|
|
556
|
+
let copied = 0;
|
|
557
|
+
for (let open = converted.indexOf(SENTINEL_OPEN); open !== -1;) {
|
|
558
|
+
const close = converted.indexOf(SENTINEL_CLOSE, open + 1);
|
|
559
|
+
if (close === -1)
|
|
560
|
+
throw new Error('Unclosed in-document link sentinel in converted filing text');
|
|
561
|
+
parts.push(converted.slice(copied, open));
|
|
562
|
+
length += open - copied;
|
|
563
|
+
offsets[Number(converted.slice(open + 1, close))] = length;
|
|
564
|
+
copied = close + 1;
|
|
565
|
+
open = converted.indexOf(SENTINEL_OPEN, copied);
|
|
566
|
+
}
|
|
567
|
+
parts.push(converted.slice(copied));
|
|
568
|
+
return { offsets, text: parts.join('') };
|
|
569
|
+
}
|
|
230
570
|
/**
|
|
231
571
|
* Extract full plain text from a filing document — HTML, or an SGML submission,
|
|
232
572
|
* every readable document in order with plain-text `<TEXT>` bodies keeping their
|
|
@@ -234,9 +574,30 @@ function stripInlineXbrl(html) {
|
|
|
234
574
|
* input always yields the same string. Caching is the caller's responsibility via
|
|
235
575
|
* `getExtractCache`/`setExtractCache` (keyed `accession:document`), which lets a
|
|
236
576
|
* cache hit skip the document fetch as well as this conversion.
|
|
577
|
+
*
|
|
578
|
+
* Given `links`, the document's in-document links are pushed onto it in text
|
|
579
|
+
* order (#239): sentinels marking each link and target go into the input
|
|
580
|
+
* `convert()` receives, before the first character it renders as text after the
|
|
581
|
+
* start tag ({@link scanMarkupAnchors}), and come back out of its output in one
|
|
582
|
+
* pass, so the text is byte-identical to the unmarked conversion. Input holding
|
|
583
|
+
* a sentinel already, or too long to mark within the parser's input limit, is
|
|
584
|
+
* converted unmarked and records no links. One conversion either way.
|
|
237
585
|
*/
|
|
238
|
-
export function filingToExtract(html) {
|
|
239
|
-
|
|
586
|
+
export function filingToExtract(html, links) {
|
|
587
|
+
const input = preservePlainTextBodies(neutralizePageMarkers(stripInlineXbrl(html)));
|
|
588
|
+
if (links === undefined)
|
|
589
|
+
return convert(input, CONVERT_OPTIONS);
|
|
590
|
+
const marking = markInDocumentLinks(input);
|
|
591
|
+
if (marking === undefined)
|
|
592
|
+
return convert(input, CONVERT_OPTIONS);
|
|
593
|
+
const { offsets, text } = stripSentinels(convert(marking.marked, CONVERT_OPTIONS), marking.markers);
|
|
594
|
+
for (const link of marking.links) {
|
|
595
|
+
const offset = offsets[link.marker] ?? -1;
|
|
596
|
+
const target = offsets[link.target] ?? -1;
|
|
597
|
+
if (offset !== -1 && target !== -1)
|
|
598
|
+
links.push({ offset, target });
|
|
599
|
+
}
|
|
600
|
+
return text;
|
|
240
601
|
}
|
|
241
602
|
/** Return true if the cache has an entry for cacheKey (allows skipping the document fetch). */
|
|
242
603
|
export function hasExtractCache(cacheKey) {
|
|
@@ -882,6 +1243,113 @@ export function foldForHeadingMatch(value) {
|
|
|
882
1243
|
.trim()
|
|
883
1244
|
.toLowerCase();
|
|
884
1245
|
}
|
|
1246
|
+
/** One whitespace code point, as `\s` reads it — the run {@link foldForHeadingMatch} collapses. */
|
|
1247
|
+
const WHITESPACE_CHAR_RE = /^\s$/u;
|
|
1248
|
+
/**
|
|
1249
|
+
* The {@link foldForHeadingMatch} fold of one code point that is not
|
|
1250
|
+
* whitespace: a typographic quote becomes its ASCII counterpart, anything else
|
|
1251
|
+
* its lowercase form, which can be longer than the source (U+0130 lowercases to
|
|
1252
|
+
* two code units).
|
|
1253
|
+
*/
|
|
1254
|
+
function foldCodePoint(codePoint) {
|
|
1255
|
+
if (codePoint === 0x2018 || codePoint === 0x2019)
|
|
1256
|
+
return "'";
|
|
1257
|
+
if (codePoint === 0x201c || codePoint === 0x201d)
|
|
1258
|
+
return '"';
|
|
1259
|
+
return String.fromCodePoint(codePoint).toLowerCase();
|
|
1260
|
+
}
|
|
1261
|
+
/**
|
|
1262
|
+
* Feed `text` through the {@link foldForHeadingMatch} fold one code point at a
|
|
1263
|
+
* time: each whitespace run as one space, every other code point as its folded
|
|
1264
|
+
* code units, each unit with the offset and end of the code point it came from.
|
|
1265
|
+
* Folding per code point is what lets every unit map back to a document offset,
|
|
1266
|
+
* since a lowercase form can be longer than its source.
|
|
1267
|
+
*/
|
|
1268
|
+
function foldEach(text, emit) {
|
|
1269
|
+
let inSpace = false;
|
|
1270
|
+
for (let at = 0; at < text.length;) {
|
|
1271
|
+
const code = text.charCodeAt(at);
|
|
1272
|
+
if (code < 0x80) {
|
|
1273
|
+
const space = code === 0x20 || (code >= 0x09 && code <= 0x0d);
|
|
1274
|
+
if (!space)
|
|
1275
|
+
emit(code >= 0x41 && code <= 0x5a ? code + 0x20 : code, at, at + 1);
|
|
1276
|
+
else if (!inSpace)
|
|
1277
|
+
emit(0x20, at, at + 1);
|
|
1278
|
+
inSpace = space;
|
|
1279
|
+
at++;
|
|
1280
|
+
continue;
|
|
1281
|
+
}
|
|
1282
|
+
const codePoint = text.codePointAt(at) ?? code;
|
|
1283
|
+
const end = at + (codePoint > 0xffff ? 2 : 1);
|
|
1284
|
+
const char = text.slice(at, end);
|
|
1285
|
+
if (WHITESPACE_CHAR_RE.test(char)) {
|
|
1286
|
+
if (!inSpace)
|
|
1287
|
+
emit(0x20, at, end);
|
|
1288
|
+
inSpace = true;
|
|
1289
|
+
}
|
|
1290
|
+
else {
|
|
1291
|
+
const folded = foldCodePoint(codePoint);
|
|
1292
|
+
for (let k = 0; k < folded.length; k++)
|
|
1293
|
+
emit(folded.charCodeAt(k), at, end);
|
|
1294
|
+
inSpace = false;
|
|
1295
|
+
}
|
|
1296
|
+
at = end;
|
|
1297
|
+
}
|
|
1298
|
+
}
|
|
1299
|
+
/**
|
|
1300
|
+
* Every occurrence of `phrase` in `text` under the {@link foldForHeadingMatch}
|
|
1301
|
+
* fold (#238) — case, whitespace runs, and quote style ignored, so a phrase
|
|
1302
|
+
* broken across a line or set with a non-breaking space matches — counted over
|
|
1303
|
+
* the whole text, with the first `limit` that start at or after `from` listed by
|
|
1304
|
+
* their document offsets. Substring matching; occurrences do not overlap.
|
|
1305
|
+
* `phrase` must hold something besides whitespace.
|
|
1306
|
+
*
|
|
1307
|
+
* Both operands fold one code point at a time ({@link foldEach}), and each
|
|
1308
|
+
* folded unit keeps the offset of the code point it came from, so a lowercase
|
|
1309
|
+
* form longer than its source (U+0130) shifts no later offset. One forward pass
|
|
1310
|
+
* of Knuth–Morris–Pratt over the folded units, holding the offsets of the last
|
|
1311
|
+
* `phrase`-length units in a ring: linear in the text, and memory bounded by
|
|
1312
|
+
* the phrase and `limit`, whatever the occurrence count.
|
|
1313
|
+
*/
|
|
1314
|
+
export function findPhrase(text, phrase, from, limit) {
|
|
1315
|
+
const units = [];
|
|
1316
|
+
foldEach(phrase, (unit) => {
|
|
1317
|
+
if (unit !== 0x20 || (units.length > 0 && units[units.length - 1] !== 0x20))
|
|
1318
|
+
units.push(unit);
|
|
1319
|
+
});
|
|
1320
|
+
if (units[units.length - 1] === 0x20)
|
|
1321
|
+
units.pop();
|
|
1322
|
+
const length = units.length;
|
|
1323
|
+
const fallback = new Int32Array(length);
|
|
1324
|
+
for (let i = 1, k = 0; i < length; i++) {
|
|
1325
|
+
while (k > 0 && units[i] !== units[k])
|
|
1326
|
+
k = fallback[k - 1] ?? 0;
|
|
1327
|
+
if (units[i] === units[k])
|
|
1328
|
+
k++;
|
|
1329
|
+
fallback[i] = k;
|
|
1330
|
+
}
|
|
1331
|
+
const starts = new Int32Array(length);
|
|
1332
|
+
const matches = { before: 0, listed: [], total: 0 };
|
|
1333
|
+
let seen = 0;
|
|
1334
|
+
let matched = 0;
|
|
1335
|
+
foldEach(text, (unit, offset, end) => {
|
|
1336
|
+
starts[seen++ % length] = offset;
|
|
1337
|
+
while (matched > 0 && unit !== units[matched])
|
|
1338
|
+
matched = fallback[matched - 1] ?? 0;
|
|
1339
|
+
if (unit === units[matched])
|
|
1340
|
+
matched++;
|
|
1341
|
+
if (matched < length)
|
|
1342
|
+
return;
|
|
1343
|
+
matched = 0;
|
|
1344
|
+
const start = starts[seen % length] ?? offset;
|
|
1345
|
+
matches.total++;
|
|
1346
|
+
if (start < from)
|
|
1347
|
+
matches.before++;
|
|
1348
|
+
else if (matches.listed.length < limit)
|
|
1349
|
+
matches.listed.push({ offset: start, end });
|
|
1350
|
+
});
|
|
1351
|
+
return matches;
|
|
1352
|
+
}
|
|
885
1353
|
/**
|
|
886
1354
|
* A TOC page cell on a line of its own: a page number, a lettered
|
|
887
1355
|
* financial-statement page ("F-3"), or a lowercase roman front-matter page
|
|
@@ -993,9 +1461,7 @@ export function isTocHeading(text, heading) {
|
|
|
993
1461
|
const lineStart = text.lastIndexOf('\n', heading.offset - 1) + 1;
|
|
994
1462
|
const newline = text.indexOf('\n', heading.offset);
|
|
995
1463
|
const lineEnd = newline === -1 ? text.length : newline;
|
|
996
|
-
const
|
|
997
|
-
text.startsWith(heading.heading, heading.offset);
|
|
998
|
-
const title = titledHere
|
|
1464
|
+
const title = isTitledAt(text, heading)
|
|
999
1465
|
? { line: text.slice(lineStart, lineEnd), end: lineEnd + 1 }
|
|
1000
1466
|
: nextContentLine(text, lineEnd + 1);
|
|
1001
1467
|
if (title === undefined)
|
|
@@ -1019,6 +1485,68 @@ export function isTocHeading(text, heading) {
|
|
|
1019
1485
|
const pageEnd = text.indexOf('\n', below.end);
|
|
1020
1486
|
return PAGE_CELL_LINE_RE.test(text.slice(below.end, pageEnd === -1 ? text.length : pageEnd));
|
|
1021
1487
|
}
|
|
1488
|
+
/**
|
|
1489
|
+
* Whether a heading's text is printed at its offset — not a bare marker, nor a
|
|
1490
|
+
* marker joined to a title printed on a later line ({@link titleAfterMarkerLine}).
|
|
1491
|
+
*/
|
|
1492
|
+
function isTitledAt(text, heading) {
|
|
1493
|
+
return (!BARE_MARKER_HEADING_RE.test(heading.heading) &&
|
|
1494
|
+
text.startsWith(heading.heading, heading.offset));
|
|
1495
|
+
}
|
|
1496
|
+
/** The span of a {@link ContentLine}, its newline excluded. */
|
|
1497
|
+
function lineSpan(line) {
|
|
1498
|
+
return { start: line.end - 1 - line.line.length, end: line.end - 1 };
|
|
1499
|
+
}
|
|
1500
|
+
/**
|
|
1501
|
+
* Where `section` continues when it lands on a table-of-contents row
|
|
1502
|
+
* ({@link isTocHeading}) that links forward to its section (#239): the start of
|
|
1503
|
+
* the line holding the first text after the link's target, or undefined when
|
|
1504
|
+
* the heading is no TOC row or its row carries no such link — every other
|
|
1505
|
+
* landing stays where it is. A row is the heading's own line; for a heading not
|
|
1506
|
+
* printed at its offset (a bare marker joined to a later title), that title's
|
|
1507
|
+
* line too; then the next line with content when it is a page cell. A link
|
|
1508
|
+
* counts when its text starts on one of those lines and its target lies past
|
|
1509
|
+
* the row, so a back-link to the table of contents never moves a landing.
|
|
1510
|
+
*
|
|
1511
|
+
* Anthropic, PBC's S-1/A (0001628280-26-068676) prints its table of contents in
|
|
1512
|
+
* capitals, a detected heading, and each section's own heading in mixed case,
|
|
1513
|
+
* which detection does not list, so `risk factors` took the TOC row alone. Over
|
|
1514
|
+
* 61 sampled 2008–2026 filings × 60 section values this moves 20 landings, each
|
|
1515
|
+
* off a TOC row onto its section, and no other. `links` is in text order; one
|
|
1516
|
+
* pass up to the row's end.
|
|
1517
|
+
*/
|
|
1518
|
+
export function linkedSectionStart(text, heading, links) {
|
|
1519
|
+
if (links.length === 0 || !isTocHeading(text, heading))
|
|
1520
|
+
return;
|
|
1521
|
+
const newline = text.indexOf('\n', heading.offset);
|
|
1522
|
+
const own = {
|
|
1523
|
+
start: text.lastIndexOf('\n', heading.offset - 1) + 1,
|
|
1524
|
+
end: newline === -1 ? text.length : newline,
|
|
1525
|
+
};
|
|
1526
|
+
const row = [own];
|
|
1527
|
+
let after = own.end + 1;
|
|
1528
|
+
if (!isTitledAt(text, heading)) {
|
|
1529
|
+
const title = nextContentLine(text, after);
|
|
1530
|
+
if (title !== undefined) {
|
|
1531
|
+
row.push(lineSpan(title));
|
|
1532
|
+
after = title.end;
|
|
1533
|
+
}
|
|
1534
|
+
}
|
|
1535
|
+
const page = nextContentLine(text, after);
|
|
1536
|
+
if (page !== undefined && PAGE_CELL_LINE_RE.test(page.line))
|
|
1537
|
+
row.push(lineSpan(page));
|
|
1538
|
+
const rowEnd = row[row.length - 1]?.end ?? own.end;
|
|
1539
|
+
for (const link of links) {
|
|
1540
|
+
if (link.offset > rowEnd)
|
|
1541
|
+
break;
|
|
1542
|
+
if (link.target <= rowEnd)
|
|
1543
|
+
continue;
|
|
1544
|
+
if (row.some((line) => link.offset >= line.start && link.offset < line.end)) {
|
|
1545
|
+
return text.lastIndexOf('\n', link.target - 1) + 1;
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
return;
|
|
1549
|
+
}
|
|
1022
1550
|
/**
|
|
1023
1551
|
* Whether a prefix line runs on past a one-word needle into a longer Title Case
|
|
1024
1552
|
* title — the next word opens on a capital in a line that is not all caps. One
|