@cyanheads/secedgar-mcp-server 0.15.11 → 0.15.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +14 -12
- package/CLAUDE.md +14 -12
- package/Dockerfile +4 -4
- package/README.md +39 -39
- package/changelog/0.15.x/0.15.12.md +33 -0
- package/changelog/0.15.x/0.15.13.md +33 -0
- package/dist/config/server-config.d.ts.map +1 -1
- package/dist/config/server-config.js +47 -0
- package/dist/config/server-config.js.map +1 -1
- package/dist/mcp-server/prompts/definitions/company-analysis.prompt.js +1 -1
- package/dist/mcp-server/prompts/definitions/company-analysis.prompt.js.map +1 -1
- package/dist/mcp-server/tools/definitions/company-search.tool.d.ts +7 -0
- package/dist/mcp-server/tools/definitions/company-search.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/company-search.tool.js +76 -19
- package/dist/mcp-server/tools/definitions/company-search.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/compare-companies.tool.d.ts +7 -0
- package/dist/mcp-server/tools/definitions/compare-companies.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/compare-companies.tool.js +132 -38
- package/dist/mcp-server/tools/definitions/compare-companies.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/dataframe-query.tool.d.ts +10 -4
- package/dist/mcp-server/tools/definitions/dataframe-query.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/dataframe-query.tool.js +11 -4
- package/dist/mcp-server/tools/definitions/dataframe-query.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.d.ts +7 -0
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.js +8 -0
- package/dist/mcp-server/tools/definitions/fetch-frames.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/find-holders.tool.d.ts +7 -0
- package/dist/mcp-server/tools/definitions/find-holders.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/find-holders.tool.js +8 -0
- package/dist/mcp-server/tools/definitions/find-holders.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.d.ts +7 -0
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.js +8 -0
- package/dist/mcp-server/tools/definitions/get-beneficial-owners.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-filing.tool.d.ts +9 -2
- package/dist/mcp-server/tools/definitions/get-filing.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-filing.tool.js +308 -54
- package/dist/mcp-server/tools/definitions/get-filing.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-financials.tool.d.ts +10 -3
- package/dist/mcp-server/tools/definitions/get-financials.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-financials.tool.js +110 -41
- package/dist/mcp-server/tools/definitions/get-financials.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-fund-holdings.tool.d.ts +7 -0
- package/dist/mcp-server/tools/definitions/get-fund-holdings.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-fund-holdings.tool.js +8 -0
- package/dist/mcp-server/tools/definitions/get-fund-holdings.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.d.ts +7 -0
- package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.js +8 -0
- package/dist/mcp-server/tools/definitions/get-insider-transactions.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.d.ts +7 -0
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.js +8 -0
- package/dist/mcp-server/tools/definitions/get-institutional-holdings.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-material-events.tool.d.ts +7 -0
- package/dist/mcp-server/tools/definitions/get-material-events.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-material-events.tool.js +8 -0
- package/dist/mcp-server/tools/definitions/get-material-events.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/get-snapshot.tool.d.ts +7 -0
- package/dist/mcp-server/tools/definitions/get-snapshot.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/get-snapshot.tool.js +8 -0
- package/dist/mcp-server/tools/definitions/get-snapshot.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/index.d.ts +104 -7
- package/dist/mcp-server/tools/definitions/index.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/search-filings.tool.d.ts +7 -0
- package/dist/mcp-server/tools/definitions/search-filings.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/search-filings.tool.js +8 -0
- package/dist/mcp-server/tools/definitions/search-filings.tool.js.map +1 -1
- package/dist/services/canvas-bridge/canvas-bridge.d.ts +2 -2
- package/dist/services/canvas-bridge/canvas-bridge.js +2 -2
- package/dist/services/edgar/concept-series.d.ts +20 -8
- package/dist/services/edgar/concept-series.d.ts.map +1 -1
- package/dist/services/edgar/concept-series.js +74 -23
- package/dist/services/edgar/concept-series.js.map +1 -1
- package/dist/services/edgar/edgar-api-service.d.ts +21 -11
- package/dist/services/edgar/edgar-api-service.d.ts.map +1 -1
- package/dist/services/edgar/edgar-api-service.js +133 -34
- package/dist/services/edgar/edgar-api-service.js.map +1 -1
- package/dist/services/edgar/filing-to-text.d.ts +106 -6
- package/dist/services/edgar/filing-to-text.d.ts.map +1 -1
- package/dist/services/edgar/filing-to-text.js +570 -66
- package/dist/services/edgar/filing-to-text.js.map +1 -1
- package/package.json +9 -9
- package/server.json +4 -4
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
* @module services/edgar/filing-to-text
|
|
5
5
|
*/
|
|
6
6
|
import { convert } from 'html-to-text';
|
|
7
|
+
import { CURRENT_EIGHT_K_ITEMS } from './eight-k-items.js';
|
|
7
8
|
/**
|
|
8
9
|
* html-to-text walks the DOM recursively, so markup nested deeply enough overflows
|
|
9
10
|
* the stack (#118). `maxDepth` stops the walk and prints `ellipsis` in place of
|
|
@@ -248,51 +249,192 @@ export function clearExtractCache() {
|
|
|
248
249
|
export function extractCacheSize() {
|
|
249
250
|
return extractCache.size;
|
|
250
251
|
}
|
|
252
|
+
/**
|
|
253
|
+
* An Item marker's code: one or two digits with an optional letter suffix (the
|
|
254
|
+
* 20-F's `16A`–`16K`), or a current Form 8-K item's two-digit decimal (`5.02`,
|
|
255
|
+
* #165). Without the decimal, the `.02` of `Item 5.02` broke every Item pattern.
|
|
256
|
+
*/
|
|
257
|
+
const ITEM_CODE = String.raw `\d{1,2}(?:\.\d{2}|[a-z])?`;
|
|
251
258
|
/**
|
|
252
259
|
* Regexes that match SEC document headings, anchored to start-of-line:
|
|
253
260
|
* - All-caps forms — "ITEM N" / "ITEM NA" lines (10-K, 10-Q structure) and
|
|
254
261
|
* all-caps lines of 9+ characters (registration statement TOC headings like
|
|
255
|
-
* "RISK FACTORS"; older filings).
|
|
262
|
+
* "RISK FACTORS"; older filings). The all-caps line admits hyphens and en/em
|
|
263
|
+
* dashes ("NON-GAAP FINANCIAL MEASURES", "PART II — OTHER INFORMATION"),
|
|
264
|
+
* digits, and apostrophes; {@link isAllCapsHeadingText} then holds the last
|
|
265
|
+
* two to a section number and a possessive (#170). A colon stays out: every
|
|
266
|
+
* colon line it admitted across 26 sampled documents was a financial-statement
|
|
267
|
+
* caption ("CURRENT ASSETS:").
|
|
256
268
|
* - Mixed-case Item/Part forms — modern styled filings render headings as
|
|
257
269
|
* "Item 1A. Risk Factors" / "Part II" spans (#71). The Item arm requires a
|
|
258
|
-
* title after the marker on the same line and
|
|
259
|
-
* body paragraphs that begin with "Item ..." don't register.
|
|
270
|
+
* title after the marker on the same line, and {@link sameLineTitleFits} bounds
|
|
271
|
+
* it so unwrapped body paragraphs that begin with "Item ..." don't register. A
|
|
272
|
+
* dotted 8-K marker may also abut its title or carry a paragraph suffix
|
|
273
|
+
* ("Item 9.01Financial Statements…", "Item 9.01(d). Financial…"), and then
|
|
274
|
+
* the title must start with a letter, so a bare "Item 5.02." line never matches
|
|
275
|
+
* on its own (#165). A Part line may carry a title after whitespace and an
|
|
276
|
+
* optional dash, colon, or period ("Part II — Other Information"), which must
|
|
277
|
+
* start uppercase (#170).
|
|
260
278
|
* - Bare Item markers — a marker alone on its line, title supplied further
|
|
261
279
|
* down (see {@link titleAfterMarkerLine}).
|
|
262
280
|
*
|
|
263
|
-
* Every whitespace match is horizontal-only (`[^\S\n]`,
|
|
264
|
-
*
|
|
281
|
+
* Every whitespace match is horizontal-only (`[^\S\n]`, or the class-set
|
|
282
|
+
* difference `[\s--\n]` nested in the all-caps arm's class): a heading is one
|
|
265
283
|
* line, and a `\s` that matches `\n` fuses a run of consecutive all-caps lines
|
|
266
284
|
* into one composite match (#105). Horizontal whitespace is not the same as
|
|
267
285
|
* space-or-tab — filings put non-breaking spaces both between a marker and its
|
|
268
286
|
* title and inside an all-caps heading (`MICROSOFT<NBSP>CORPORATION`), so the
|
|
269
|
-
* all-caps arm has to admit them without admitting `\n`.
|
|
287
|
+
* all-caps arm has to admit them without admitting `\n`. That arm is one class
|
|
288
|
+
* (the `v` flag's set syntax) rather than an alternation of the class and
|
|
289
|
+
* `[^\S\n]`: the quantified alternation kept a backtrack frame per character
|
|
290
|
+
* and fell off a cliff past about 100,000 characters — a 160,000-character
|
|
291
|
+
* all-caps line ending in a lowercase letter took 25 ms against 0.4 ms at
|
|
292
|
+
* 80,000 — while a single class scans the line flat.
|
|
270
293
|
*
|
|
271
294
|
* The Item suffix spans the full 20-F range (`Item 16A`–`Item 16K`, not just
|
|
272
295
|
* `[a-c]`) and the marker's trailing period is optional — 20-F filers write
|
|
273
296
|
* the lettered markers unpunctuated.
|
|
274
297
|
*
|
|
298
|
+
* An indented line is no heading — admitting indentation fills the outline with
|
|
299
|
+
* centered cover legends and promotes indented TOC rows — with one exception: a
|
|
300
|
+
* same-line Item line, on either Item arm, may open on one NBSP (#176). An
|
|
301
|
+
* ` ` ahead of a styled marker leaves the extracted line starting on
|
|
302
|
+
* U+00A0 (Horizon Space's 8-K: "<NBSP>Item 1.01. Entry into a Material
|
|
303
|
+
* Definitive Agreement."), and the line anchor kept that Item out of the
|
|
304
|
+
* outline. A lookahead admits the NBSP only before an Item marker, and
|
|
305
|
+
* {@link itemEntryOffset} records the entry at the marker and drops a line that
|
|
306
|
+
* is a bare marker once its page cell is stripped. Part lines get no such
|
|
307
|
+
* allowance: Microsoft's 2004 10-K stamps an NBSP-led "Part I" on every page,
|
|
308
|
+
* and counting those would turn its body Parts into running-header furniture.
|
|
309
|
+
*
|
|
275
310
|
* Both all-caps alternatives already absorb trailing whitespace, so that arm
|
|
276
311
|
* ends at `$` with no `[^\S\n]*` of its own: the two quantifiers competing for one
|
|
277
312
|
* whitespace run backtracked quadratically on a long run before a lowercase
|
|
278
|
-
* letter — 3.5 s for an 80,000-character line.
|
|
313
|
+
* letter — 3.5 s for an 80,000-character line. The same-line Item title runs to
|
|
314
|
+
* the end of the line for the same reason; its bound is checked afterwards.
|
|
315
|
+
*
|
|
316
|
+
* Capture groups: the all-caps scan's 2 is the all-caps line (not an `ITEM N`
|
|
317
|
+
* line). The Item/Part scan's 2 is the Item code, 3 the Item title, and 4 the
|
|
318
|
+
* Part title.
|
|
279
319
|
*
|
|
280
320
|
* Heuristics — best-effort, not guaranteed to match all or only headings.
|
|
281
321
|
*/
|
|
282
|
-
const ALL_CAPS_HEADING_RE = /^(ITEM[^\S\n]+\d+[A-Z]?\b[^\n]
|
|
283
|
-
const ITEM_PART_HEADING_RE =
|
|
284
|
-
|
|
322
|
+
const ALL_CAPS_HEADING_RE = /^(?:\xA0(?=ITEM[^\S\n]+\d+[A-Z]?\b))?(ITEM[^\S\n]+\d+[A-Z]?\b[^\n]*|([A-Z][[A-Z0-9,\(\)\&.\/'’–—\-][\s--\n]]{8,}))$/gmv;
|
|
323
|
+
const ITEM_PART_HEADING_RE = new RegExp(String.raw `^(?:\xA0(?=item[^\S\n]))?(item[^\S\n]+(${ITEM_CODE})(?:\.?[^\S\n]+|(?<=\.\d{2})(?:\([a-z]\))?\.?[^\S\n]*(?=[a-z]))(\S[^\n]*)` +
|
|
324
|
+
String.raw `|part[^\S\n]+[ivx]{1,4}\b\.?(?:[^\S\n]*(?:[-–—:.][^\S\n]*)?([a-z][^\n]{0,140}))?)[^\S\n]*$`, 'gim');
|
|
325
|
+
/** The one leading character a same-line Item heading may open on (#176). */
|
|
326
|
+
const NBSP = '\xA0';
|
|
327
|
+
const BARE_ITEM_MARKER_RE = new RegExp(String.raw `^(item[^\S\n]+(${ITEM_CODE})\.?)[^\S\n]*$`, 'gim');
|
|
285
328
|
/**
|
|
286
329
|
* An Item marker with no period or colon after it, capturing the first
|
|
287
330
|
* character of what follows. Non-global: a predicate, not a scanner.
|
|
288
331
|
*/
|
|
289
|
-
const UNPUNCTUATED_ITEM_RE =
|
|
332
|
+
const UNPUNCTUATED_ITEM_RE = new RegExp(String.raw `^item[^\S\n]+${ITEM_CODE}(?![.:\w])[^\S\n]*(\S)`, 'i');
|
|
290
333
|
/**
|
|
291
334
|
* A line that ends mid-sentence — on a comma or a semicolon. A line ending on a
|
|
292
335
|
* lowercase word is no evidence: running page headers end that way ("Table of
|
|
293
336
|
* Contents" above "Item 5 — Other information" at the top of a page).
|
|
294
337
|
*/
|
|
295
338
|
const MID_SENTENCE_END_RE = /[,;]$/;
|
|
339
|
+
/** Whether the line before the one starting at `lineStart` ends mid-sentence. */
|
|
340
|
+
function followsMidSentence(text, lineStart) {
|
|
341
|
+
if (lineStart === 0)
|
|
342
|
+
return false;
|
|
343
|
+
const previousLine = text.slice(text.lastIndexOf('\n', lineStart - 2) + 1, lineStart - 1);
|
|
344
|
+
return MID_SENTENCE_END_RE.test(previousLine.trimEnd());
|
|
345
|
+
}
|
|
346
|
+
/** A whitespace-delimited token holding a digit must be a section number: `10`, `14A`, `2.`. */
|
|
347
|
+
const SECTION_NUMBER_TOKEN_RE = /^\d{1,2}[A-Z]?\.?$/;
|
|
348
|
+
/** An apostrophe inside a word ("MANAGEMENT'S") or after a word-final S ("STOCKHOLDERS’ EQUITY"). */
|
|
349
|
+
const HEADING_APOSTROPHE_RE = /(?<=[A-Z])['’](?=[A-Z])|(?<=S)['’](?=\s)/g;
|
|
350
|
+
/**
|
|
351
|
+
* Whether an all-caps line's length, digits, and apostrophes fit a heading
|
|
352
|
+
* (#170). The heading text runs at most {@link MAX_TITLE_CHARS} characters: past
|
|
353
|
+
* it the line is a paragraph set in capitals (a 308-character risk-factor coda,
|
|
354
|
+
* an 860-character tax legend), which the widened class otherwise admits. A
|
|
355
|
+
* digit may appear only in a whitespace-delimited section-number token ("NOTE
|
|
356
|
+
* 10", "SCHEDULE 14A"), so table rows ("EMEA 2,222 2,365"), footnoted names
|
|
357
|
+
* ("FMR LLC(4)"), and cover-page lines ("WASHINGTON, D.C. 20549", "…ACT OF
|
|
358
|
+
* 1934") stay out: digits admitted freely added 61 lines across 26 sampled
|
|
359
|
+
* documents, 12 of them numeric table rows, against 20 and none under the token
|
|
360
|
+
* rule. The digits are judged on the whole line, a TOC row's page cell
|
|
361
|
+
* included. An apostrophe must sit inside a word or after a word-final S,
|
|
362
|
+
* judged on the heading text the line yields, so a heading never opens or
|
|
363
|
+
* closes on one.
|
|
364
|
+
*/
|
|
365
|
+
function isAllCapsHeadingText(line) {
|
|
366
|
+
const heading = stripPageNumberTail(line).trim();
|
|
367
|
+
if (heading.length > MAX_TITLE_CHARS)
|
|
368
|
+
return false;
|
|
369
|
+
const tokens = line.trim().split(/\s+/);
|
|
370
|
+
if (!tokens.every((token) => !/\d/.test(token) || SECTION_NUMBER_TOKEN_RE.test(token))) {
|
|
371
|
+
return false;
|
|
372
|
+
}
|
|
373
|
+
return !/['’]/.test(heading.replace(HEADING_APOSTROPHE_RE, ''));
|
|
374
|
+
}
|
|
375
|
+
/** A second Item marker after the one an Item line opens on. Non-global: a predicate. */
|
|
376
|
+
const LATER_ITEM_MARKER_RE = /[^\S\n]item[^\S\n]+\d/i;
|
|
377
|
+
/**
|
|
378
|
+
* A TOC page cell inside a line rather than at its end: a page number, a
|
|
379
|
+
* lettered page, or either after a `Page` word, set off by whitespace before it
|
|
380
|
+
* and two or more whitespace characters before the next cell. Non-global: a
|
|
381
|
+
* predicate.
|
|
382
|
+
*/
|
|
383
|
+
const MID_LINE_PAGE_CELL_RE = /[^\S\n](?:(?:Page|PAGE)[^\S\n]+)?(?:\d{1,4}|[A-Z]-\d{1,3})[^\S\n]{2,}\S/;
|
|
384
|
+
/**
|
|
385
|
+
* Whether an Item line is a table of contents run together on one line: it
|
|
386
|
+
* carries a second Item marker or a page cell mid-line. Amazon's 8-K
|
|
387
|
+
* 0001104659-26-107526 prints its whole TOC as one 238-character line ("ITEM
|
|
388
|
+
* 8.01. OTHER EVENTS. 3 ITEM 9.01. FINANCIAL STATEMENTS AND EXHIBITS. 4
|
|
389
|
+
* SIGNATURES 5 …"), which as one heading took `item 8`, `item 9`, and
|
|
390
|
+
* `exhibits` from the body Items. The test is on shape, not length: a proxy's
|
|
391
|
+
* proposal heading runs as long ("ITEM 1 – TO APPROVE THE PROPOSAL TO AMEND
|
|
392
|
+
* THE Amended and Restated Articles…", 185 characters) and is the meeting's one
|
|
393
|
+
* heading. Across 104 cached filings, 244 all-caps and 568 mixed-case Item lines,
|
|
394
|
+
* it flags only that TOC line.
|
|
395
|
+
*/
|
|
396
|
+
function isRunTogetherToc(line) {
|
|
397
|
+
return LATER_ITEM_MARKER_RE.test(line) || MID_LINE_PAGE_CELL_RE.test(line);
|
|
398
|
+
}
|
|
399
|
+
/** A title that opens on another Item or Part marker. Non-global: a predicate. */
|
|
400
|
+
const MARKER_OPENER_RE = /^(?:item|part)\b/i;
|
|
401
|
+
/**
|
|
402
|
+
* Whether a titled Part line is a cross-reference rather than a heading: its
|
|
403
|
+
* title opens on another marker — a cross-reference index row ("Part II — Item 5.
|
|
404
|
+
* Market for Registrant’s Common Equity") — or it sits under a line ending
|
|
405
|
+
* mid-sentence, a wrapped reference ("…described in our Form 10-K,\nPART II —
|
|
406
|
+
* OTHER INFORMATION"). Either would take `section: "part ii"`, and the first
|
|
407
|
+
* would also displace the bare body Part it names (`dropBarePartsTitledLater`).
|
|
408
|
+
*/
|
|
409
|
+
function isPartCrossReference(text, lineStart, title) {
|
|
410
|
+
return MARKER_OPENER_RE.test(title) || followsMidSentence(text, lineStart);
|
|
411
|
+
}
|
|
412
|
+
/**
|
|
413
|
+
* Whether a same-line Item title fits the title bound. That arm has always
|
|
414
|
+
* counted the bound after the title's first character (`\S[^\n]{0,140}`), one
|
|
415
|
+
* more than a title on a line of its own.
|
|
416
|
+
*/
|
|
417
|
+
function sameLineTitleFits(code, title) {
|
|
418
|
+
return title.trimEnd().length <= maxTitleChars(code) + 1;
|
|
419
|
+
}
|
|
420
|
+
/**
|
|
421
|
+
* The title bound for a current Form 8-K item. Item 5.02's title runs 146
|
|
422
|
+
* characters, past {@link MAX_TITLE_CHARS} (#165), and is the longest any of 53
|
|
423
|
+
* sampled 8-Ks prints; the 14-character margin admits a short trailing note
|
|
424
|
+
* ("(continued)", a paragraph list) while a paragraph opening on the marker
|
|
425
|
+
* line ("Item 5.02(b) On June 2, 2026, the Board…") runs to hundreds of
|
|
426
|
+
* characters and stays out.
|
|
427
|
+
*/
|
|
428
|
+
const EIGHT_K_MAX_TITLE_CHARS = 160;
|
|
429
|
+
/**
|
|
430
|
+
* The title bound for an Item code: {@link EIGHT_K_MAX_TITLE_CHARS} for a code
|
|
431
|
+
* in the current Form 8-K item table, {@link MAX_TITLE_CHARS} for every other
|
|
432
|
+
* marker. Keyed on the table rather than the dotted shape, so a dotted code
|
|
433
|
+
* outside it ("Item 7.09") keeps the tighter bound.
|
|
434
|
+
*/
|
|
435
|
+
function maxTitleChars(code) {
|
|
436
|
+
return Object.hasOwn(CURRENT_EIGHT_K_ITEMS, code) ? EIGHT_K_MAX_TITLE_CHARS : MAX_TITLE_CHARS;
|
|
437
|
+
}
|
|
296
438
|
/**
|
|
297
439
|
* Whether an Item line that starts at `lineStart` is a wrapped sentence, not a
|
|
298
440
|
* heading. Plain-text filings wrap prose at a fixed column, so a cross-reference
|
|
@@ -310,10 +452,7 @@ function isWrappedProse(text, lineStart, line) {
|
|
|
310
452
|
return false;
|
|
311
453
|
if (/[a-z]/.test(marker[1] ?? ''))
|
|
312
454
|
return true;
|
|
313
|
-
|
|
314
|
-
return false;
|
|
315
|
-
const previousLine = text.slice(text.lastIndexOf('\n', lineStart - 2) + 1, lineStart - 1);
|
|
316
|
-
return MID_SENTENCE_END_RE.test(previousLine.trimEnd());
|
|
455
|
+
return followsMidSentence(text, lineStart);
|
|
317
456
|
}
|
|
318
457
|
/**
|
|
319
458
|
* Sticky: the marker line's own newline, at least one blank line, then the first
|
|
@@ -331,12 +470,25 @@ const TIGHT_TITLE_RE = /\n[^\S\n]*(\S[^\n]*)\n?([^\n]*)/y;
|
|
|
331
470
|
/** A whole line holding nothing but a page number — the TOC-cell tell. */
|
|
332
471
|
const PAGE_NUMBER_LINE_RE = /^[^\S\n]*\d{1,4}[^\S\n]*$/;
|
|
333
472
|
/**
|
|
334
|
-
* A
|
|
335
|
-
*
|
|
336
|
-
*
|
|
473
|
+
* A TOC row's page cell at the end of its line, set off by a tab or two or more
|
|
474
|
+
* spaces ("Key Information 3", "Consolidated Balance Sheets F-3"), its
|
|
475
|
+
* number optionally after a `Page` word ("Item 8.01 Other Events Page 1",
|
|
476
|
+
* the cell First Bancorp's 8-K TOC prints, which otherwise kept the row from
|
|
477
|
+
* deduping against its body Item and took `item 8`). One space is not enough: a
|
|
478
|
+
* heading can end in its own number ("PROPOSAL NO. 1", "ARTICLE 5", "…Exchange
|
|
479
|
+
* Act of 1934", "…as of June 30, 2026"), which a single-space rule stripped or
|
|
480
|
+
* read as a TOC row (#172, #166). The lookbehind starts a try only where a
|
|
481
|
+
* whitespace run begins; a try from inside the run rescanned the rest of it,
|
|
482
|
+
* quadratic on a long run.
|
|
483
|
+
*/
|
|
484
|
+
const TOC_PAGE_TAIL_RE = /(?<![^\S\n])(?:[^\S\n]{2,}|\t)(?:(?:Page|PAGE)[^\S\n]+)?(?:\d{1,4}|[A-Z]-\d{1,3})[^\S\n]*$/;
|
|
485
|
+
/**
|
|
486
|
+
* The title bound for an Item marker's title, a titled Part's title, and an
|
|
487
|
+
* all-caps heading line — past it the line reads as body prose. The same-line
|
|
488
|
+
* Item arm counts it after the title's first character, one more
|
|
489
|
+
* ({@link sameLineTitleFits}), and a current 8-K item takes
|
|
490
|
+
* {@link EIGHT_K_MAX_TITLE_CHARS}.
|
|
337
491
|
*/
|
|
338
|
-
const PAGE_NUMBER_TAIL_RE = /(?<![^\S\n])[^\S\n]+\d{1,4}[^\S\n]*$/;
|
|
339
|
-
/** Same bound the same-line Item arm uses — past it the line reads as body prose. */
|
|
340
492
|
const MAX_TITLE_CHARS = 140;
|
|
341
493
|
/**
|
|
342
494
|
* A line repeated this many times is page furniture (a running header), not a
|
|
@@ -359,25 +511,109 @@ const MAX_TITLE_CHARS = 140;
|
|
|
359
511
|
*/
|
|
360
512
|
const RUNNING_HEADER_MIN_OCCURRENCES = 5;
|
|
361
513
|
/**
|
|
362
|
-
* A heading that is nothing but a structural marker, with no title of
|
|
363
|
-
*
|
|
364
|
-
*
|
|
365
|
-
*
|
|
514
|
+
* A line or heading that is nothing but a structural marker, with no title of
|
|
515
|
+
* its own: a bare Part line, or an Item marker alone on its line or reached the
|
|
516
|
+
* outline unjoined. Non-global on purpose — a predicate, not a scanner, so it
|
|
517
|
+
* carries no `lastIndex`.
|
|
518
|
+
*/
|
|
519
|
+
const BARE_MARKER_HEADING_RE = new RegExp(String.raw `^(?:item[^\S\n]+${ITEM_CODE}|part[^\S\n]+[ivx]{1,4})\.?$`, 'i');
|
|
520
|
+
/** A bare Part heading, capturing its numeral. */
|
|
521
|
+
const BARE_PART_HEADING_RE = /^part[^\S\n]+([ivx]{1,4})\.?$/i;
|
|
522
|
+
/**
|
|
523
|
+
* A Part heading with a title after its numeral ("PART II — OTHER INFORMATION",
|
|
524
|
+
* "Part I. Financial Information"), capturing the numeral and the title, which
|
|
525
|
+
* opens on a letter.
|
|
366
526
|
*/
|
|
367
|
-
const
|
|
368
|
-
/**
|
|
527
|
+
const TITLED_PART_HEADING_RE = /^part[^\S\n]+([ivx]{1,4})\b\.?[^\S\n]*(?:[-–—:.][^\S\n]*)?([a-z][^\n]*)/i;
|
|
528
|
+
/**
|
|
529
|
+
* Drop each bare Part entry that a later titled heading for the same numeral
|
|
530
|
+
* follows (#170). A 10-Q's TOC prints "Part II" as a bare row and its body prints
|
|
531
|
+
* "PART II — OTHER INFORMATION"; the two keys differ, so dedup keeps both and
|
|
532
|
+
* `section: "part ii"` lands on the TOC row, 50,000 characters early (Apple's
|
|
533
|
+
* Q3 FY2026 10-Q). A bare Part with no later titled heading stays — it is the
|
|
534
|
+
* only entry for that Part. A title must open uppercase and on neither marker,
|
|
535
|
+
* the same test the detection arms apply ({@link isPartCrossReference}).
|
|
536
|
+
*/
|
|
537
|
+
function dropBarePartsTitledLater(headings) {
|
|
538
|
+
const titledLater = new Set();
|
|
539
|
+
const kept = [];
|
|
540
|
+
for (const entry of [...headings].reverse()) {
|
|
541
|
+
const [, numeral, title = ''] = TITLED_PART_HEADING_RE.exec(entry.heading) ?? [];
|
|
542
|
+
if (numeral !== undefined && /^[A-Z]/.test(title) && !MARKER_OPENER_RE.test(title)) {
|
|
543
|
+
titledLater.add(numeral.toLowerCase());
|
|
544
|
+
}
|
|
545
|
+
else {
|
|
546
|
+
const bare = BARE_PART_HEADING_RE.exec(entry.heading)?.[1]?.toLowerCase();
|
|
547
|
+
if (bare !== undefined && titledLater.has(bare))
|
|
548
|
+
continue;
|
|
549
|
+
}
|
|
550
|
+
kept.push(entry);
|
|
551
|
+
}
|
|
552
|
+
return kept.reverse();
|
|
553
|
+
}
|
|
554
|
+
/** Drop a TOC line's trailing page cell so it dedups against the body heading. */
|
|
369
555
|
function stripPageNumberTail(heading) {
|
|
370
|
-
return heading.replace(
|
|
556
|
+
return heading.replace(TOC_PAGE_TAIL_RE, '');
|
|
371
557
|
}
|
|
372
558
|
/**
|
|
373
|
-
*
|
|
374
|
-
*
|
|
375
|
-
*
|
|
376
|
-
*
|
|
559
|
+
* A folded key's Item or Part marker — an Item code with an optional 8-K
|
|
560
|
+
* paragraph suffix, or a Part numeral — capturing the marker and the title
|
|
561
|
+
* after it, past an optional period and one dash, colon, or period separator. A
|
|
562
|
+
* bare marker captures an empty title.
|
|
563
|
+
*/
|
|
564
|
+
export const MARKER_TITLE_RE = /^(item \d{1,2}(?:\.\d{2}|[a-z])?(?:\([a-z]\))?|part [ivx]{1,4}\b)\.? ?(?:[-–—:.] ?)?(.*)$/;
|
|
565
|
+
/** Terminal sentence punctuation on a folded key. */
|
|
566
|
+
const TERMINAL_PUNCTUATION_RE = /(?<![.,;:])[.,;:]+$/;
|
|
567
|
+
/**
|
|
568
|
+
* Comparison key for a heading, a line, or a `section` needle — the dedup key
|
|
569
|
+
* for detected headings and the operand of `section`'s exact tiers.
|
|
570
|
+
* {@link foldForHeadingMatch} covers the whitespace and quote differences a
|
|
571
|
+
* filing prints between its TOC and its body. Two remain:
|
|
572
|
+
* - Terminal sentence punctuation: C3is' 20-F writes "Item 16J Insider Trading
|
|
573
|
+
* Policies" in the TOC and "…Policies." in the body.
|
|
574
|
+
* - The separator between an Item or Part marker and its title: Worthington
|
|
575
|
+
* Steel's 10-Q prints "Item 1A. Risk Factors" in its TOC and "Item 1A. – Risk
|
|
576
|
+
* Factors" in its body, so both kept an entry and `section` landed on the TOC
|
|
577
|
+
* row (#174). A marker key is rebuilt as the marker, one space, and the title
|
|
578
|
+
* ("item 1a risk factors"), whatever period, dash, or colon stood between them.
|
|
579
|
+
*
|
|
377
580
|
* Key-only — the entry keeps the document's own heading text.
|
|
378
581
|
*/
|
|
379
|
-
function headingKey(heading) {
|
|
380
|
-
|
|
582
|
+
export function headingKey(heading) {
|
|
583
|
+
const key = printedKey(heading);
|
|
584
|
+
const [, marker, title] = MARKER_TITLE_RE.exec(key) ?? [];
|
|
585
|
+
if (marker === undefined)
|
|
586
|
+
return key;
|
|
587
|
+
return title ? `${marker} ${title}` : marker;
|
|
588
|
+
}
|
|
589
|
+
/**
|
|
590
|
+
* A heading's printed form for the running-header count: {@link headingKey}
|
|
591
|
+
* short of the marker-separator fold. Page furniture repeats one printed form;
|
|
592
|
+
* counted on the dedup key, a 10-Q's per-page "Part I. Financial Information"
|
|
593
|
+
* pooled with its TOC row and body heading "PART I – FINANCIAL INFORMATION",
|
|
594
|
+
* and the TOC row, first of the pool, took the body heading's place.
|
|
595
|
+
*/
|
|
596
|
+
function printedKey(heading) {
|
|
597
|
+
return foldForHeadingMatch(heading).replace(TERMINAL_PUNCTUATION_RE, '');
|
|
598
|
+
}
|
|
599
|
+
/**
|
|
600
|
+
* A 20-F sub-item label opening a folded key: a digit-led one ("3d. ") or a
|
|
601
|
+
* single letter ("b. "). Anchored, so it costs the same on any key.
|
|
602
|
+
*/
|
|
603
|
+
const SUB_ITEM_LABEL_RE = /^(?:\d{1,2}[a-z]|[a-z])\. /;
|
|
604
|
+
/**
|
|
605
|
+
* A folded key with its leading 20-F sub-item label removed ("3d. risk factors"
|
|
606
|
+
* → "risk factors"), or undefined when it opens on none (#175). 20-F filers label
|
|
607
|
+
* sub-items ("3D. RISK FACTORS", "B. Business Overview"), so a plain-phrase
|
|
608
|
+
* `section` never equalled the line. Comparison-time only: `section`'s exact and
|
|
609
|
+
* line tiers compare a heading or line key both with and without its label,
|
|
610
|
+
* while the caller's value is compared as given, and no labelled line becomes a
|
|
611
|
+
* detected heading. A line opening on an initial ("L. John Doerr") strips the
|
|
612
|
+
* same way, so it matches a value equal to the rest of the line.
|
|
613
|
+
*/
|
|
614
|
+
export function withoutSubItemLabel(key) {
|
|
615
|
+
const label = SUB_ITEM_LABEL_RE.exec(key);
|
|
616
|
+
return label === null ? undefined : key.slice(label[0].length);
|
|
381
617
|
}
|
|
382
618
|
/**
|
|
383
619
|
* The title for a bare Item marker whose line ends at `markerLineEnd`, or
|
|
@@ -391,12 +627,13 @@ function headingKey(heading) {
|
|
|
391
627
|
* that puts a marker on its own line — a page footer, an unwrapped body
|
|
392
628
|
* paragraph — is admitted too, taking the next line as its title; the entry's
|
|
393
629
|
* offset is the marker's, so `section` still lands in the right place.
|
|
630
|
+
* `maxChars` is the marker's title bound ({@link maxTitleChars}).
|
|
394
631
|
*/
|
|
395
|
-
function titleAfterMarkerLine(text, markerLineEnd) {
|
|
632
|
+
function titleAfterMarkerLine(text, markerLineEnd, maxChars) {
|
|
396
633
|
SEPARATED_TITLE_RE.lastIndex = markerLineEnd;
|
|
397
634
|
const separated = SEPARATED_TITLE_RE.exec(text)?.[1];
|
|
398
635
|
if (separated !== undefined)
|
|
399
|
-
return usableTitle(separated);
|
|
636
|
+
return usableTitle(separated, maxChars);
|
|
400
637
|
TIGHT_TITLE_RE.lastIndex = markerLineEnd;
|
|
401
638
|
const tight = TIGHT_TITLE_RE.exec(text);
|
|
402
639
|
if (tight === null)
|
|
@@ -404,56 +641,101 @@ function titleAfterMarkerLine(text, markerLineEnd) {
|
|
|
404
641
|
const [, title = '', lineBelow = ''] = tight;
|
|
405
642
|
if (PAGE_NUMBER_LINE_RE.test(lineBelow))
|
|
406
643
|
return;
|
|
407
|
-
return usableTitle(title);
|
|
644
|
+
return usableTitle(title, maxChars);
|
|
408
645
|
}
|
|
409
646
|
/**
|
|
410
647
|
* A candidate title line reduced to heading text, or undefined when it is not a
|
|
411
|
-
* title at all — empty, a bare page number, the next marker, or
|
|
412
|
-
*
|
|
648
|
+
* title at all — empty, a bare page number, the next marker, or longer than
|
|
649
|
+
* `maxChars`, where it reads as body prose.
|
|
413
650
|
*/
|
|
414
|
-
function usableTitle(raw) {
|
|
651
|
+
function usableTitle(raw, maxChars) {
|
|
415
652
|
const title = stripPageNumberTail(raw).trim();
|
|
416
|
-
if (title.length === 0 || title.length >
|
|
653
|
+
if (title.length === 0 || title.length > maxChars)
|
|
417
654
|
return;
|
|
418
655
|
if (/^\d+$/.test(title))
|
|
419
656
|
return;
|
|
420
|
-
if (
|
|
657
|
+
if (MARKER_OPENER_RE.test(title))
|
|
421
658
|
return;
|
|
422
659
|
return title;
|
|
423
660
|
}
|
|
424
661
|
/**
|
|
425
|
-
*
|
|
426
|
-
*
|
|
427
|
-
*
|
|
428
|
-
*
|
|
429
|
-
|
|
662
|
+
* The offset of the entry a heading match at `lineStart` records: the line
|
|
663
|
+
* start, or past the one NBSP a same-line Item line may open on (#176). An
|
|
664
|
+
* NBSP-led line yields no entry when its page cell stripped leaves a bare marker
|
|
665
|
+
* ("ITEM 2.02", "Item 1. 3"), since the exception admits titled headings only.
|
|
666
|
+
*/
|
|
667
|
+
function itemEntryOffset(text, lineStart, heading) {
|
|
668
|
+
if (!text.startsWith(NBSP, lineStart))
|
|
669
|
+
return lineStart;
|
|
670
|
+
if (BARE_MARKER_HEADING_RE.test(stripPageNumberTail(heading).trim()))
|
|
671
|
+
return;
|
|
672
|
+
return lineStart + 1;
|
|
673
|
+
}
|
|
674
|
+
/**
|
|
675
|
+
* Detect every heading in extracted text and return them with their offsets,
|
|
676
|
+
* sorted by offset. Deduplicates by {@link headingKey}, keeping the later
|
|
677
|
+
* occurrence (handles TOC vs body duplicate headings — the body occurrence is
|
|
678
|
+
* the more useful landing spot), drops running page headers, and drops a bare
|
|
679
|
+
* Part entry that a later titled heading for the same Part follows. Uncapped:
|
|
680
|
+
* `section` matches every heading, and the rendered outline makes its own cut.
|
|
430
681
|
*/
|
|
431
|
-
export function detectHeadings(text
|
|
682
|
+
export function detectHeadings(text) {
|
|
432
683
|
const matches = [];
|
|
433
684
|
const add = (heading, index) => {
|
|
434
685
|
const trimmed = stripPageNumberTail(heading).trim();
|
|
435
686
|
if (trimmed.length > 0)
|
|
436
687
|
matches.push({ heading: trimmed, index });
|
|
437
688
|
};
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
689
|
+
// The guards read a line from its real start (`match.index`), the one NBSP an
|
|
690
|
+
// Item line may open on included (#176); `itemEntryOffset` places the entry.
|
|
691
|
+
ALL_CAPS_HEADING_RE.lastIndex = 0;
|
|
692
|
+
for (let match = ALL_CAPS_HEADING_RE.exec(text); match !== null; match = ALL_CAPS_HEADING_RE.exec(text)) {
|
|
693
|
+
const [, heading, capsLine] = match;
|
|
694
|
+
if (heading === undefined || isWrappedProse(text, match.index, heading))
|
|
695
|
+
continue;
|
|
696
|
+
if (capsLine === undefined) {
|
|
697
|
+
if (isRunTogetherToc(heading))
|
|
698
|
+
continue;
|
|
699
|
+
}
|
|
700
|
+
else {
|
|
701
|
+
if (!isAllCapsHeadingText(capsLine))
|
|
702
|
+
continue;
|
|
703
|
+
const partTitle = TITLED_PART_HEADING_RE.exec(capsLine)?.[2];
|
|
704
|
+
if (partTitle !== undefined && isPartCrossReference(text, match.index, partTitle))
|
|
705
|
+
continue;
|
|
706
|
+
}
|
|
707
|
+
const offset = itemEntryOffset(text, match.index, heading);
|
|
708
|
+
if (offset !== undefined)
|
|
709
|
+
add(heading, offset);
|
|
710
|
+
}
|
|
711
|
+
ITEM_PART_HEADING_RE.lastIndex = 0;
|
|
712
|
+
for (let match = ITEM_PART_HEADING_RE.exec(text); match !== null; match = ITEM_PART_HEADING_RE.exec(text)) {
|
|
713
|
+
const [, heading, itemCode, itemTitle = '', partTitle] = match;
|
|
714
|
+
if (heading === undefined || isWrappedProse(text, match.index, heading))
|
|
715
|
+
continue;
|
|
716
|
+
if (itemCode !== undefined &&
|
|
717
|
+
(!sameLineTitleFits(itemCode, itemTitle) || isRunTogetherToc(heading))) {
|
|
718
|
+
continue;
|
|
719
|
+
}
|
|
720
|
+
if (partTitle !== undefined &&
|
|
721
|
+
(!/^[A-Z]/.test(partTitle) || isPartCrossReference(text, match.index, partTitle))) {
|
|
722
|
+
continue;
|
|
445
723
|
}
|
|
724
|
+
const offset = itemEntryOffset(text, match.index, heading);
|
|
725
|
+
if (offset !== undefined)
|
|
726
|
+
add(heading, offset);
|
|
446
727
|
}
|
|
447
728
|
// Bare marker, title on a later line — the shape every Item takes in a 20-F
|
|
448
729
|
// rendered one cell per line, in both the TOC and the body (#105).
|
|
449
730
|
const bareMarkers = [];
|
|
450
731
|
BARE_ITEM_MARKER_RE.lastIndex = 0;
|
|
451
732
|
for (let match = BARE_ITEM_MARKER_RE.exec(text); match !== null; match = BARE_ITEM_MARKER_RE.exec(text)) {
|
|
452
|
-
const marker = match
|
|
453
|
-
if (marker === undefined)
|
|
733
|
+
const [, marker, code] = match;
|
|
734
|
+
if (marker === undefined || code === undefined)
|
|
454
735
|
continue;
|
|
455
736
|
bareMarkers.push({
|
|
456
737
|
marker: marker.trim(),
|
|
738
|
+
code,
|
|
457
739
|
index: match.index,
|
|
458
740
|
lineEnd: match.index + match[0].length,
|
|
459
741
|
});
|
|
@@ -470,11 +752,11 @@ export function detectHeadings(text, maxEntries = 50) {
|
|
|
470
752
|
const key = foldForHeadingMatch(marker);
|
|
471
753
|
markerCounts.set(key, (markerCounts.get(key) ?? 0) + 1);
|
|
472
754
|
}
|
|
473
|
-
for (const { marker, index, lineEnd } of bareMarkers) {
|
|
755
|
+
for (const { marker, code, index, lineEnd } of bareMarkers) {
|
|
474
756
|
if ((markerCounts.get(foldForHeadingMatch(marker)) ?? 0) >= RUNNING_HEADER_MIN_OCCURRENCES) {
|
|
475
757
|
continue;
|
|
476
758
|
}
|
|
477
|
-
const title = titleAfterMarkerLine(text, lineEnd);
|
|
759
|
+
const title = titleAfterMarkerLine(text, lineEnd, maxTitleChars(code));
|
|
478
760
|
if (title !== undefined)
|
|
479
761
|
add(`${marker} ${title}`, index);
|
|
480
762
|
}
|
|
@@ -484,7 +766,7 @@ export function detectHeadings(text, maxEntries = 50) {
|
|
|
484
766
|
// would read three occurrences of one heading as page furniture.
|
|
485
767
|
const positions = new Map();
|
|
486
768
|
for (const { heading, index } of matches) {
|
|
487
|
-
const key =
|
|
769
|
+
const key = printedKey(heading);
|
|
488
770
|
const seen = positions.get(key) ?? new Set();
|
|
489
771
|
seen.add(index);
|
|
490
772
|
positions.set(key, seen);
|
|
@@ -493,20 +775,31 @@ export function detectHeadings(text, maxEntries = 50) {
|
|
|
493
775
|
// collapses "Part I" TOC vs "PART I" body). Page furniture inverts that: it
|
|
494
776
|
// keeps the first occurrence, or none at all when it is a bare marker.
|
|
495
777
|
// `matches` is offset-sorted, so the first write wins for furniture and the
|
|
496
|
-
// last for everything else.
|
|
778
|
+
// last for everything else. Furniture keeps its first occurrence in a map of
|
|
779
|
+
// its own, so a TOC row or body heading sharing its dedup key in another
|
|
780
|
+
// separator form neither joins the count nor gives way to it.
|
|
497
781
|
const byHeading = new Map();
|
|
782
|
+
const furniture = new Map();
|
|
498
783
|
for (const { heading, index } of matches) {
|
|
499
|
-
const
|
|
500
|
-
if ((positions.get(
|
|
784
|
+
const printed = printedKey(heading);
|
|
785
|
+
if ((positions.get(printed)?.size ?? 0) >= RUNNING_HEADER_MIN_OCCURRENCES) {
|
|
501
786
|
if (BARE_MARKER_HEADING_RE.test(heading))
|
|
502
787
|
continue;
|
|
503
|
-
if (!
|
|
504
|
-
|
|
788
|
+
if (!furniture.has(printed))
|
|
789
|
+
furniture.set(printed, { heading, offset: index });
|
|
505
790
|
continue;
|
|
506
791
|
}
|
|
507
|
-
byHeading.set(
|
|
792
|
+
byHeading.set(headingKey(heading), { heading, offset: index });
|
|
508
793
|
}
|
|
509
|
-
|
|
794
|
+
// An all-caps marker line ("ITEM 2.02") matches the all-caps arm on its own
|
|
795
|
+
// and also joins the title below it; when both survive dedup at one offset,
|
|
796
|
+
// the bare entry is the same heading listed twice.
|
|
797
|
+
const entries = [...byHeading.values(), ...furniture.values()];
|
|
798
|
+
const titledOffsets = new Set(entries.filter((h) => !BARE_MARKER_HEADING_RE.test(h.heading)).map((h) => h.offset));
|
|
799
|
+
const sorted = entries
|
|
800
|
+
.filter((h) => !(BARE_MARKER_HEADING_RE.test(h.heading) && titledOffsets.has(h.offset)))
|
|
801
|
+
.sort((a, b) => a.offset - b.offset);
|
|
802
|
+
return dropBarePartsTitledLater(sorted);
|
|
510
803
|
}
|
|
511
804
|
/**
|
|
512
805
|
* Fold a heading or a `section` needle into a comparison form: typographic
|
|
@@ -529,6 +822,217 @@ export function foldForHeadingMatch(value) {
|
|
|
529
822
|
.trim()
|
|
530
823
|
.toLowerCase();
|
|
531
824
|
}
|
|
825
|
+
/**
|
|
826
|
+
* A TOC page cell on a line of its own: a page number, a lettered
|
|
827
|
+
* financial-statement page ("F-3"), or a lowercase roman front-matter page
|
|
828
|
+
* ("ii"), any of them optionally after a `Page` word ("Page 3"). Roman cells
|
|
829
|
+
* count only on their own line — at the end of a line set off by spaces,
|
|
830
|
+
* "Part iv" would read as a page cell.
|
|
831
|
+
*/
|
|
832
|
+
const PAGE_CELL_LINE_RE = /^[^\S\n]*(?:(?:Page|PAGE)[^\S\n]+)?(?:\d{1,4}|[A-Z]-\d{1,3}|[ivxlc]{1,7})[^\S\n]*$/;
|
|
833
|
+
/** A line opening on an Item or Part marker, bare or titled. Non-global: a predicate. */
|
|
834
|
+
const MARKER_LINE_RE = new RegExp(String.raw `^(?:item[^\S\n]+${ITEM_CODE}|part[^\S\n]+[ivx]{1,4})\b`, 'i');
|
|
835
|
+
/**
|
|
836
|
+
* The first line at or after `from` holding anything but whitespace (NBSP
|
|
837
|
+
* spacer rows count as whitespace), or undefined when none does. An HTML TOC
|
|
838
|
+
* table renders each cell on its own line with spacer lines between ("Legal
|
|
839
|
+
* Proceedings\n\n \n\n31"). A loop rather than a sticky `(?:[^\S\n]*\n)*`
|
|
840
|
+
* regex, which kept a backtrack frame per spacer line and stepped up past about
|
|
841
|
+
* 100,000 characters of them.
|
|
842
|
+
*/
|
|
843
|
+
function nextContentLine(text, from) {
|
|
844
|
+
for (let at = from; at < text.length;) {
|
|
845
|
+
const newline = text.indexOf('\n', at);
|
|
846
|
+
const end = newline === -1 ? text.length : newline;
|
|
847
|
+
const line = text.slice(at, end);
|
|
848
|
+
if (/\S/.test(line))
|
|
849
|
+
return { end: end + 1, line };
|
|
850
|
+
at = end + 1;
|
|
851
|
+
}
|
|
852
|
+
return;
|
|
853
|
+
}
|
|
854
|
+
/**
|
|
855
|
+
* Whether `line`, whose newline ends just before `after`, is a TOC row. Its page
|
|
856
|
+
* cell sits at its end ({@link TOC_PAGE_TAIL_RE}) or on the next line with
|
|
857
|
+
* content ({@link PAGE_CELL_LINE_RE}), spacer lines passed over. A line that is
|
|
858
|
+
* not itself a marker, followed directly by an Item/Part marker, is a title cell
|
|
859
|
+
* of a TOC with no page column ("Business Overview" above the next row's "Item
|
|
860
|
+
* 1A."). A marker line (a Part over its first Item) is a TOC row when that Item
|
|
861
|
+
* row is: the Item line ends in a page cell, or the line below it holds one or
|
|
862
|
+
* ends in one (a title cell, or a wrapped title: "Item 5. Market for…Issuer
|
|
863
|
+
* Purchases / of Equity Securities....... 12"), or — a bare marker — the line
|
|
864
|
+
* below its title holds one ("PART IV / Item 15. / Exhibit and Financial
|
|
865
|
+
* Statement Schedules / 93"). A body Part sits over its Item too, but prose, not
|
|
866
|
+
* a page cell, follows.
|
|
867
|
+
*/
|
|
868
|
+
function isTocRow(text, line, after) {
|
|
869
|
+
if (TOC_PAGE_TAIL_RE.test(line))
|
|
870
|
+
return true;
|
|
871
|
+
const item = nextContentLine(text, after);
|
|
872
|
+
if (item === undefined)
|
|
873
|
+
return false;
|
|
874
|
+
if (PAGE_CELL_LINE_RE.test(item.line))
|
|
875
|
+
return true;
|
|
876
|
+
if (!MARKER_LINE_RE.test(item.line.trim()))
|
|
877
|
+
return false;
|
|
878
|
+
if (!MARKER_LINE_RE.test(line.trim()))
|
|
879
|
+
return true;
|
|
880
|
+
if (TOC_PAGE_TAIL_RE.test(item.line))
|
|
881
|
+
return true;
|
|
882
|
+
const below = nextContentLine(text, item.end);
|
|
883
|
+
if (below === undefined)
|
|
884
|
+
return false;
|
|
885
|
+
if (PAGE_CELL_LINE_RE.test(below.line) || TOC_PAGE_TAIL_RE.test(below.line))
|
|
886
|
+
return true;
|
|
887
|
+
if (!BARE_MARKER_HEADING_RE.test(item.line.trim()))
|
|
888
|
+
return false;
|
|
889
|
+
const page = nextContentLine(text, below.end);
|
|
890
|
+
return page !== undefined && PAGE_CELL_LINE_RE.test(page.line);
|
|
891
|
+
}
|
|
892
|
+
/**
|
|
893
|
+
* Whether a detected heading sits on a table-of-contents row, so `section` ranks
|
|
894
|
+
* it after every body heading of its tier (#174). The test runs on the heading's
|
|
895
|
+
* title line: its own line, or for a bare marker or a marker joined to a later
|
|
896
|
+
* title, the next line with content. That line is a TOC row by {@link isTocRow}
|
|
897
|
+
* (the line tiers' own test), or when the next line with content ends in a page
|
|
898
|
+
* cell — a TOC title wrapped onto a second line (Aware's "Item 5. Market for…
|
|
899
|
+
* Issuer Purchases / of Equity Securities....... 12").
|
|
900
|
+
*
|
|
901
|
+
* Three TOC shapes survive dedup: a body heading worded apart from its TOC row
|
|
902
|
+
* (First Bancorp's 8-K: "Item 9.01 Financial Statements and Exhibits. Page 1"
|
|
903
|
+
* over the body's "…Pro Forma Financial Information and Exhibits."), a bare
|
|
904
|
+
* all-caps marker over a title cell and a page cell (DRDGOLD's 20-F: "ITEM 3." /
|
|
905
|
+
* "KEY INFORMATION" / "6"), and the title cell itself.
|
|
906
|
+
*
|
|
907
|
+
* The test reads a page number under a body heading the same way (CCC's 10-K sets
|
|
908
|
+
* "Item 6. Reserved." over its page number), which is why a TOC row is only
|
|
909
|
+
* ranked last in its tier rather than dropped: alone in the tier, it still lands.
|
|
910
|
+
* A heading's offset can sit one past its line start (an NBSP-led Item, #176), so
|
|
911
|
+
* the line is found from the newline before it.
|
|
912
|
+
*/
|
|
913
|
+
export function isTocHeading(text, heading) {
|
|
914
|
+
const lineStart = text.lastIndexOf('\n', heading.offset - 1) + 1;
|
|
915
|
+
const newline = text.indexOf('\n', heading.offset);
|
|
916
|
+
const lineEnd = newline === -1 ? text.length : newline;
|
|
917
|
+
const titledHere = !BARE_MARKER_HEADING_RE.test(heading.heading) &&
|
|
918
|
+
text.startsWith(heading.heading, heading.offset);
|
|
919
|
+
const title = titledHere
|
|
920
|
+
? { line: text.slice(lineStart, lineEnd), end: lineEnd + 1 }
|
|
921
|
+
: nextContentLine(text, lineEnd + 1);
|
|
922
|
+
if (title === undefined)
|
|
923
|
+
return false;
|
|
924
|
+
if (isTocRow(text, title.line, title.end))
|
|
925
|
+
return true;
|
|
926
|
+
const below = nextContentLine(text, title.end);
|
|
927
|
+
return below !== undefined && TOC_PAGE_TAIL_RE.test(below.line);
|
|
928
|
+
}
|
|
929
|
+
/**
|
|
930
|
+
* Whether a prefix line runs on past a one-word needle into a longer Title Case
|
|
931
|
+
* title — the next word opens on a capital in a line that is not all caps. One
|
|
932
|
+
* word heads many unrelated titles: `business` reached "Business Combinations.",
|
|
933
|
+
* "Business Seasonality and Product Introductions", and a cross-reference line
|
|
934
|
+
* "Business Overview—Governmental regulations…" across the sampled filings. A
|
|
935
|
+
* needle of two or more words is specific enough to keep its continuation
|
|
936
|
+
* ("Human Capital" → "Human Capital Management"), and a suffix opening
|
|
937
|
+
* lowercase ("as of June 30, 2026") or on punctuation ("(continued)",
|
|
938
|
+
* "—Parent Company Only") continues the needle's own heading. `key` is the
|
|
939
|
+
* line's {@link headingKey}, which starts with `needle` — for a labelled line,
|
|
940
|
+
* the key and the line both past the label (#175); folding maps every whitespace
|
|
941
|
+
* run to one space, so the needle is exactly the line's first word.
|
|
942
|
+
*/
|
|
943
|
+
function continuesIntoLongerTitle(line, key, needle) {
|
|
944
|
+
if (needle.includes(' ') || key.charAt(needle.length) !== ' ')
|
|
945
|
+
return false;
|
|
946
|
+
if (!/\p{Ll}/u.test(line))
|
|
947
|
+
return false;
|
|
948
|
+
return /^\p{Lu}/u.test(line.trim().split(/\s+/)[1] ?? '');
|
|
949
|
+
}
|
|
950
|
+
/**
|
|
951
|
+
* Whether a line whose key is `key` is a prefix-tier landing for `needle`: the
|
|
952
|
+
* key starts with it at a word boundary, runs at most {@link MAX_TITLE_CHARS}
|
|
953
|
+
* characters, and does not run on into a longer title.
|
|
954
|
+
*/
|
|
955
|
+
function startsWithTitle(line, key, needle) {
|
|
956
|
+
return (key.length <= MAX_TITLE_CHARS &&
|
|
957
|
+
key.startsWith(needle) &&
|
|
958
|
+
!/[\p{L}\p{N}]/u.test(key.charAt(needle.length)) &&
|
|
959
|
+
!continuesIntoLongerTitle(line, key, needle));
|
|
960
|
+
}
|
|
961
|
+
/**
|
|
962
|
+
* The document line `section` names — a mixed-case subheading ("Liquidity and
|
|
963
|
+
* Capital Resources") that {@link detectHeadings} never lists (#166). Lines
|
|
964
|
+
* compare by {@link headingKey} on both sides, in two tiers:
|
|
965
|
+
* 1. A line equal to the needle.
|
|
966
|
+
* 2. Only when no line is equal: a line that starts with the needle at a word
|
|
967
|
+
* boundary and runs at most {@link MAX_TITLE_CHARS} characters — a heading
|
|
968
|
+
* with a date suffix ("Consolidated Schedule of Investments as of June 30,
|
|
969
|
+
* 2026") — unless it runs on into a longer title
|
|
970
|
+
* ({@link continuesIntoLongerTitle}). Equality goes first because the prefix
|
|
971
|
+
* tier alone would take a bullet such as "Competition among vendors…" ahead
|
|
972
|
+
* of the "Competition" heading.
|
|
973
|
+
*
|
|
974
|
+
* The result says which tier hit, because `section` ranks the two apart (#177):
|
|
975
|
+
* an equal line ahead of a heading that only contains the value, a prefix line
|
|
976
|
+
* after every heading. One pass yields both, so a caller needing each pays for
|
|
977
|
+
* one scan.
|
|
978
|
+
*
|
|
979
|
+
* Both tiers also compare the line's key with a leading 20-F sub-item label
|
|
980
|
+
* removed ({@link withoutSubItemLabel}, #175), so `risk factors` reaches "3D.
|
|
981
|
+
* RISK FACTORS"; the longer-title test then judges the line after its label
|
|
982
|
+
* ("B. Business Overview" keeps `business` out). The needle is never stripped,
|
|
983
|
+
* so a labelled value still matches its own line. The landing is the line's
|
|
984
|
+
* first character, label included.
|
|
985
|
+
*
|
|
986
|
+
* In both tiers a line is skipped when it opens on a lowercase letter (a
|
|
987
|
+
* wrapped sentence or bullet: "competition in our market…", a lone "business."
|
|
988
|
+
* ending a wrapped paragraph), sits under a line ending mid-sentence or opens on
|
|
989
|
+
* an unpunctuated Item marker followed by a lowercase word (a wrapped
|
|
990
|
+
* cross-reference, the guards heading detection applies: "…in Part II,\nItem 8
|
|
991
|
+
* on this Form 10-K…"), or is a TOC row ({@link isTocRow}). The first remaining
|
|
992
|
+
* line wins; its offset is the line's first non-blank character. One forward
|
|
993
|
+
* pass over the text; a line shorter than the needle is never folded.
|
|
994
|
+
*/
|
|
995
|
+
export function matchSectionLine(text, section) {
|
|
996
|
+
const needle = headingKey(section);
|
|
997
|
+
if (needle.length === 0)
|
|
998
|
+
return;
|
|
999
|
+
let prefixHit;
|
|
1000
|
+
for (let start = 0; start <= text.length;) {
|
|
1001
|
+
const newline = text.indexOf('\n', start);
|
|
1002
|
+
const end = newline === -1 ? text.length : newline;
|
|
1003
|
+
const line = text.slice(start, end);
|
|
1004
|
+
const lineStart = start;
|
|
1005
|
+
start = end + 1;
|
|
1006
|
+
if (line.length < needle.length)
|
|
1007
|
+
continue;
|
|
1008
|
+
const key = headingKey(line);
|
|
1009
|
+
const unlabelled = withoutSubItemLabel(key);
|
|
1010
|
+
const equal = key === needle || unlabelled === needle;
|
|
1011
|
+
if (!equal &&
|
|
1012
|
+
(prefixHit !== undefined ||
|
|
1013
|
+
!(startsWithTitle(line, key, needle) ||
|
|
1014
|
+
(unlabelled !== undefined &&
|
|
1015
|
+
startsWithTitle(line.trimStart().replace(/^\S+\s+/, ''), unlabelled, needle))))) {
|
|
1016
|
+
continue;
|
|
1017
|
+
}
|
|
1018
|
+
const content = line.trimStart();
|
|
1019
|
+
if (/^\p{Ll}/u.test(content))
|
|
1020
|
+
continue;
|
|
1021
|
+
if (followsMidSentence(text, lineStart) || isWrappedProse(text, lineStart, content))
|
|
1022
|
+
continue;
|
|
1023
|
+
if (isTocRow(text, line, start))
|
|
1024
|
+
continue;
|
|
1025
|
+
const offset = lineStart + line.length - content.length;
|
|
1026
|
+
if (equal)
|
|
1027
|
+
return { equal, offset };
|
|
1028
|
+
prefixHit = offset;
|
|
1029
|
+
}
|
|
1030
|
+
return prefixHit === undefined ? undefined : { equal: false, offset: prefixHit };
|
|
1031
|
+
}
|
|
1032
|
+
/** The offset of the line {@link matchSectionLine} lands on: the equal line, else the prefix line. */
|
|
1033
|
+
export function findSectionLine(text, section) {
|
|
1034
|
+
return matchSectionLine(text, section)?.offset;
|
|
1035
|
+
}
|
|
532
1036
|
/** Convert filing HTML to plain text, optionally truncating to a character limit. */
|
|
533
1037
|
export function filingToText(html, limit) {
|
|
534
1038
|
const full = filingToExtract(html);
|