@cyanheads/secedgar-mcp-server 0.14.0 → 0.15.0

package/dist/mcp-server/tools/definitions/search-filings.tool.js

@@ -1,19 +1,24 @@
 /**
  * @fileoverview Search EDGAR filings since 1993 — EFTS full-text for 2001-present,
  * archive-backed browse (submissions history + quarterly full-index) for pre-2001
- * date ranges.
+ * date ranges, a bounded local document scan for pre-2001 entity-scoped free text,
+ * and an auto-split merge for a range straddling the full-text floor.
  * @module mcp-server/tools/definitions/search-filings
  */
 import { tool, z } from '@cyanheads/mcp-ts-core';
 import { JsonRpcErrorCode, McpError, notFound, validationError, } from '@cyanheads/mcp-ts-core/errors';
 import { getCanvasBridge, toDatasetField } from '../../../services/canvas-bridge/canvas-bridge.js';
 import { cleanDisplayName, getEdgarApiService, quartersInRange, selectArchivePages, } from '../../../services/edgar/edgar-api-service.js';
+import { filingToExtract } from '../../../services/edgar/filing-to-text.js';
 /**
  * The EFTS full-text index only covers filings from this date onward, even
  * though the archives reach 1993. A date range entirely before this floor is
- * routed to the archive-backed paths; a range straddling it is rejected (#77).
+ * routed to the archive-backed paths; a range straddling it is split at the
+ * floor and the two sides merged (#87).
  */
 const EFTS_FULLTEXT_FLOOR = '2001-01-01';
+/** Last day the archive arms serve when a range is split at {@link EFTS_FULLTEXT_FLOOR}. */
+const ARCHIVE_ERA_END = '2000-12-31';
 /**
  * Cap on submissions archive pages fetched in one call (pre-2001 entity-scoped
  * arm) — bounds latency and the rate-limited request budget. Mirrors the same
@@ -28,6 +33,14 @@ const ARCHIVE_PAGE_SCAN_CAP = 10;
  * hard, and `dataset.truncated` discloses when older quarters went unscanned.
  */
 const FULL_INDEX_QUARTER_SCAN_CAP = 8;
+/**
+ * Cap on filing documents fetched in one pre-2001 local text scan. The cost is
+ * SEC's rate limit, not the work: a document fetches in ~47ms and converts in
+ * ~16ms, so at `EDGAR_RATE_LIMIT_RPS` (default 10) a full 50-document scan is
+ * ~5s of throttling and little else. `scan.capped` discloses when candidates
+ * were left unscanned (#87).
+ */
+const DOCUMENT_SCAN_CAP = 50;
 function eftsHitToRow(hit) {
     const displayName = hit._source.display_names?.[0] || '';
     return {
@@ -134,20 +147,84 @@ async function resolveEntityTargeting(query, ctx) {
     return { query };
 }
 /**
- * Shared tail for the archive-backed arms: sort the fully-matched rows, slice
- * the inline window, recompute the form distribution, register the full set to
- * the canvas (source-tagged), and emit the query echo / boundary notices.
- * `scanTruncated` (a scan cap bound before the range was exhausted) drives both
- * `total_is_exact` and the dataframe `truncated` flag.
+ * Compile one query term to a matcher over converted filing text. Bare words and
+ * phrases are word-boundary anchored so `revenue` does not match `unrevenued`;
+ * a `*` suffix drops the trailing boundary for prefix matching. Whitespace inside
+ * a phrase matches any run, because line wrapping in the converted text is not
+ * the filer's.
+ */
+function termToRegExp(term) {
+    const wildcard = term.endsWith('*');
+    const body = wildcard ? term.slice(0, -1) : term;
+    const escaped = body.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/\s+/g, '\\s+');
+    const lead = /^\w/.test(body) ? '\\b' : '';
+    const tail = wildcard ? '' : /\w$/.test(body) ? '\\b' : '';
+    return new RegExp(`${lead}${escaped}${tail}`, 'i');
+}
+/**
+ * Parse a free-text query into {@link LocalQuery}. Tokens are whitespace-separated
+ * except inside double quotes; a leading `-` marks an exclusion; a bare uppercase
+ * `OR` opens a new alternative group (EDGAR's own convention — lowercase "or" is
+ * an ordinary search term).
  */
-async function finishArchiveResult(ctx, args) {
-    // Full-index and submissions have no relevance score, so relevance collapses
-    // to date-descending; only an explicit ascending request reorders.
-    const sorted = [...args.matched].sort((a, b) => args.sort === 'filing_date_asc'
+function parseLocalQuery(query) {
+    const groups = [];
+    const exclusions = [];
+    let current = [];
+    for (const token of query.match(/-?"[^"]*"|\S+/g) ?? []) {
+        if (token === 'OR') {
+            groups.push(current);
+            current = [];
+            continue;
+        }
+        const negated = token.startsWith('-');
+        const term = (negated ? token.slice(1) : token).replaceAll('"', '').trim();
+        if (!term)
+            continue;
+        if (negated)
+            exclusions.push(termToRegExp(term));
+        else
+            current.push(termToRegExp(term));
+    }
+    groups.push(current);
+    return { groups: groups.filter((group) => group.length > 0), exclusions };
+}
+/** True when the converted filing text satisfies the compiled query. */
+function matchesLocalQuery(text, parsed) {
+    if (parsed.exclusions.some((re) => re.test(text)))
+        return false;
+    if (parsed.groups.length === 0)
+        return true;
+    return parsed.groups.some((group) => group.every((re) => re.test(text)));
+}
+/**
+ * Order rows for the inline window. The archives carry no relevance score, so
+ * relevance collapses to date-descending; only an explicit ascending request
+ * reorders. Also picks which end of a candidate list the scan cap keeps.
+ */
+function sortRows(rows, sort) {
+    return [...rows].sort((a, b) => sort === 'filing_date_asc'
         ? a.filing_date.localeCompare(b.filing_date)
         : b.filing_date.localeCompare(a.filing_date));
-    const total = sorted.length;
-    const totalIsExact = !args.scanTruncated;
+}
+/**
+ * Shared tail for every row set this tool assembles in memory (the pre-2001 arms
+ * and the straddling merge): sort, slice the inline window, recompute the form
+ * distribution, register the full set to the canvas (source-tagged), and emit the
+ * query echo / boundary notices.
+ *
+ * **`offset` model.** `offset` indexes the sorted row list assembled by this call
+ * — every row the tool holds, across every source it read. The 2001+ full-text
+ * path instead offsets into a single ≤100-row EFTS window (or, under
+ * sort=relevance, EDGAR's server-side pagination). The assembled-list model is
+ * the only one a merge can honor: once archive rows interleave with EFTS rows by
+ * date, an offset relative to either source's own window addresses nothing the
+ * caller can see. `total` may exceed the assembled row count (EFTS reports its
+ * true match count while returning one window), so paging past the rows in hand
+ * is a real state — the offset-exceeded notice names it.
+ */
+async function finishAssembledResult(ctx, args) {
+    const sorted = sortRows(args.matched, args.sort);
     const sliced = sorted.slice(args.offset, args.offset + args.limit);
     const results = sliced.map(toInlineResult);
     const formDistribution = buildFormDistribution(sorted.map((row) => row.form));
@@ -158,44 +235,45 @@ async function finishArchiveResult(ctx, args) {
             rows: sorted,
             sourceTool: 'secedgar_search_filings',
             queryParams: args.queryParams,
-            truncated: args.scanTruncated,
+            truncated: args.truncated,
         });
         if (registered)
-            dataset = { ...toDatasetField(registered), truncated: args.scanTruncated };
+            dataset = { ...toDatasetField(registered), truncated: args.truncated };
     }
     ctx.enrich.echo(args.effectiveQuery);
-    if (total === 0) {
+    if (args.total === 0) {
         ctx.enrich.notice(`No filings matched ${args.zeroHitCriteria}. ${args.coverageNote}`);
     }
     else if (results.length === 0 && args.offset >= sorted.length) {
-        ctx.enrich.notice(`Offset (${args.offset}) exceeds the ${sorted.length} matched filings. ${total} filings matched — lower the offset or query the full set via secedgar_dataframe_query.`);
+        ctx.enrich.notice(`Offset (${args.offset}) exceeds the ${sorted.length} rows this search assembled. ${args.total} filings matched — lower the offset or query the full set via secedgar_dataframe_query.`);
     }
-    else if (total > results.length) {
+    else if (args.total > results.length) {
         ctx.enrich.truncated({
             shown: results.length,
             cap: args.limit,
-            guidance: args.scanTruncated
-                ? `${args.coverageNote} The scan hit its cap before exhausting the range — older filings may exist beyond the scanned window. Narrow the range, or query the dataframe.`
+            guidance: args.truncated
+                ? `${args.coverageNote} More matches exist beyond the rows fetched — narrow the range, or query the dataframe.`
                 : `${args.coverageNote} Query the full matched set via secedgar_dataframe_query, or page with offset.`,
         });
     }
     return {
-        total,
-        total_is_exact: totalIsExact,
+        total: args.total,
+        total_is_exact: args.totalIsExact,
         results,
         form_distribution: formDistribution,
+        scan: args.scan,
         dataset,
     };
 }
 /**
- * Pre-2001 entity-scoped arm: resolve the CIK's full submissions history (the
- * `recent` window plus the `filings.files[]` archive pages reused from #78) and
- * filter by form + date. Reuses `getSubmissions`/`selectArchivePages`/
- * `fetchArchivePage` as-is. Rows are tagged `source: 'submissions'`; the
- * EFTS-only fields (period_ending, ticker, file_description, sic, location) are
- * left null for a uniform archive-row shape.
+ * Pre-2001 entity-scoped rows: the CIK's full submissions history (the `recent`
+ * window plus the `filings.files[]` archive pages reused from #78) filtered by
+ * form + date. Reuses `getSubmissions`/`selectArchivePages`/`fetchArchivePage`
+ * as-is. Rows are tagged `source: 'submissions'`; the EFTS-only fields
+ * (period_ending, ticker, file_description, sic, location) are left null for a
+ * uniform archive-row shape.
  */
-async function serveSubmissionsArchive(ctx, args) {
+async function collectSubmissionsRows(ctx, args) {
     const api = getEdgarApiService();
     // A `cik:` targeting token is only shape-validated (1-10 digits), never checked
     // against the ticker cache, so a well-formed filing-agent/transmitter CIK reaches
@@ -255,32 +333,15 @@ async function serveSubmissionsArchive(ctx, args) {
             break;
         pushBlock(await api.fetchArchivePage(page.name));
     }
-    const formsNote = forms ? `, forms ${forms.join(', ')}` : '';
-    return finishArchiveResult(ctx, {
-        matched: rows,
-        sort: args.sort,
-        limit: args.limit,
-        offset: args.offset,
-        scanTruncated,
-        queryParams: {
-            entity_cik: args.entityCik,
-            forms: args.forms,
-            start_date: args.startDate,
-            end_date: args.endDate,
-            source: 'submissions',
-        },
-        effectiveQuery: `(pre-2001 archive: CIK ${args.entityCik}${formsNote}, ${args.startDate} to ${args.endDate})`,
-        coverageNote: 'Served from the entity submissions history — full-text search covers 2001-present only.',
-        zeroHitCriteria: `CIK ${args.entityCik}${forms ? `, forms [${forms.join(', ')}]` : ''}, dates ${args.startDate} to ${args.endDate}`,
-    });
+    return { rows, scanTruncated };
 }
 /**
- * Pre-2001 unscoped browse arm: fetch the quarterly full-index (`master.idx`)
+ * Pre-2001 unscoped browse rows: fetch the quarterly full-index (`master.idx`)
  * files spanning the range and filter by form + date client-side (the index is
  * not form-filterable server-side). Bounded by FULL_INDEX_QUARTER_SCAN_CAP —
  * the heaviest arm. Rows are tagged `source: 'full-index'`.
  */
-async function serveFullIndexBrowse(ctx, args) {
+async function collectFullIndexRows(args) {
     const api = getEdgarApiService();
     const quarters = quartersInRange(args.startDate, args.endDate);
     const quarterLimit = Math.min(quarters.length, FULL_INDEX_QUARTER_SCAN_CAP);
@@ -311,25 +372,69 @@ async function serveFullIndexBrowse(ctx, args) {
             });
         }
     }
-    return finishArchiveResult(ctx, {
-        matched: rows,
-        sort: args.sort,
-        limit: args.limit,
-        offset: args.offset,
-        scanTruncated,
-        queryParams: {
-            forms: args.forms,
-            start_date: args.startDate,
-            end_date: args.endDate,
-            source: 'full-index',
-        },
-        effectiveQuery: `(pre-2001 full-index browse: forms ${args.forms.join(', ')}, ${args.startDate} to ${args.endDate})`,
-        coverageNote: 'Served from the quarterly EDGAR full-index — full-text search covers 2001-present only.',
-        zeroHitCriteria: `forms [${args.forms.join(', ')}], dates ${args.startDate} to ${args.endDate}`,
+    return { rows, scanTruncated };
+}
+/**
+ * Read candidate filings and keep the ones whose text satisfies the query — the
+ * pre-2001 entity-scoped free-text arm. There is no pre-2001 full-text index, so
+ * the match is local and the candidate set must already be bounded by the form +
+ * date pre-filter that produced it.
+ *
+ * The unit fetched is the whole accession `.txt`. A pre-1997 filing's index lists
+ * its exhibits with **empty `name` fields**, so per-document URLs do not exist —
+ * SEC's original flat-submission format concatenates every exhibit into that one
+ * file as SGML `<DOCUMENT>` blocks (Apple's 1996 10-K holds 7). A hit can
+ * therefore land in an attached exhibit rather than the primary form's body,
+ * which is why the scan discloses its shape rather than presenting a match as
+ * necessarily being in the requested form.
+ *
+ * Candidates arrive pre-sorted, so the cap keeps the end the caller's `sort`
+ * asked for. Documents are fetched concurrently; `EdgarApiService`'s throttle
+ * serializes them at the configured request rate, which is the real cost.
+ */
+async function scanFilingText(ctx, candidates, parsed) {
+    const api = getEdgarApiService();
+    const targets = candidates.slice(0, DOCUMENT_SCAN_CAP);
+    const outcomes = await Promise.all(targets.map(async (row) => {
+        const text = await api.tryGetFilingDocument(row.cik ?? '', row.accession_number, `${row.accession_number}.txt`);
+        if (text === null)
+            return null;
+        return matchesLocalQuery(filingToExtract(text), parsed) ? row : null;
+    }));
+    const rows = outcomes.filter((row) => row !== null);
+    const scan = {
+        candidates: candidates.length,
+        scanned: targets.length,
+        matched: rows.length,
+        capped: candidates.length > targets.length,
+    };
+    ctx.log.info('Pre-2001 local text scan completed', scan);
+    return { rows, scan };
+}
+/**
+ * Fetch one 100-row EFTS window plus the index's own match total. Used by the
+ * straddling merge for the ≥2001 side, where the rows join an archive row set and
+ * are sorted and sliced together — so the window is always fetched from offset 0
+ * regardless of the caller's `offset`, which addresses the merged list instead.
+ */
+async function fetchEftsRows(args) {
+    const response = await getEdgarApiService().searchFilings({
+        query: args.query,
+        forms: args.forms,
+        ciks: args.entityCik ? [args.entityCik] : undefined,
+        startDate: args.startDate,
+        endDate: args.endDate,
+        from: 0,
+        size: 100,
     });
+    return {
+        rows: response.hits.hits.map(eftsHitToRow),
+        total: response.hits.total.value,
+        totalIsExact: response.hits.total.relation === 'eq',
+    };
 }
 export const searchFilingsTool = tool('secedgar_search_filings', {
-    description: 'Search EDGAR filings since 1993. Full-text search covers 2001-present (the EFTS index floor); pre-2001 date ranges (to 1993) are served from the archives by form and entity/date — pre-2001 full-text matching requires entity scope (ticker:/cik:). Supports exact phrases, boolean operators, wildcards, and entity targeting (ticker:AAPL or cik:320193 in query).',
+    description: "Search EDGAR filings since 1993. Full-text search covers 2001-present (the EFTS index floor); pre-2001 date ranges (to 1993) are served from the archives by form and entity/date. Pre-2001 free text needs entity scope (ticker:/cik:) — with it, the tool reads the entity's matching filings and matches the terms locally, which costs a few seconds (SEC's request rate caps the scan at roughly 5s for the 50-document maximum). A range crossing 2001-01-01 is split at the boundary and the two eras merged, each row tagged with its source. Supports exact phrases, boolean operators, wildcards, and entity targeting (ticker:AAPL or cik:320193 in query).",
     annotations: { readOnlyHint: true, idempotentHint: true, openWorldHint: true },
     // Agent-facing context for the success path — the query as EDGAR executed it and
     // an optional notice for empty results. Populated via ctx.enrich so it reaches
@@ -377,23 +482,11 @@ export const searchFilingsTool = tool('secedgar_search_filings', {
             when: 'Neither a full-text query nor a forms filter was provided (a date range cannot stand alone)',
             recovery: 'Provide search terms, or browse by form type (forms: ["S-1"]) or entity (ticker:AAPL / cik:320193 in the query) — optionally narrowed by date.',
         },
-        {
-            reason: 'straddling_date_range',
-            code: JsonRpcErrorCode.ValidationError,
-            when: 'The date range crosses 2001-01-01, spanning both the EFTS full-text era and the pre-2001 archives',
-            recovery: 'Split at 2001-01-01: run one search for the ≥2001 side and another for the <2001 side, then combine the results.',
-        },
         {
             reason: 'pre2001_full_text_unscoped',
             code: JsonRpcErrorCode.ValidationError,
-            when: 'A pre-2001 date range carries free-text terms with no entity scope — no pre-2001 full-text index exists',
-            recovery: 'Add ticker: or cik: entity scope to the query, or drop the text terms to browse pre-2001 filings by form and date.',
-        },
-        {
-            reason: 'pre2001_full_text_scoped',
-            code: JsonRpcErrorCode.ValidationError,
-            when: 'A pre-2001 date range carries free-text terms alongside entity scope — local document matching is not yet supported',
-            recovery: "Drop the text terms to browse this entity's pre-2001 filings by form and date.",
+            when: 'A date range reaching before 2001-01-01 carries free-text terms with no entity scope — no pre-2001 full-text index exists, and nothing bounds a local scan',
+            recovery: 'Add ticker: or cik: entity scope to the query, drop the text terms to browse by form and date, or start the range at 2001-01-01.',
         },
     ],
     input: z.object({
@@ -407,11 +500,11 @@ export const searchFilingsTool = tool('secedgar_search_filings', {
                 .describe('Full-text search query. Supports: exact phrases ("material weakness"), boolean operators (revenue OR income), exclusion (-preliminary), wildcard suffix (account*), entity targeting (ticker:AAPL or cik:320193 in the query). Terms are AND\'d by default.'),
         ])
             .optional()
-            .describe('Full-text search query. Optional — omit (or pass "") to browse by form type and/or entity instead, e.g. every S-1 in a date window, or a company\'s filings via ticker:/cik:. A date range alone is not a valid search; pair it with forms or entity targeting. Full-text terms match only filings from 2001 onward (the EFTS index floor); for a pre-2001 date range, drop the text terms (browse by form/date) or add ticker:/cik: entity scope. When present, supports exact phrases ("material weakness"), boolean operators (revenue OR income), exclusion (-preliminary), wildcard suffix (account*), and entity targeting (ticker:AAPL or cik:320193 in the query); terms are AND\'d by default.'),
+            .describe('Full-text search query. Optional — omit (or pass "") to browse by form type and/or entity instead, e.g. every S-1 in a date window, or a company\'s filings via ticker:/cik:. A date range alone is not a valid search; pair it with forms or entity targeting. The EFTS index that serves free text starts at 2001-01-01; a date range reaching earlier needs ticker:/cik: entity scope, which lets the tool read that entity\'s filings and match the terms locally (bounded to 50 documents, a few seconds at SEC\'s request rate), or drop the text terms to browse by form and date. When present, supports exact phrases ("material weakness"), boolean operators (revenue OR income), exclusion (-preliminary), wildcard suffix (account*), and entity targeting (ticker:AAPL or cik:320193 in the query); terms are AND\'d by default. The pre-2001 local scan honors the same phrase / OR / exclusion / wildcard syntax.'),
         forms: z
             .array(z.string())
             .optional()
-            .describe('Filter to specific form types (e.g., ["10-K", "10-Q", "8-K"]). Without this, searches all form types. Note: "10-K" also matches amendments filed as 10-K/A. Ownership forms (3, 4, 5) are indexed by the reporting person (e.g., "LEVINSON ARTHUR D"), not the issuer — rows carry no transaction code, share count, or price. Use secedgar_get_insider_transactions to retrieve parsed ownership XML with person, relationship, transaction code, shares, and price.'),
+            .describe('Filter to specific form types (e.g., ["10-K", "10-Q", "8-K"]). Without this, searches all form types. Note: "10-K" also matches amendments filed as 10-K/A. SEC renamed the blockholder schedules on 2024-12-18 — filings before that date are "SC 13D"/"SC 13G", filings after are "SCHEDULE 13D"/"SCHEDULE 13G" — so a filter spanning that boundary must list both spellings. Ownership forms (3, 4, 5) are indexed by the reporting person (e.g., "LEVINSON ARTHUR D"), not the issuer — rows carry no transaction code, share count, or price. Use secedgar_get_insider_transactions to retrieve parsed ownership XML with person, relationship, transaction code, shares, and price.'),
         start_date: z
             .union([
             z.literal(''),
@@ -439,7 +532,7 @@ export const searchFilingsTool = tool('secedgar_search_filings', {
             .min(0)
             .max(9999)
             .default(0)
-            .describe('Pagination offset. For sort=relevance, EDGAR pages server-side up to its 10,000-result cap. For date sorts (the default) and entity targeting, the tool fetches a single 100-row window and slices it client-side — offsets at or past the window return nothing; switch to sort=relevance for deep pagination, or narrow the search (forms, dates, entity targeting).'),
+            .describe('Pagination offset. For sort=relevance on a 2001-onward search, EDGAR pages server-side up to its 10,000-result cap. Everywhere else the offset indexes the rows this call assembled and sorted: a single 100-row window for date sorts and entity targeting, the full matched set on a pre-2001 archive path, or both together on a range that crosses 2001-01-01. Offsets at or past those rows return nothing even when total is larger — switch to sort=relevance for deep pagination on a 2001-onward search, narrow the search (forms, dates, entity targeting), or query the dataframe. On a crossing range the two sides are assembled unevenly — the archive side contributes every row it matched, the full-text side one window of its total — so once the window runs out the rows jump to the pre-2001 era with the remaining full-text matches absent from the middle; search the 2001-onward era on its own to page through those.'),
         sort: z
             .enum(['filing_date_desc', 'filing_date_asc', 'relevance'])
             .default('filing_date_desc')
@@ -448,10 +541,10 @@ export const searchFilingsTool = tool('secedgar_search_filings', {
     output: z.object({
         total: z
             .number()
-            .describe("Total matching filings. On the full-text (2001+) path this is capped at 10,000; entity targeting (ticker:/cik:) scopes server-side via the EFTS ciks param, so it is the entity's exact match count up to the cap. On a pre-2001 archive path it is the exact count within the scanned window (see total_is_exact)."),
+            .describe("Total matching filings, which can exceed the rows returned inline or materialized. On the full-text (2001+) path this is capped at 10,000; entity targeting (ticker:/cik:) scopes server-side via the EFTS ciks param, so it is the entity's exact match count up to the cap. On a pre-2001 archive path it is the exact count within the scanned window (see total_is_exact). On a range crossing 2001-01-01 it is the sum of both eras' counts."),
         total_is_exact: z
             .boolean()
-            .describe('False when total is a lower bound — the full-text path hit its 10,000 cap, or a pre-2001 archive scan hit its page/quarter cap before exhausting the range.'),
+            .describe('False when total is a lower bound — the full-text path hit its 10,000 cap, a pre-2001 archive scan hit its page/quarter cap before exhausting the range, or a pre-2001 local text scan hit its document cap (scan.capped).'),
         results: z
             .array(z
             .object({
@@ -466,7 +559,7 @@ export const searchFilingsTool = tool('secedgar_search_filings', {
             period_ending: z
                 .string()
                 .optional()
-                .describe('Period the filing reports on (YYYY-MM-DD). Absent for filings without a reporting period (e.g., proxy statements, ownership reports) and for all pre-2001 archive-sourced rows (source submissions/full-index), which carry no period field.'),
+                .describe('Period the filing reports on (YYYY-MM-DD). Absent for filings without a reporting period (e.g., proxy statements, ownership reports) and for all pre-2001 archive-sourced rows (source submissions/full-index), which carry no period field. A range crossing 2001-01-01 returns both kinds of row together, so this field is populated on source=efts rows only.'),
             company_name: z
                 .string()
                 .describe('Filing entity, with ticker/CIK parentheticals stripped.'),
@@ -474,23 +567,23 @@ export const searchFilingsTool = tool('secedgar_search_filings', {
             ticker: z
                 .string()
                 .optional()
-                .describe('Primary ticker symbol parsed from the EFTS display name. Absent for private filers, foreign filers without a US listing, filings whose display name omits the ticker parenthetical, and all pre-2001 archive-sourced rows. For multi-class issuers (e.g., BRK-A / BRK-B), this is the first class listed.'),
+                .describe('Primary ticker symbol parsed from the EFTS display name. Absent for private filers, foreign filers without a US listing, filings whose display name omits the ticker parenthetical, and all pre-2001 archive-sourced rows. A range crossing 2001-01-01 returns both kinds of row together, so this field is populated on source=efts rows only. For multi-class issuers (e.g., BRK-A / BRK-B), this is the first class listed.'),
             file_description: z
                 .string()
                 .optional()
-                .describe('SEC-provided description of the matching document (e.g., "EX-99.1"). Absent when SEC published none, and for pre-2001 archive-sourced rows.'),
+                .describe('SEC-provided description of the matching document (e.g., "EX-99.1"). Absent when SEC published none, and for pre-2001 archive-sourced rows. A range crossing 2001-01-01 returns both kinds of row together, so this field is populated on source=efts rows only.'),
             sic: z
                 .string()
                 .optional()
-                .describe('SIC industry code for the filer. Absent for filers without a classification, and for pre-2001 archive-sourced rows.'),
+                .describe('SIC industry code for the filer. Absent for filers without a classification, and for pre-2001 archive-sourced rows. A range crossing 2001-01-01 returns both kinds of row together, so this field is populated on source=efts rows only.'),
             location: z
                 .string()
                 .optional()
-                .describe('Business location (state or country code). Absent when SEC has no location for this filer, and for pre-2001 archive-sourced rows.'),
+                .describe('Business location (state or country code). Absent when SEC has no location for this filer, and for pre-2001 archive-sourced rows. A range crossing 2001-01-01 returns both kinds of row together, so this field is populated on source=efts rows only.'),
             source: z
                 .enum(['efts', 'submissions', 'full-index'])
                 .optional()
-                .describe('Which EDGAR backend served this row: "efts" (2001+ full-text index), "submissions" (a pre-2001 entity-scoped filing history), or "full-index" (a pre-2001 unscoped quarterly index browse). Provenance is carried into the canvas dataframe as a `source` column.'),
+                .describe('Which EDGAR backend served this row: "efts" (2001+ full-text index), "submissions" (a pre-2001 entity-scoped filing history), or "full-index" (a pre-2001 unscoped quarterly index browse). A date range crossing 2001-01-01 is split at the boundary and returns rows of two sources in one result set, so read this per row rather than per result. Provenance is carried into the canvas dataframe as a `source` column.'),
         })
             .describe('One matching filing hit.'))
             .describe('Matching filings.'),
@@ -498,6 +591,21 @@ export const searchFilingsTool = tool('secedgar_search_filings', {
             .record(z.string(), z.number())
             .optional()
             .describe('Count of results by form type. Helps narrow follow-up searches.'),
+        scan: z
+            .object({
+            candidates: z
+                .number()
+                .describe('Filings the form + date pre-filter selected before any document was read.'),
+            scanned: z
+                .number()
+                .describe('Candidate documents actually fetched and matched against. Capped at 50 per call.'),
+            matched: z.number().describe('Scanned filings whose text satisfied the query terms.'),
+            capped: z
+                .boolean()
+                .describe('True when candidates exceeded the document cap, so the unscanned remainder may hold further matches — narrow the form or date filter to bring them into range.'),
+        })
+            .optional()
+            .describe("Present only on the pre-2001 entity-scoped free-text path, where no full-text index exists and terms are matched by reading documents. Reports the scan's shape so a partial read is never presented as a complete one. Each document read is the whole accession .txt — SEC's original flat-submission format concatenates every exhibit into one file, and pre-1997 filings expose no per-document URL at all — so a match may sit in an attached exhibit rather than the body of the requested form. Absent on every other path."),
         dataset: z
             .object({
             name: z
@@ -531,46 +639,106 @@ export const searchFilingsTool = tool('secedgar_search_filings', {
         const { query, entityCik } = await resolveEntityTargeting(input.query ?? '', ctx);
         const api = getEdgarApiService();
         // Date-based routing at the EFTS full-text floor (2001-01-01). EFTS only
-        // indexes filings from 2001; a range entirely before it is served from the
-        // archives, a range crossing it is rejected with a split instruction (#77).
+        // indexes filings from 2001, so a range reaching earlier is served from the
+        // archives — entirely, or (when it also reaches 2001 onward) as the older half
+        // of a two-source merge split at the floor (#77, #87).
         const startDate = input.start_date || undefined;
         const endDate = input.end_date || undefined;
-        if (startDate && endDate && endDate < EFTS_FULLTEXT_FLOOR) {
-            const hasFreeText = query.trim().length > 0;
-            if (hasFreeText) {
-                // No pre-2001 full-text index exists. Reject honestly rather than silently
-                // dropping the text terms; bounded local matching is a deferred follow-up.
-                if (entityCik) {
-                    throw ctx.fail('pre2001_full_text_scoped', "Full-text matching of pre-2001 filings is not yet supported. Drop the text terms to browse this entity's pre-2001 filings by form and date.", { ...ctx.recoveryFor('pre2001_full_text_scoped') });
-                }
-                throw ctx.fail('pre2001_full_text_unscoped', 'Full-text search is unavailable before 2001 (the EFTS index starts at 2001-01-01). For pre-2001 filings, add ticker:/cik: entity scope, or drop the text terms to browse by form and date.', { ...ctx.recoveryFor('pre2001_full_text_unscoped') });
+        const hasFreeText = query.trim().length > 0;
+        if (startDate && endDate && startDate < EFTS_FULLTEXT_FLOOR) {
+            const straddles = endDate >= EFTS_FULLTEXT_FLOOR;
+            const archiveEnd = straddles ? ARCHIVE_ERA_END : endDate;
+            // Free text before 2001 is matched by reading documents, which only an entity
+            // scope bounds. Unscoped, the pre-2001 half of the range cannot be served at
+            // all — reject rather than answer half a range and call it the answer.
+            if (hasFreeText && !entityCik) {
+                throw ctx.fail('pre2001_full_text_unscoped', `Full-text search is unavailable before ${EFTS_FULLTEXT_FLOOR} (where the EFTS index starts), so the ${startDate}..${archiveEnd} part of this range has no index to search. Add ticker:/cik: entity scope to match those filings by reading them, drop the text terms to browse by form and date, or start the range at ${EFTS_FULLTEXT_FLOOR}.`, { ...ctx.recoveryFor('pre2001_full_text_unscoped') });
             }
+            let archive;
+            let scan;
             if (entityCik) {
-                return await serveSubmissionsArchive(ctx, {
+                archive = await collectSubmissionsRows(ctx, {
                     entityCik,
                     forms: input.forms,
                     startDate,
-                    endDate,
-                    sort: input.sort,
-                    limit: input.limit,
-                    offset: input.offset,
+                    endDate: archiveEnd,
+                });
+                if (hasFreeText) {
+                    const scanned = await scanFilingText(ctx, sortRows(archive.rows, input.sort), parseLocalQuery(query));
+                    archive = { rows: scanned.rows, scanTruncated: archive.scanTruncated };
+                    scan = scanned.scan;
+                }
+            }
+            else {
+                // Unscoped + no free text → forms are guaranteed present by the missing_criteria guard.
+                archive = await collectFullIndexRows({
+                    forms: input.forms ?? [],
+                    startDate,
+                    endDate: archiveEnd,
                 });
             }
-            // Unscoped + no free text → forms are guaranteed present by the missing_criteria guard.
-            return await serveFullIndexBrowse(ctx, {
-                forms: input.forms ?? [],
-                startDate,
-                endDate,
+            const efts = straddles
+                ? await fetchEftsRows({
+                    query,
+                    forms: input.forms,
+                    entityCik,
+                    startDate: EFTS_FULLTEXT_FLOOR,
+                    endDate,
+                })
+                : undefined;
+            // `total` counts both eras' true match totals: the archive side's count is
+            // exact within what it scanned, while EFTS reports its own total independently
+            // of the single window fetched.
+            const archiveIncomplete = archive.scanTruncated || (scan?.capped ?? false);
+            const total = archive.rows.length + (efts?.total ?? 0);
+            const eftsWindowIncomplete = efts !== undefined && efts.total > efts.rows.length;
+            const formsNote = input.forms?.length ? `, forms ${input.forms.join(', ')}` : '';
+            const scopeNote = entityCik ? `CIK ${entityCik}` : 'full-index browse';
+            const archiveSource = entityCik
+                ? 'the entity submissions history'
+                : 'the quarterly EDGAR full-index';
+            const scanNote = scan
+                ? ` Read ${scan.scanned} of ${scan.candidates} candidate filings for the text terms${scan.capped ? ` (document cap ${DOCUMENT_SCAN_CAP})` : ''}; ${scan.matched} matched. Each read covers the whole accession .txt, so a match may sit in an attached exhibit rather than the body of the requested form.`
+                : '';
+            // The archive side contributes every row it matched; the full-text side contributes one
+            // window of a total it reports independently. When that window is short of its total,
+            // the merged list runs out of full-text rows and drops straight into the archive era —
+            // a gap in the middle of the date order that neither offset nor the dataframe reaches,
+            // since both address the rows this call assembled.
+            const windowNote = efts && eftsWindowIncomplete
+                ? ` The ${EFTS_FULLTEXT_FLOOR}-onward side contributed one ${efts.rows.length}-row window of its ${efts.total} matches, so the merged rows jump from the end of that window straight to ${ARCHIVE_ERA_END} — search ${EFTS_FULLTEXT_FLOOR} to ${endDate} on its own to page through the rest.`
+                : '';
+            return await finishAssembledResult(ctx, {
+                matched: [...archive.rows, ...(efts?.rows ?? [])],
+                total,
+                totalIsExact: !archiveIncomplete && (efts?.totalIsExact ?? true),
                 sort: input.sort,
                 limit: input.limit,
                 offset: input.offset,
+                truncated: archiveIncomplete || eftsWindowIncomplete,
+                queryParams: {
+                    query: hasFreeText ? query : undefined,
+                    entity_cik: entityCik,
+                    forms: input.forms,
+                    start_date: startDate,
+                    end_date: endDate,
+                    source: straddles ? 'efts+archive' : entityCik ? 'submissions' : 'full-index',
+                },
+                effectiveQuery: straddles
+                    ? `${hasFreeText ? `${query} ` : ''}(split at ${EFTS_FULLTEXT_FLOOR}: ${scopeNote}${formsNote}, ${startDate} to ${endDate})`
+                    : `${hasFreeText ? `${query} ` : ''}(pre-2001 archive: ${scopeNote}${formsNote}, ${startDate} to ${endDate})`,
+                coverageNote: straddles
+                    ? `Merged across the ${EFTS_FULLTEXT_FLOOR} boundary — filings from ${EFTS_FULLTEXT_FLOOR} on come from the full-text index, earlier ones from ${archiveSource}. Read each row's source; the EFTS-only fields (period_ending, ticker, file_description, sic, location) are null on archive rows.${windowNote}${scanNote}`
+                    : `Served from ${archiveSource} — full-text search covers 2001-present only.${scanNote}`,
+                zeroHitCriteria: [
+                    ...(hasFreeText ? [`"${query}"`] : []),
+                    ...(entityCik ? [`CIK ${entityCik}`] : []),
+                    ...(input.forms?.length ? [`forms [${input.forms.join(', ')}]`] : []),
+                    `dates ${startDate} to ${endDate}`,
+                ].join(', '),
+                scan,
             });
         }
-        if (startDate && endDate && startDate < EFTS_FULLTEXT_FLOOR) {
-            // endDate ≥ 2001 (handled above) but startDate < 2001 → straddling. Reject
-            // with a split instruction; merged straddling output is a deferred follow-up.
-            throw ctx.fail('straddling_date_range', `The date range ${startDate}..${endDate} crosses 2001-01-01. Full-text search covers 2001-present; pre-2001 filings are served from the archives. Search the two eras separately: ${EFTS_FULLTEXT_FLOOR} onward, and before ${EFTS_FULLTEXT_FLOOR}.`, { ...ctx.recoveryFor('straddling_date_range') });
-        }
         // EFTS scores by relevance and exposes no sort param. When the caller wants
         // a date sort (the default) or entity filtering, we over-fetch the EFTS
         // window and reorder/slice client-side. Pure relevance mode keeps the
@@ -713,10 +881,10 @@ export const searchFilingsTool = tool('secedgar_search_filings', {
         };
     },
     format: (result) => {
-        const source = result.results[0]?.source;
+        const fromArchive = result.results.some((r) => r.source === 'submissions' || r.source === 'full-index');
         const exactness = result.total_is_exact
             ? 'exact'
-            : source === 'submissions' || source === 'full-index'
+            : fromArchive || result.scan
                 ? 'partial — scan cap reached, more may exist'
                 : 'capped at 10,000';
         const lines = [`Found ${result.total} filings (${exactness})`];
@@ -735,6 +903,12 @@ export const searchFilingsTool = tool('secedgar_search_filings', {
                 .join(', ');
             lines.push(`\nForm distribution: ${dist}`);
         }
+        if (result.scan) {
+            const cappedNote = result.scan.capped
+                ? `Capped — the remaining ${result.scan.candidates - result.scan.scanned} went unread; narrow the form or date filter to reach them.`
+                : 'Not capped — every candidate was read.';
+            lines.push(`\nPre-2001 text scan: read ${result.scan.scanned} of ${result.scan.candidates} candidate filings, ${result.scan.matched} matched. ${cappedNote} Each read covers the whole accession .txt, so a match may sit in an attached exhibit rather than the body of the requested form.`);
+        }
         if (result.dataset) {
             const truncatedNote = result.dataset.truncated
                 ? ' (truncated — more matches exist beyond the materialized set)'