@cyanheads/congressgov-mcp-server 0.4.3 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +20 -5
- package/CLAUDE.md +20 -5
- package/README.md +3 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/mcp-server/prompts/definitions/legislative-research.prompt.d.ts +9 -0
- package/dist/mcp-server/prompts/definitions/legislative-research.prompt.d.ts.map +1 -1
- package/dist/mcp-server/prompts/definitions/legislative-research.prompt.js +46 -15
- package/dist/mcp-server/prompts/definitions/legislative-research.prompt.js.map +1 -1
- package/dist/mcp-server/tools/definitions/bill-lookup.tool.d.ts +38 -0
- package/dist/mcp-server/tools/definitions/bill-lookup.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/bill-lookup.tool.js +72 -4
- package/dist/mcp-server/tools/definitions/bill-lookup.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/committee-reports.tool.d.ts +37 -0
- package/dist/mcp-server/tools/definitions/committee-reports.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/committee-reports.tool.js +66 -7
- package/dist/mcp-server/tools/definitions/committee-reports.tool.js.map +1 -1
- package/dist/mcp-server/tools/definitions/daily-record.tool.d.ts +41 -3
- package/dist/mcp-server/tools/definitions/daily-record.tool.d.ts.map +1 -1
- package/dist/mcp-server/tools/definitions/daily-record.tool.js +61 -8
- package/dist/mcp-server/tools/definitions/daily-record.tool.js.map +1 -1
- package/dist/mcp-server/tools/format-helpers.d.ts +9 -5
- package/dist/mcp-server/tools/format-helpers.d.ts.map +1 -1
- package/dist/mcp-server/tools/format-helpers.js +397 -89
- package/dist/mcp-server/tools/format-helpers.js.map +1 -1
- package/dist/mcp-server/tools/tool-helpers.d.ts +67 -0
- package/dist/mcp-server/tools/tool-helpers.d.ts.map +1 -1
- package/dist/mcp-server/tools/tool-helpers.js +85 -0
- package/dist/mcp-server/tools/tool-helpers.js.map +1 -1
- package/dist/services/congress-documents/congress-documents-service.d.ts +85 -0
- package/dist/services/congress-documents/congress-documents-service.d.ts.map +1 -0
- package/dist/services/congress-documents/congress-documents-service.js +289 -0
- package/dist/services/congress-documents/congress-documents-service.js.map +1 -0
- package/dist/services/congress-documents/document-formats.d.ts +29 -0
- package/dist/services/congress-documents/document-formats.d.ts.map +1 -0
- package/dist/services/congress-documents/document-formats.js +51 -0
- package/dist/services/congress-documents/document-formats.js.map +1 -0
- package/dist/services/congress-documents/extract-text-stream.d.ts +58 -0
- package/dist/services/congress-documents/extract-text-stream.d.ts.map +1 -0
- package/dist/services/congress-documents/extract-text-stream.js +689 -0
- package/dist/services/congress-documents/extract-text-stream.js.map +1 -0
- package/dist/services/congress-documents/extract-text.d.ts +27 -0
- package/dist/services/congress-documents/extract-text.d.ts.map +1 -0
- package/dist/services/congress-documents/extract-text.js +87 -0
- package/dist/services/congress-documents/extract-text.js.map +1 -0
- package/dist/services/congress-documents/types.d.ts +45 -0
- package/dist/services/congress-documents/types.d.ts.map +1 -0
- package/dist/services/congress-documents/types.js +6 -0
- package/dist/services/congress-documents/types.js.map +1 -0
- package/package.json +1 -1
- package/server.json +3 -3
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Bounded retrieval of legislative document text from www.congress.gov.
|
|
3
|
+
*
|
|
4
|
+
* The document bodies behind Congress.gov's format URLs live on `www.congress.gov`
|
|
5
|
+
* — a different host from the `api.congress.gov` that `CongressApiService` is
|
|
6
|
+
* scoped to, with no API key, no JSON, and documents that run past a megabyte.
|
|
7
|
+
* That is a second upstream, not a second method on the API client, so it gets
|
|
8
|
+
* its own service the way `SenateVoteService` does for the Senate LIS feed.
|
|
9
|
+
*
|
|
10
|
+
* Two bounds, deliberately separate:
|
|
11
|
+
* - **The fetch** is bounded by {@link MAX_DOCUMENT_BYTES}, a request timeout, and
|
|
12
|
+
* a content-type allowlist. The body is extracted as it streams and only the
|
|
13
|
+
* requested window is kept, so what a read *retains* is flat in the size of
|
|
14
|
+
* the document and the ceiling bounds how long a response may run rather than
|
|
15
|
+
* how much of one fits in a buffer.
|
|
16
|
+
* - **The response** is bounded by an exact character offset/limit window. Offsets
|
|
17
|
+
* index into the extracted plain text and are never snapped to section or
|
|
18
|
+
* paragraph breaks, so feeding `nextOffset` back walks a document with every
|
|
19
|
+
* character returned exactly once — no overlap, no gap.
|
|
20
|
+
*
|
|
21
|
+
* @module services/congress-documents/congress-documents-service
|
|
22
|
+
*/
|
|
23
|
+
import type { Context } from '@cyanheads/mcp-ts-core';
|
|
24
|
+
import type { DocumentContent, FetchDocumentParams } from './types.js';
|
|
25
|
+
/**
|
|
26
|
+
* Ceiling on a single document. The largest bodies Congress.gov publishes are
|
|
27
|
+
* the omnibus appropriations acts — 116th Congress H.R. 133 enrolled runs
|
|
28
|
+
* 6,790,482 bytes as Formatted Text and 9,812,888 as XML — and both formats of a
|
|
29
|
+
* document have to clear it, since a caller sent to the other one by a recovery
|
|
30
|
+
* hint would hit the same refusal. This clears the largest by better than a
|
|
31
|
+
* factor of two; the request deadline is what a runaway response meets first.
|
|
32
|
+
*/
|
|
33
|
+
export declare const MAX_DOCUMENT_BYTES = 25000000;
|
|
34
|
+
/** Bounds the whole exchange, headers and body — a megabyte-scale body needs the room. */
|
|
35
|
+
export declare const REQUEST_TIMEOUT_MS = 30000;
|
|
36
|
+
export declare class CongressDocumentsService {
|
|
37
|
+
private readonly maxDocumentBytes;
|
|
38
|
+
constructor(maxDocumentBytes?: number);
|
|
39
|
+
/**
|
|
40
|
+
* Fetch one document and return the requested character window of its text.
|
|
41
|
+
*
|
|
42
|
+
* `url` comes from Congress.gov's own format metadata — a caller never supplies
|
|
43
|
+
* one — and is still checked against {@link ALLOWED_HOSTS} before any request
|
|
44
|
+
* goes out, so a redirected or malformed upstream link cannot turn this into a
|
|
45
|
+
* general-purpose fetcher.
|
|
46
|
+
*/
|
|
47
|
+
fetchDocument(params: FetchDocumentParams, ctx: Context): Promise<DocumentContent>;
|
|
48
|
+
/** Describe the extracted `[offset, offset + limit)` window and what remains. */
|
|
49
|
+
private describeWindow;
|
|
50
|
+
private assertAllowedHost;
|
|
51
|
+
private doFetch;
|
|
52
|
+
/**
|
|
53
|
+
* A 404 means Congress.gov's metadata named a document its own file host does
|
|
54
|
+
* not hold — a missing document, not a transport failure. Everything else
|
|
55
|
+
* (5xx, timeout, socket error) is the transport, and stays retryable.
|
|
56
|
+
*/
|
|
57
|
+
private classifyFetchError;
|
|
58
|
+
private assertContentType;
|
|
59
|
+
/** Refuse an oversized document on its advertised length, before reading a byte of it. */
|
|
60
|
+
private assertAdvertisedSize;
|
|
61
|
+
/**
|
|
62
|
+
* Extract the requested character window as the body streams, under a hard
|
|
63
|
+
* byte budget that cancels the stream the moment it is exceeded.
|
|
64
|
+
*
|
|
65
|
+
* Nothing but the window is retained: the decoder carries a partial character
|
|
66
|
+
* across a chunk boundary and the extractor carries its parse state, so the
|
|
67
|
+
* live set stays near the window's own size however large the document is.
|
|
68
|
+
* Transient allocation still scales with the body — peak RSS is higher than a
|
|
69
|
+
* buffered read of the same document — but it is garbage, not live, so it is
|
|
70
|
+
* concurrent reads rather than one large read that the ceiling has to survive.
|
|
71
|
+
*/
|
|
72
|
+
private readWindow;
|
|
73
|
+
/**
|
|
74
|
+
* A body that dies partway through is the transport failing, not a malformed
|
|
75
|
+
* document — the same classification a connection that never opened gets.
|
|
76
|
+
*/
|
|
77
|
+
private readChunk;
|
|
78
|
+
private tooLarge;
|
|
79
|
+
private isRetryableError;
|
|
80
|
+
private getRequestContext;
|
|
81
|
+
private getAbortSignal;
|
|
82
|
+
}
|
|
83
|
+
export declare function initCongressDocuments(): void;
|
|
84
|
+
export declare function getCongressDocuments(): CongressDocumentsService;
|
|
85
|
+
//# sourceMappingURL=congress-documents-service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"congress-documents-service.d.ts","sourceRoot":"","sources":["../../../src/services/congress-documents/congress-documents-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAUtD,OAAO,KAAK,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAEvE;;;;;;;GAOG;AACH,eAAO,MAAM,kBAAkB,WAAa,CAAC;AAE7C,0FAA0F;AAC1F,eAAO,MAAM,kBAAkB,QAAS,CAAC;AA4CzC,qBAAa,wBAAwB;IACvB,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IAA7C,YAA6B,gBAAgB,GAAE,MAA2B,EAAI;IAE9E;;;;;;;OAOG;IACG,aAAa,CAAC,MAAM,EAAE,mBAAmB,EAAE,GAAG,EAAE,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC,CAiBvF;IAED,iFAAiF;IACjF,OAAO,CAAC,cAAc;IAiCtB,OAAO,CAAC,iBAAiB;YAoBX,OAAO;IAsBrB;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;IAqB1B,OAAO,CAAC,iBAAiB;IAgBzB,0FAA0F;IAC1F,OAAO,CAAC,oBAAoB;IAQ5B;;;;;;;;;;OAUG;YACW,UAAU;IAuCxB;;;OAGG;YACW,SAAS;IAevB,OAAO,CAAC,QAAQ;IAahB,OAAO,CAAC,gBAAgB;IAWxB,OAAO,CAAC,iBAAiB;IASzB,OAAO,CAAC,cAAc;CAIvB;AAID,wBAAgB,qBAAqB,IAAI,IAAI,CAE5C;AAED,wBAAgB,oBAAoB,IAAI,wBAAwB,CAO/D"}
|
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Bounded retrieval of legislative document text from www.congress.gov.
|
|
3
|
+
*
|
|
4
|
+
* The document bodies behind Congress.gov's format URLs live on `www.congress.gov`
|
|
5
|
+
* — a different host from the `api.congress.gov` that `CongressApiService` is
|
|
6
|
+
* scoped to, with no API key, no JSON, and documents that run past a megabyte.
|
|
7
|
+
* That is a second upstream, not a second method on the API client, so it gets
|
|
8
|
+
* its own service the way `SenateVoteService` does for the Senate LIS feed.
|
|
9
|
+
*
|
|
10
|
+
* Two bounds, deliberately separate:
|
|
11
|
+
* - **The fetch** is bounded by {@link MAX_DOCUMENT_BYTES}, a request timeout, and
|
|
12
|
+
* a content-type allowlist. The body is extracted as it streams and only the
|
|
13
|
+
* requested window is kept, so what a read *retains* is flat in the size of
|
|
14
|
+
* the document and the ceiling bounds how long a response may run rather than
|
|
15
|
+
* how much of one fits in a buffer.
|
|
16
|
+
* - **The response** is bounded by an exact character offset/limit window. Offsets
|
|
17
|
+
* index into the extracted plain text and are never snapped to section or
|
|
18
|
+
* paragraph breaks, so feeding `nextOffset` back walks a document with every
|
|
19
|
+
* character returned exactly once — no overlap, no gap.
|
|
20
|
+
*
|
|
21
|
+
* @module services/congress-documents/congress-documents-service
|
|
22
|
+
*/
|
|
23
|
+
import { invalidParams, JsonRpcErrorCode, McpError, notFound, serviceUnavailable, } from '@cyanheads/mcp-ts-core/errors';
|
|
24
|
+
import { fetchWithTimeout, withRetry } from '@cyanheads/mcp-ts-core/utils';
|
|
25
|
+
import { createStreamingExtractor } from './extract-text-stream.js';
|
|
26
|
+
/**
|
|
27
|
+
* Ceiling on a single document. The largest bodies Congress.gov publishes are
|
|
28
|
+
* the omnibus appropriations acts — 116th Congress H.R. 133 enrolled runs
|
|
29
|
+
* 6,790,482 bytes as Formatted Text and 9,812,888 as XML — and both formats of a
|
|
30
|
+
* document have to clear it, since a caller sent to the other one by a recovery
|
|
31
|
+
* hint would hit the same refusal. This clears the largest by better than a
|
|
32
|
+
* factor of two; the request deadline is what a runaway response meets first.
|
|
33
|
+
*/
|
|
34
|
+
export const MAX_DOCUMENT_BYTES = 25_000_000;
|
|
35
|
+
/** Bounds the whole exchange, headers and body — a megabyte-scale body needs the room. */
|
|
36
|
+
export const REQUEST_TIMEOUT_MS = 30_000;
|
|
37
|
+
const MAX_ATTEMPTS = 3;
|
|
38
|
+
const BASE_BACKOFF_MS = 500;
|
|
39
|
+
/** Every format URL these tools resolve points here; nothing else is fetched. */
|
|
40
|
+
const ALLOWED_HOSTS = new Set(['www.congress.gov']);
|
|
41
|
+
/** Content types the text extractor can read. PDF and everything else are refused. */
|
|
42
|
+
const ALLOWED_CONTENT_TYPES = new Set(['application/xml', 'text/html', 'text/plain', 'text/xml']);
|
|
43
|
+
/**
|
|
44
|
+
* Contract reasons this service raises. The matching `recovery` hints live once
|
|
45
|
+
* in `documentErrorContracts` (tool-helpers) and reach the wire via
|
|
46
|
+
* `ctx.recoveryFor` — services have no `ctx.fail`, but carrying `reason` in
|
|
47
|
+
* `data` plus the resolved hint gives clients the same shape.
|
|
48
|
+
*/
|
|
49
|
+
const DOCUMENT_UNAVAILABLE = 'document_unavailable';
|
|
50
|
+
const DOCUMENT_FETCH_FAILED = 'document_fetch_failed';
|
|
51
|
+
const DOCUMENT_TOO_LARGE = 'document_too_large';
|
|
52
|
+
const OFFSET_PAST_END = 'offset_past_end';
|
|
53
|
+
function isNativeAbortSignal(value) {
|
|
54
|
+
if (typeof AbortSignal !== 'function' ||
|
|
55
|
+
typeof AbortSignal.prototype.throwIfAborted !== 'function' ||
|
|
56
|
+
!value) {
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
try {
|
|
60
|
+
AbortSignal.prototype.throwIfAborted.call(value);
|
|
61
|
+
return true;
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
return !(error instanceof TypeError);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
export class CongressDocumentsService {
|
|
68
|
+
maxDocumentBytes;
|
|
69
|
+
constructor(maxDocumentBytes = MAX_DOCUMENT_BYTES) {
|
|
70
|
+
this.maxDocumentBytes = maxDocumentBytes;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Fetch one document and return the requested character window of its text.
|
|
74
|
+
*
|
|
75
|
+
* `url` comes from Congress.gov's own format metadata — a caller never supplies
|
|
76
|
+
* one — and is still checked against {@link ALLOWED_HOSTS} before any request
|
|
77
|
+
* goes out, so a redirected or malformed upstream link cannot turn this into a
|
|
78
|
+
* general-purpose fetcher.
|
|
79
|
+
*/
|
|
80
|
+
async fetchDocument(params, ctx) {
|
|
81
|
+
this.assertAllowedHost(params.url, ctx);
|
|
82
|
+
const operation = 'CongressDocumentsService GET document';
|
|
83
|
+
const requestContext = this.getRequestContext(ctx, operation);
|
|
84
|
+
const signal = this.getAbortSignal(ctx);
|
|
85
|
+
const window = await withRetry(() => this.doFetch(params, requestContext, ctx, signal), {
|
|
86
|
+
operation,
|
|
87
|
+
context: requestContext,
|
|
88
|
+
baseDelayMs: BASE_BACKOFF_MS,
|
|
89
|
+
maxRetries: MAX_ATTEMPTS - 1,
|
|
90
|
+
isTransient: (error) => this.isRetryableError(error),
|
|
91
|
+
...(signal ? { signal } : {}),
|
|
92
|
+
});
|
|
93
|
+
return this.describeWindow(window, params, ctx);
|
|
94
|
+
}
|
|
95
|
+
/** Describe the extracted `[offset, offset + limit)` window and what remains. */
|
|
96
|
+
describeWindow(window, params, ctx) {
|
|
97
|
+
const { text, totalCharacters } = window;
|
|
98
|
+
const offset = params.characterOffset;
|
|
99
|
+
/** An empty document is a real (if unusual) answer at offset 0 — not a bad offset. */
|
|
100
|
+
if (totalCharacters > 0 && offset >= totalCharacters) {
|
|
101
|
+
throw invalidParams(`characterOffset ${offset} is past the end of this document, which has ${totalCharacters} characters.`, {
|
|
102
|
+
reason: OFFSET_PAST_END,
|
|
103
|
+
...ctx.recoveryFor(OFFSET_PAST_END),
|
|
104
|
+
totalCharacters,
|
|
105
|
+
offset,
|
|
106
|
+
retryable: false,
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
const end = offset + text.length;
|
|
110
|
+
const truncated = end < totalCharacters;
|
|
111
|
+
return {
|
|
112
|
+
text,
|
|
113
|
+
totalCharacters,
|
|
114
|
+
offset,
|
|
115
|
+
truncated,
|
|
116
|
+
nextOffset: truncated ? end : null,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
assertAllowedHost(url, ctx) {
|
|
120
|
+
let host;
|
|
121
|
+
try {
|
|
122
|
+
const parsed = new URL(url);
|
|
123
|
+
if (parsed.protocol === 'https:' || parsed.protocol === 'http:')
|
|
124
|
+
host = parsed.hostname;
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
host = undefined;
|
|
128
|
+
}
|
|
129
|
+
if (host === undefined || !ALLOWED_HOSTS.has(host.toLowerCase())) {
|
|
130
|
+
throw notFound('The upstream metadata points at a document this server does not retrieve — only documents hosted on www.congress.gov are read.', {
|
|
131
|
+
reason: DOCUMENT_UNAVAILABLE,
|
|
132
|
+
...ctx.recoveryFor(DOCUMENT_UNAVAILABLE),
|
|
133
|
+
retryable: false,
|
|
134
|
+
});
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
async doFetch(params, requestContext, ctx, signal) {
|
|
138
|
+
let response;
|
|
139
|
+
try {
|
|
140
|
+
response = await fetchWithTimeout(params.url, REQUEST_TIMEOUT_MS, requestContext, {
|
|
141
|
+
headers: { Accept: 'text/html, application/xml, text/plain, */*' },
|
|
142
|
+
expectedStatuses: [404],
|
|
143
|
+
...(signal ? { signal } : {}),
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
throw this.classifyFetchError(error, ctx);
|
|
148
|
+
}
|
|
149
|
+
this.assertContentType(response, ctx);
|
|
150
|
+
this.assertAdvertisedSize(response, ctx);
|
|
151
|
+
return this.readWindow(response, params, ctx);
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* A 404 means Congress.gov's metadata named a document its own file host does
|
|
155
|
+
* not hold — a missing document, not a transport failure. Everything else
|
|
156
|
+
* (5xx, timeout, socket error) is the transport, and stays retryable.
|
|
157
|
+
*/
|
|
158
|
+
classifyFetchError(error, ctx) {
|
|
159
|
+
if (!(error instanceof McpError))
|
|
160
|
+
return error;
|
|
161
|
+
if (error.code === JsonRpcErrorCode.NotFound) {
|
|
162
|
+
return notFound('Congress.gov does not hold the document its metadata pointed to.', {
|
|
163
|
+
reason: DOCUMENT_UNAVAILABLE,
|
|
164
|
+
...ctx.recoveryFor(DOCUMENT_UNAVAILABLE),
|
|
165
|
+
retryable: false,
|
|
166
|
+
}, { cause: error });
|
|
167
|
+
}
|
|
168
|
+
/** The framework message embeds the request URL; re-message without echoing it. */
|
|
169
|
+
return serviceUnavailable('Unable to retrieve the document from Congress.gov.', { reason: DOCUMENT_FETCH_FAILED, ...ctx.recoveryFor(DOCUMENT_FETCH_FAILED) }, { cause: error });
|
|
170
|
+
}
|
|
171
|
+
assertContentType(response, ctx) {
|
|
172
|
+
const raw = response.headers.get('content-type') ?? '';
|
|
173
|
+
const mediaType = raw.split(';')[0]?.trim().toLowerCase() ?? '';
|
|
174
|
+
if (ALLOWED_CONTENT_TYPES.has(mediaType))
|
|
175
|
+
return;
|
|
176
|
+
throw serviceUnavailable(`Congress.gov returned content type '${mediaType || 'unknown'}', which this server cannot read as text.`, {
|
|
177
|
+
reason: DOCUMENT_FETCH_FAILED,
|
|
178
|
+
...ctx.recoveryFor(DOCUMENT_FETCH_FAILED),
|
|
179
|
+
contentType: mediaType,
|
|
180
|
+
/** A wrong media type is deterministic — retrying re-downloads the same bytes. */
|
|
181
|
+
retryable: false,
|
|
182
|
+
});
|
|
183
|
+
}
|
|
184
|
+
/** Refuse an oversized document on its advertised length, before reading a byte of it. */
|
|
185
|
+
assertAdvertisedSize(response, ctx) {
|
|
186
|
+
const header = response.headers.get('content-length');
|
|
187
|
+
if (header === null)
|
|
188
|
+
return;
|
|
189
|
+
const contentLength = Number(header);
|
|
190
|
+
if (!Number.isFinite(contentLength) || contentLength <= this.maxDocumentBytes)
|
|
191
|
+
return;
|
|
192
|
+
throw this.tooLarge(ctx, { contentLength });
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* Extract the requested character window as the body streams, under a hard
|
|
196
|
+
* byte budget that cancels the stream the moment it is exceeded.
|
|
197
|
+
*
|
|
198
|
+
* Nothing but the window is retained: the decoder carries a partial character
|
|
199
|
+
* across a chunk boundary and the extractor carries its parse state, so the
|
|
200
|
+
* live set stays near the window's own size however large the document is.
|
|
201
|
+
* Transient allocation still scales with the body — peak RSS is higher than a
|
|
202
|
+
* buffered read of the same document — but it is garbage, not live, so it is
|
|
203
|
+
* concurrent reads rather than one large read that the ceiling has to survive.
|
|
204
|
+
*/
|
|
205
|
+
async readWindow(response, params, ctx) {
|
|
206
|
+
const reader = response.body?.getReader();
|
|
207
|
+
const decoder = new TextDecoder('utf-8');
|
|
208
|
+
const extractor = createStreamingExtractor({
|
|
209
|
+
characterLimit: params.characterLimit,
|
|
210
|
+
characterOffset: params.characterOffset,
|
|
211
|
+
});
|
|
212
|
+
let bytes = 0;
|
|
213
|
+
if (reader) {
|
|
214
|
+
for (;;) {
|
|
215
|
+
const { done, value } = await this.readChunk(reader, ctx);
|
|
216
|
+
if (done)
|
|
217
|
+
break;
|
|
218
|
+
if (!value)
|
|
219
|
+
continue;
|
|
220
|
+
bytes += value.byteLength;
|
|
221
|
+
if (bytes > this.maxDocumentBytes) {
|
|
222
|
+
await reader.cancel().catch(() => undefined);
|
|
223
|
+
throw this.tooLarge(ctx, { bytesRead: bytes });
|
|
224
|
+
}
|
|
225
|
+
extractor.push(decoder.decode(value, { stream: true }));
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
if (bytes === 0) {
|
|
229
|
+
throw serviceUnavailable('Congress.gov returned an empty document body.', {
|
|
230
|
+
reason: DOCUMENT_FETCH_FAILED,
|
|
231
|
+
...ctx.recoveryFor(DOCUMENT_FETCH_FAILED),
|
|
232
|
+
retryable: false,
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
extractor.push(decoder.decode());
|
|
236
|
+
return extractor.finish();
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* A body that dies partway through is the transport failing, not a malformed
|
|
240
|
+
* document — the same classification a connection that never opened gets.
|
|
241
|
+
*/
|
|
242
|
+
async readChunk(reader, ctx) {
|
|
243
|
+
try {
|
|
244
|
+
return await reader.read();
|
|
245
|
+
}
|
|
246
|
+
catch (error) {
|
|
247
|
+
throw serviceUnavailable('The document body from Congress.gov ended before it was complete.', { reason: DOCUMENT_FETCH_FAILED, ...ctx.recoveryFor(DOCUMENT_FETCH_FAILED) }, { cause: error });
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
tooLarge(ctx, data) {
|
|
251
|
+
return invalidParams(`This document exceeds the ${this.maxDocumentBytes.toLocaleString('en-US')}-byte ceiling this server retrieves.`, {
|
|
252
|
+
reason: DOCUMENT_TOO_LARGE,
|
|
253
|
+
...ctx.recoveryFor(DOCUMENT_TOO_LARGE),
|
|
254
|
+
...data,
|
|
255
|
+
maxBytes: this.maxDocumentBytes,
|
|
256
|
+
retryable: false,
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
isRetryableError(error) {
|
|
260
|
+
if (error instanceof McpError) {
|
|
261
|
+
if (error.data?.retryable === false)
|
|
262
|
+
return false;
|
|
263
|
+
return (error.code === JsonRpcErrorCode.ServiceUnavailable ||
|
|
264
|
+
error.code === JsonRpcErrorCode.Timeout);
|
|
265
|
+
}
|
|
266
|
+
return true;
|
|
267
|
+
}
|
|
268
|
+
getRequestContext(ctx, operation) {
|
|
269
|
+
const ctxRecord = ctx;
|
|
270
|
+
const requestId = typeof ctxRecord.requestId === 'string' ? ctxRecord.requestId : 'congress-documents-service';
|
|
271
|
+
const timestamp = typeof ctxRecord.timestamp === 'string' ? ctxRecord.timestamp : new Date().toISOString();
|
|
272
|
+
return { operation, requestId, timestamp };
|
|
273
|
+
}
|
|
274
|
+
getAbortSignal(ctx) {
|
|
275
|
+
const signal = ctx.signal;
|
|
276
|
+
return isNativeAbortSignal(signal) ? signal : undefined;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
let _service;
|
|
280
|
+
export function initCongressDocuments() {
|
|
281
|
+
_service = new CongressDocumentsService();
|
|
282
|
+
}
|
|
283
|
+
export function getCongressDocuments() {
|
|
284
|
+
if (!_service) {
|
|
285
|
+
throw new Error('CongressDocumentsService not initialized — call initCongressDocuments() in setup()');
|
|
286
|
+
}
|
|
287
|
+
return _service;
|
|
288
|
+
}
|
|
289
|
+
//# sourceMappingURL=congress-documents-service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"congress-documents-service.js","sourceRoot":"","sources":["../../../src/services/congress-documents/congress-documents-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAGH,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,QAAQ,EACR,QAAQ,EACR,kBAAkB,GACnB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,8BAA8B,CAAC;AAC3E,OAAO,EAAE,wBAAwB,EAAwB,MAAM,0BAA0B,CAAC;AAG1F;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,UAAU,CAAC;AAE7C,0FAA0F;AAC1F,MAAM,CAAC,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAEzC,MAAM,YAAY,GAAG,CAAC,CAAC;AACvB,MAAM,eAAe,GAAG,GAAG,CAAC;AAE5B,iFAAiF;AACjF,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAEpD,sFAAsF;AACtF,MAAM,qBAAqB,GAAG,IAAI,GAAG,CAAC,CAAC,iBAAiB,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC,CAAC;AAElG;;;;;GAKG;AACH,MAAM,oBAAoB,GAAG,sBAAsB,CAAC;AACpD,MAAM,qBAAqB,GAAG,uBAAuB,CAAC;AACtD,MAAM,kBAAkB,GAAG,oBAAoB,CAAC;AAChD,MAAM,eAAe,GAAG,iBAAiB,CAAC;AAQ1C,SAAS,mBAAmB,CAAC,KAAc;IACzC,IACE,OAAO,WAAW,KAAK,UAAU;QACjC,OAAO,WAAW,CAAC,SAAS,CAAC,cAAc,KAAK,UAAU;QAC1D,CAAC,KAAK,EACN,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,CAAC;QACH,WAAW,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjD,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,CAAC,KAAK,YAAY,SAAS,CAAC,CAAC;IACvC,CAAC;AACH,CAAC;AAED,MAAM,OAAO,wBAAwB;IACN,gBAAgB;IAA7C,YAA6B,gBAAgB,GAAW,kBAAkB;gCAA7C,gBAAgB;IAAgC,CAAC;IAE9E;;;;;;;OAOG;IACH,KAAK,CAAC,aAAa,CAAC,MAA2B,EAAE,GAAY;QAC3D,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAExC,MAAM,SAAS,GAAG,uCAAuC,CAAC;QAC1D,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QAC9D,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QAExC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,GAAG,EAAE,MAAM,CAAC,EAAE;YACtF,SAAS;YACT,OAAO,EAAE,cAAc;YACvB,WAAW,EAAE,eAAe;YAC5B,UAAU,EAAE,YAAY,GAAG,CAAC;YAC5B,WAAW,EAAE,CAAC,KAAc,EAAE,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC;YAC7D,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC9B,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC,cAAc,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;IAClD,CAAC;IAED,iFAAiF;IACzE,cAAc,CACpB,MAAuB,EACvB,MAA2B,EAC3B,GAAY;QAEZ,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,MAAM,CAAC;QACzC,MAAM,MAAM,GAAG,MAAM,CAAC,eAAe,CAAC;QAEtC,sFAAsF;QACtF,IAAI,eAAe,GAAG,CAAC,IAAI,MAAM,IAAI,eAAe,EAAE,CAAC;YACrD,MAAM,aAAa,CACjB,mBAAmB,MAAM,gDAAgD,eAAe,cAAc,EACtG;gBACE,MAAM,EAAE,eAAe;gBACvB,GAAG,GAAG,CAAC,WAAW,CAAC,eAAe,CAAC;gBACnC,eAAe;gBACf,MAAM;gBACN,SAAS,EAAE,KAAK;aACjB,CACF,CAAC;QACJ,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QACjC,MAAM,SAAS,GAAG,GAAG,GAAG,eAAe,CAAC;QACxC,OAAO;YACL,IAAI;YACJ,eAAe;YACf,MAAM;YACN,SAAS;YACT,UAAU,EAAE,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI;SACnC,CAAC;IACJ,CAAC;IAEO,iBAAiB,CAAC,GAAW,EAAE,GAAY;QACjD,IAAI,IAAwB,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO;gBAAE,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC;QAC1F,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,GAAG,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;YACjE,MAAM,QAAQ,CACZ,gIAAgI,EAChI;gBACE,MAAM,EAAE,oBAAoB;gBAC5B,GAAG,GAAG,CAAC,WAAW,CAAC,oBAAoB,CAAC;gBACxC,SAAS,EAAE,KAAK;aACjB,CACF,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,OAAO,CACnB,MAA2B,EAC3B,cAAkC,EAClC,GAAY,EACZ,MAAoB;QAEpB,IAAI,QAAkB,CAAC;QACvB,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,GAAG,EAAE,kBAAkB,EAAE,cAAc,EAAE;gBAChF,OAAO,EAAE,EAAE,MAAM,EAAE,6CAA6C,EAAE;gBAClE,gBAAgB,EAAE,CAAC,GAAG,CAAC;gBACvB,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aAC9B,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC5C,CAAC;QAED,IAAI,CAAC,iBAAiB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACtC,IAAI,CAAC,oBAAoB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACzC,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;IAChD,CAAC;IAED;;;;OAIG;IACK,kBAAkB,CAAC,KAAc,EAAE,GAAY;QACrD,IAAI,CAAC,CAAC,KAAK,YAAY,QAAQ,CAAC;YAAE,OAAO,KAAK,CAAC;QAC/C,IAAI,KAAK,CAAC,IAAI,KAAK,gBAAgB,CAAC,QAAQ,EAAE,CAAC;YAC7C,OAAO,QAAQ,CACb,kEAAkE,EAClE;gBACE,MAAM,EAAE,oBAAoB;gBAC5B,GAAG,GAAG,CAAC,WAAW,CAAC,oBAAoB,CAAC;gBACxC,SAAS,EAAE,KAAK;aACjB,EACD,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;QACJ,CAAC;QACD,mFAAmF;QACnF,OAAO,kBAAkB,CACvB,oDAAoD,EACpD,EAAE,MAAM,EAAE,qBAAqB,EAAE,GAAG,GAAG,CAAC,WAAW,CAAC,qBAAqB,CAAC,EAAE,EAC5E,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;IACJ,CAAC;IAEO,iBAAiB,CAAC,QAAkB,EAAE,GAAY;QACxD,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;QACvD,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;QAChE,IAAI,qBAAqB,CAAC,GAAG,CAAC,SAAS,CAAC;YAAE,OAAO;QACjD,MAAM,kBAAkB,CACtB,uCAAuC,SAAS,IAAI,SAAS,2CAA2C,EACxG;YACE,MAAM,EAAE,qBAAqB;YAC7B,GAAG,GAAG,CAAC,WAAW,CAAC,qBAAqB,CAAC;YACzC,WAAW,EAAE,SAAS;YACtB,kFAAkF;YAClF,SAAS,EAAE,KAAK;SACjB,CACF,CAAC;IACJ,CAAC;IAED,0FAA0F;IAClF,oBAAoB,CAAC,QAAkB,EAAE,GAAY;QAC3D,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,MAAM,KAAK,IAAI;YAAE,OAAO;QAC5B,MAAM,aAAa,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,aAAa,IAAI,IAAI,CAAC,gBAAgB;YAAE,OAAO;QACtF,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED;;;;;;;;;;OAUG;IACK,KAAK,CAAC,UAAU,CACtB,QAAkB,EAClB,MAA2B,EAC3B,GAAY;QAEZ,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;QAC1C,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,CAAC,CAAC;QACzC,MAAM,SAAS,GAAG,wBAAwB,CAAC;YACzC,cAAc,EAAE,MAAM,CAAC,cAAc;YACrC,eAAe,EAAE,MAAM,CAAC,eAAe;SACxC,CAAC,CAAC;QACH,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,IAAI,MAAM,EAAE,CAAC;YACX,SAAS,CAAC;gBACR,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAC1D,IAAI,IAAI;oBAAE,MAAM;gBAChB,IAAI,CAAC,KAAK;oBAAE,SAAS;gBACrB,KAAK,IAAI,KAAK,CAAC,UAAU,CAAC;gBAC1B,IAAI,KAAK,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBAClC,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;oBAC7C,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;gBACjD,CAAC;gBACD,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC1D,CAAC;QACH,CAAC;QAED,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;YAChB,MAAM,kBAAkB,CAAC,+CAA+C,EAAE;gBACxE,MAAM,EAAE,qBAAqB;gBAC7B,GAAG,GAAG,CAAC,WAAW,CAAC,qBAAqB,CAAC;gBACzC,SAAS,EAAE,KAAK;aACjB,CAAC,CAAC;QACL,CAAC;QAED,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACjC,OAAO,SAAS,CAAC,MAAM,EAAE,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,SAAS,CACrB,MAA+C,EAC/C,GAAY;QAEZ,IAAI,CAAC;YACH,OAAO,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,kBAAkB,CACtB,mEAAmE,EACnE,EAAE,MAAM,EAAE,qBAAqB,EAAE,GAAG,GAAG,CAAC,WAAW,CAAC,qBAAqB,CAAC,EAAE,EAC5E,EAAE,KAAK,EAAE,KAAK,EAAE,CACjB,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,GAAY,EAAE,IAA6B;QAC1D,OAAO,aAAa,CAClB,6BAA6B,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,OAAO,CAAC,sCAAsC,EAChH;YACE,MAAM,EAAE,kBAAkB;YAC1B,GAAG,GAAG,CAAC,WAAW,CAAC,kBAAkB,CAAC;YACtC,GAAG,IAAI;YACP,QAAQ,EAAE,IAAI,CAAC,gBAAgB;YAC/B,SAAS,EAAE,KAAK;SACjB,CACF,CAAC;IACJ,CAAC;IAEO,gBAAgB,CAAC,KAAc;QACrC,IAAI,KAAK,YAAY,QAAQ,EAAE,CAAC;YAC9B,IAAI,KAAK,CAAC,IAAI,EAAE,SAAS,KAAK,KAAK;gBAAE,OAAO,KAAK,CAAC;YAClD,OAAO,CACL,KAAK,CAAC,IAAI,KAAK,gBAAgB,CAAC,kBAAkB;gBAClD,KAAK,CAAC,IAAI,KAAK,gBAAgB,CAAC,OAAO,CACxC,CAAC;QACJ,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,iBAAiB,CAAC,GAAY,EAAE,SAAiB;QACvD,MAAM,SAAS,GAAG,GAAyC,CAAC;QAC5D,MAAM,SAAS,GACb,OAAO,SAAS,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,4BAA4B,CAAC;QAC/F,MAAM,SAAS,GACb,OAAO,SAAS,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3F,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC;IAC7C,CAAC;IAEO,cAAc,CAAC,GAAY;QACjC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;QAC1B,OAAO,mBAAmB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1D,CAAC;CACF;AAED,IAAI,QAA8C,CAAC;AAEnD,MAAM,UAAU,qBAAqB;IACnC,QAAQ,GAAG,IAAI,wBAAwB,EAAE,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,oBAAoB;IAClC,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CACb,oFAAoF,CACrF,CAAC;IACJ,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Resolve a requested document format to its upstream URL.
|
|
3
|
+
* @module services/congress-documents/document-formats
|
|
4
|
+
*/
|
|
5
|
+
import type { DocumentFormat } from './types.js';
|
|
6
|
+
export declare const DOCUMENT_FORMATS: readonly ['text', 'xml'];
|
|
7
|
+
/**
|
|
8
|
+
* Upstream `type` labels for each supported format, in preference order.
|
|
9
|
+
*
|
|
10
|
+
* `Formatted Text` is GPO's `<pre>`-wrapped print output and is the only format
|
|
11
|
+
* present on every document checked — bills back to 1999, committee reports, and
|
|
12
|
+
* Daily Record articles. XML is inconsistent: `United States Legislative Markup`
|
|
13
|
+
* ships on some newer bill versions, the legacy `Formatted XML` on many more, and
|
|
14
|
+
* neither on committee reports or the Daily Record. PDF is not offered.
|
|
15
|
+
*/
|
|
16
|
+
export declare const DOCUMENT_FORMAT_LABELS: Record<DocumentFormat, readonly string[]>;
|
|
17
|
+
/**
|
|
18
|
+
* Pick the URL for `format` from an upstream format list.
|
|
19
|
+
*
|
|
20
|
+
* Labels are tried in preference order, and within one label a non-errata entry
|
|
21
|
+
* wins over an errata reprint — committee reports publish both under the same
|
|
22
|
+
* `Formatted Text` label and the errata is the correction, not the report.
|
|
23
|
+
* Returns `undefined` when the document publishes nothing in this format, which
|
|
24
|
+
* callers surface as the `format_unavailable` contract failure.
|
|
25
|
+
*/
|
|
26
|
+
export declare function selectDocumentUrl(entries: unknown, format: DocumentFormat): string | undefined;
|
|
27
|
+
/** The upstream labels a format resolves to, for error messages that name what was searched. */
|
|
28
|
+
export declare function describeFormat(format: DocumentFormat): string;
|
|
29
|
+
//# sourceMappingURL=document-formats.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"document-formats.d.ts","sourceRoot":"","sources":["../../../src/services/congress-documents/document-formats.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAqB,MAAM,YAAY,CAAC;AAEpE,eAAO,MAAM,gBAAgB,YAAI,MAAM,EAAE,KAAK,CAAU,CAAC;AAEzD;;;;;;;;GAQG;AACH,eAAO,MAAM,sBAAsB,EAAE,MAAM,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAG5E,CAAC;AAUF;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,cAAc,GAAG,MAAM,GAAG,SAAS,CAa9F;AAED,gGAAgG;AAChG,wBAAgB,cAAc,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CAE7D"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Resolve a requested document format to its upstream URL.
|
|
3
|
+
* @module services/congress-documents/document-formats
|
|
4
|
+
*/
|
|
5
|
+
export const DOCUMENT_FORMATS = ['text', 'xml'];
|
|
6
|
+
/**
|
|
7
|
+
* Upstream `type` labels for each supported format, in preference order.
|
|
8
|
+
*
|
|
9
|
+
* `Formatted Text` is GPO's `<pre>`-wrapped print output and is the only format
|
|
10
|
+
* present on every document checked — bills back to 1999, committee reports, and
|
|
11
|
+
* Daily Record articles. XML is inconsistent: `United States Legislative Markup`
|
|
12
|
+
* ships on some newer bill versions, the legacy `Formatted XML` on many more, and
|
|
13
|
+
* neither on committee reports or the Daily Record. PDF is not offered.
|
|
14
|
+
*/
|
|
15
|
+
export const DOCUMENT_FORMAT_LABELS = {
|
|
16
|
+
text: ['Formatted Text'],
|
|
17
|
+
xml: ['United States Legislative Markup', 'Formatted XML'],
|
|
18
|
+
};
|
|
19
|
+
function refUrl(ref) {
|
|
20
|
+
return typeof ref.url === 'string' && ref.url.trim() !== '' ? ref.url.trim() : undefined;
|
|
21
|
+
}
|
|
22
|
+
function matchesLabel(ref, label) {
|
|
23
|
+
return typeof ref.type === 'string' && ref.type.trim().toLowerCase() === label.toLowerCase();
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Pick the URL for `format` from an upstream format list.
|
|
27
|
+
*
|
|
28
|
+
* Labels are tried in preference order, and within one label a non-errata entry
|
|
29
|
+
* wins over an errata reprint — committee reports publish both under the same
|
|
30
|
+
* `Formatted Text` label and the errata is the correction, not the report.
|
|
31
|
+
* Returns `undefined` when the document publishes nothing in this format, which
|
|
32
|
+
* callers surface as the `format_unavailable` contract failure.
|
|
33
|
+
*/
|
|
34
|
+
export function selectDocumentUrl(entries, format) {
|
|
35
|
+
if (!Array.isArray(entries))
|
|
36
|
+
return;
|
|
37
|
+
const refs = entries.filter((entry) => typeof entry === 'object' && entry !== null);
|
|
38
|
+
for (const label of DOCUMENT_FORMAT_LABELS[format]) {
|
|
39
|
+
const matched = refs.filter((ref) => matchesLabel(ref, label) && refUrl(ref) !== undefined);
|
|
40
|
+
const preferred = matched.find((ref) => ref.isErrata !== 'Y') ?? matched[0];
|
|
41
|
+
const url = preferred ? refUrl(preferred) : undefined;
|
|
42
|
+
if (url)
|
|
43
|
+
return url;
|
|
44
|
+
}
|
|
45
|
+
return;
|
|
46
|
+
}
|
|
47
|
+
/** The upstream labels a format resolves to, for error messages that name what was searched. */
|
|
48
|
+
export function describeFormat(format) {
|
|
49
|
+
return DOCUMENT_FORMAT_LABELS[format].join(' / ');
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=document-formats.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"document-formats.js","sourceRoot":"","sources":["../../../src/services/congress-documents/document-formats.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,MAAM,EAAE,KAAK,CAAU,CAAC;AAEzD;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAA8C;IAC/E,IAAI,EAAE,CAAC,gBAAgB,CAAC;IACxB,GAAG,EAAE,CAAC,kCAAkC,EAAE,eAAe,CAAC;CAC3D,CAAC;AAEF,SAAS,MAAM,CAAC,GAAsB;IACpC,OAAO,OAAO,GAAG,CAAC,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AAC3F,CAAC;AAED,SAAS,YAAY,CAAC,GAAsB,EAAE,KAAa;IACzD,OAAO,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,KAAK,CAAC,WAAW,EAAE,CAAC;AAC/F,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB,CAAC,OAAgB,EAAE,MAAsB;IACxE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO;IACpC,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CACzB,CAAC,KAAK,EAA8B,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CACnF,CAAC;IAEF,KAAK,MAAM,KAAK,IAAI,sBAAsB,CAAC,MAAM,CAAC,EAAE,CAAC;QACnD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC;QAC5F,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QAC5E,MAAM,GAAG,GAAG,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACtD,IAAI,GAAG;YAAE,OAAO,GAAG,CAAC;IACtB,CAAC;IACD,OAAO;AACT,CAAC;AAED,gGAAgG;AAChG,MAAM,UAAU,cAAc,CAAC,MAAsB;IACnD,OAAO,sBAAsB,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACpD,CAAC"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Incremental form of {@link extractDocumentText} — consumes a
|
|
3
|
+
* document body chunk by chunk and retains only the requested character window.
|
|
4
|
+
*
|
|
5
|
+
* `extract-text.ts` is the normative definition of the extraction: every
|
|
6
|
+
* character offset in the `content` contract indexes into the string it
|
|
7
|
+
* produces. This module reproduces that string exactly while holding constant
|
|
8
|
+
* memory, so a document is bounded by how long it takes to stream rather than by
|
|
9
|
+
* how much of it fits in a buffer. The two are pinned against each other by a
|
|
10
|
+
* differential test — if this file drifts, that test fails, not production.
|
|
11
|
+
*
|
|
12
|
+
* The whole-string pipeline is `normalize newlines → strip comments → unwrap
|
|
13
|
+
* <pre> → strip tags → decode entities → trim`, and three of those stages are
|
|
14
|
+
* decided by content that arrives arbitrarily late:
|
|
15
|
+
*
|
|
16
|
+
* - **Is there a `<pre>` at all?** Only a body with none is read whole. Both
|
|
17
|
+
* readings run at once, in two sinks; the losing one is dropped the moment a
|
|
18
|
+
* `<pre>` open tag settles the question.
|
|
19
|
+
* - **Where is the last `</pre>`?** Every close is a candidate, so each one
|
|
20
|
+
* records a mark on the `<pre>` sink and the last mark standing is restored at
|
|
21
|
+
* end of stream.
|
|
22
|
+
* - **Does this `<!--` (or this `<`) ever close?** An unterminated one is not a
|
|
23
|
+
* comment or a tag at all — the regexes leave it in the text. Its content is
|
|
24
|
+
* therefore written through as literal text and rolled back if the closing
|
|
25
|
+
* delimiter turns up.
|
|
26
|
+
*
|
|
27
|
+
* Every one of those rewinds is a mark: a handful of counters plus a length to
|
|
28
|
+
* truncate the captured window to. Nothing rewindable grows with the document.
|
|
29
|
+
*
|
|
30
|
+
* @module services/congress-documents/extract-text-stream
|
|
31
|
+
*/
|
|
32
|
+
/** The character window to retain while streaming. */
|
|
33
|
+
export interface TextWindowRequest {
|
|
34
|
+
/** Maximum characters to keep, counted from `characterOffset`. */
|
|
35
|
+
characterLimit: number;
|
|
36
|
+
/** First character of the extracted text to keep, 0-based. */
|
|
37
|
+
characterOffset: number;
|
|
38
|
+
}
|
|
39
|
+
/** The retained window plus the length of the whole extracted text. */
|
|
40
|
+
export interface ExtractedWindow {
|
|
41
|
+
/** The requested window of the extracted text. */
|
|
42
|
+
text: string;
|
|
43
|
+
/** Characters in the whole extracted document, not this window. */
|
|
44
|
+
totalCharacters: number;
|
|
45
|
+
}
|
|
46
|
+
/** Consumes a document body in chunks and yields one character window. */
|
|
47
|
+
export interface StreamingExtractor {
|
|
48
|
+
/** Finish the document and return its window. Call once, after the last push. */
|
|
49
|
+
finish(): ExtractedWindow;
|
|
50
|
+
/** Feed the next decoded chunk of the body. */
|
|
51
|
+
push(chunk: string): void;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Create an extractor that reproduces {@link extractDocumentText} over a chunked
|
|
55
|
+
* body while retaining only `characterLimit` characters of it.
|
|
56
|
+
*/
|
|
57
|
+
export declare function createStreamingExtractor(request: TextWindowRequest): StreamingExtractor;
|
|
58
|
+
//# sourceMappingURL=extract-text-stream.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extract-text-stream.d.ts","sourceRoot":"","sources":["../../../src/services/congress-documents/extract-text-stream.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAIH,sDAAsD;AACtD,MAAM,WAAW,iBAAiB;IAChC,kEAAkE;IAClE,cAAc,EAAE,MAAM,CAAC;IACvB,8DAA8D;IAC9D,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,uEAAuE;AACvE,MAAM,WAAW,eAAe;IAC9B,kDAAkD;IAClD,IAAI,EAAE,MAAM,CAAC;IACb,mEAAmE;IACnE,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,0EAA0E;AAC1E,MAAM,WAAW,kBAAkB;IACjC,iFAAiF;IACjF,MAAM,IAAI,eAAe,CAAC;IAC1B,+CAA+C;IAC/C,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAuqBD;;;GAGG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,iBAAiB,GAAG,kBAAkB,CAEvF"}
|