@pipeworx/mcp-guam-code 0.0.0-stage → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts ADDED
@@ -0,0 +1,2023 @@
1
+ interface McpToolDefinition {
2
+ name: string;
3
+ description: string;
4
+ /** Human-facing one-liner (fleet #1967). Optional; consumers fall back to
5
+ * description. Kept in step with shared/src/types.ts — scripts/lib/
6
+ * check-inlined-types.mjs reports drift at publish time. */
7
+ summary?: string;
8
+ inputSchema: {
9
+ type: 'object';
10
+ properties: Record<string, unknown>;
11
+ required?: string[];
12
+ anyOf?: Array<{ required: string[] }>;
13
+ oneOf?: Array<{ required: string[] }>;
14
+ allOf?: Array<{ required: string[] }>;
15
+ };
16
+ outputSchema?: Record<string, unknown>;
17
+ }
18
+
19
+ interface McpToolExport {
20
+ tools: McpToolDefinition[];
21
+ callTool: (name: string, args: Record<string, unknown>) => Promise<unknown>;
22
+ meter?: { credits: number };
23
+ cost?: Record<string, unknown>;
24
+ provider?: string;
25
+ }
26
+
27
+ /**
28
+ * Was this failure OUR OWN web service? — the other half of `internal-db-class.ts`.
29
+ *
30
+ * fleet #1089 pulled failures from our own Postgres out of `upstream_down` by
31
+ * keying on the SQLSTATE inside PostgREST's four-key error envelope. That
32
+ * covered the majority and structurally could not cover the rest: the rest
33
+ * never reach Postgres, so they carry no SQLSTATE. What was left, measured over
34
+ * the 24h to 2026-09-02T15:00Z (fleet #1096):
35
+ *
36
+ * 5 pipeworx-catalog get_pack_tools Pipeworx catalog error: 522 — error code: 522
37
+ * 3 fleet fleet_list_open … upstream_down: Fleet task queue did not respond within 25s
38
+ *
39
+ * 521/522/523/526 are Cloudflare saying its edge could not reach an ORIGIN, and
40
+ * in both of those rows the origin is ours — `gateway.pipeworx.io` for the
41
+ * catalog pack (it self-fetches when the gateway hasn't injected a manifest),
42
+ * our own Supabase for fleet. There is no third party anywhere in either call.
43
+ * Same defect as #1089: our own outage filed under `upstream_down`, the one
44
+ * class that means "the source is unreachable and there is nothing for us to
45
+ * fix", which is why the problem-tools triage skips it.
46
+ *
47
+ * WHY NOT A WORDING RULE. The obvious fix is to match `fleet db error:` and
48
+ * `Pipeworx catalog error:` in classifyToolError. Each is emitted from exactly
49
+ * one site today, so it would work today. It would also rot the first time
50
+ * somebody rewords a label — silently, and in the direction of hiding our own
51
+ * outage, which is worse than the bug being fixed. Every prose rule in
52
+ * error-class.ts has needed widening as packs invented new wording (#409/#450/
53
+ * #584); that history is most of that file's comment budget.
54
+ *
55
+ * WHAT THIS KEYS ON INSTEAD: **the host the call actually reached.** A URL's
56
+ * hostname is a fact about the call, not a guess about its prose. Two
57
+ * consequences that a pack-level flag could not give us, and the reason the
58
+ * flag was rejected:
59
+ *
60
+ * - It describes the CALL, not the pack. `govcon-intel` fans out to our own
61
+ * Supabase AND to genuine third parties; `court-listener` holds our cache
62
+ * in Supabase and fetches courtlistener.com. An `internallyHosted: true` on
63
+ * either pack would relabel a real third-party outage as ours — inventing
64
+ * work, which is the same class of error in the opposite direction.
65
+ * - It covers every future internal pack for free, instead of one declared
66
+ * slug at a time.
67
+ *
68
+ * WHY IT SURVIVES A REWORD. The marker below is not matched as a literal by two
69
+ * separate files. `markInternalOrigin()` writes it and `internalHostMetricsClass()`
70
+ * reads it, both from the single exported `INTERNAL_ORIGIN_MARKER` constant in
71
+ * this module — so changing the wording changes both sides in the same edit and
72
+ * cannot desynchronise them. The pack's own label (`fleet db error:`,
73
+ * `Pipeworx catalog error:`) is not read at all: reword it freely, the class is
74
+ * unaffected. That is the property `stripClassPrefix` lacked when it drifted
75
+ * from its own classifier three times and needed a CI gate to hold them
76
+ * together.
77
+ *
78
+ * WHERE THE 5xx TEST LIVES. `markInternalOrigin` is called from the places that
79
+ * hold the real `Response` — `httpError`/`httpErrorMessage` and the timeout
80
+ * branch of `fetchWithTimeout` in `shared/src/http.ts` — so "is this an
81
+ * availability failure" is decided from the actual status code, never re-derived
82
+ * by scraping a number out of a sentence. A 404 from our own registry for a slug
83
+ * that does not exist is a caller's bad argument and is deliberately NOT marked.
84
+ */
85
+
86
+ /**
87
+ * OUR OWN web service was unreachable — not an upstream, and never `upstream_down`.
88
+ *
89
+ * ONE value, not three, unlike `internal_db_*`. That split existed because a
90
+ * slow query, an exhausted pool and an unknown SQLSTATE have different owners
91
+ * and different fixes. Here there is only one story to tell — an origin we run
92
+ * did not answer the edge — and one owner. A bucket with no distinct owner per
93
+ * value is decoration; #724 is what happens when a class holds several
94
+ * situations, and inventing sub-values ahead of a reason to act on them
95
+ * differently is the same mistake with the sign flipped.
96
+ *
97
+ * METRICS ONLY, exactly like PLATFORM_KEY_ERROR_CLASS and the internal_db
98
+ * values. `classifyToolError` still answers `upstream_down` for the retry and
99
+ * hint paths, which only care whether retrying or a sibling tool might work —
100
+ * and it might. Nothing a caller sees or is charged changes here.
101
+ *
102
+ * READ SIDE: this value is in BROKEN_TOOL_CLASSES, FAULT_CLASSES and
103
+ * ALL_ERROR_CLASSES in `workers/registry-api/src/index.ts`. All three, or it
104
+ * lands on no dashboard — fleet #721 is the warning, where the #719 split
105
+ * worked on the write side and was invisible for weeks.
106
+ */
107
+ const INTERNAL_SERVICE_UNREACHABLE_CLASS = 'internal_service_unreachable';
108
+
109
+ /**
110
+ * The token that carries "this origin is ours" from the call site to the
111
+ * classifier.
112
+ *
113
+ * Appended to the error message rather than attached to the Error object,
114
+ * because the object does not survive the trip: 275 packs return `{ error:
115
+ * string }` instead of throwing, the gateway reads `observedError` as a string,
116
+ * and the fleet pack rebuilds its error from a captured status + body across a
117
+ * retry loop. A property on an Error would be dropped by every one of those
118
+ * paths and the class would work in tests and vanish in production.
119
+ *
120
+ * WORDING IS LOAD-BEARING, same rule as labelAge's note in authority.ts. This
121
+ * string is appended to a pack's thrown Error message (shared/src/http.ts),
122
+ * and a thrown Error's message is exactly what the gateway hands back to the
123
+ * caller as `content[0].text` when nothing rewrites it (workers/gateway/src
124
+ * catches the throw and sets `rawResult.message = stripClassPrefix(error)`,
125
+ * which does not touch this suffix) — so the original wording,
126
+ * " [pipeworx-hosted origin — our own service, not a third party]", was not a
127
+ * theoretical leak: it shipped live on pipeworx-catalog's 522s, 7 times in 6
128
+ * hours on 2026-09-02 (see tests/golden-internal-service.test.ts), verbatim
129
+ * naming Pipeworx as the host. check:hosting-claims never caught it because it
130
+ * did not scan shared/ at all (task #2009). Reworded to describe the
131
+ * OBSERVATION (the origin did not answer) without a claim about who runs it —
132
+ * the identical fix labelAge got: drop the possessive, keep the fact.
133
+ */
134
+ const INTERNAL_ORIGIN_MARKER = ' [origin did not respond — retry before concluding the named source is down]';
135
+
136
+ /**
137
+ * Supabase's data plane for a project is `<ref>.supabase.co`, where the ref is
138
+ * exactly twenty lowercase letters (ours is `pqauisounztsgdgfkhke`).
139
+ *
140
+ * Matching the shape rather than listing the ref keeps this correct when we add
141
+ * a project — `supabaseEnv` on a pack entry already points some packs at a
142
+ * second one — while still excluding `status.supabase.co`, which is Supabase's
143
+ * own status page and emphatically not our database. Verified 2026-09-02 by
144
+ * `grep -rhoE '[a-z0-9-]+\.supabase\.(co|in)' mcps shared workers scripts`: the
145
+ * only real project ref anywhere in the tree is ours, the rest are doc
146
+ * placeholders (`abc`, `xyz`, `example`) which this pattern also excludes. Same
147
+ * finding internal-db-class.ts relies on for the PostgREST envelope being ours
148
+ * by construction.
149
+ */
150
+ const SUPABASE_PROJECT_HOST = /^[a-z]{20}\.supabase\.(co|in)$/;
151
+
152
+ /**
153
+ * Is this a host WE run?
154
+ *
155
+ * Deliberately NOT including `*.workers.dev`: plenty of third-party APIs are
156
+ * hosted on workers.dev, so the suffix says where something runs and not who
157
+ * owns it. Every internal call we actually make goes to a `pipeworx.io`
158
+ * hostname or to our Supabase project, both of which are ownership facts.
159
+ *
160
+ * `workers/gateway/src/provenance.ts`'s `OUR_HOSTS` answers the same
161
+ * question and DOES include `workers.dev` — a documented divergence
162
+ * (task #2051), not a bug to converge. That list decides what a response may
163
+ * cite as a data SOURCE, where a false negative (citing our own worker as an
164
+ * external source) is the hosting-disclosure leak this whole file exists to
165
+ * prevent, so it errs broad. This one decides who gets BLAMED for a 5xx in
166
+ * outage metrics read by on-call, where a false positive (crediting our own
167
+ * infra with a third party's outage) hides the real failure, so it errs
168
+ * narrow. Same suffix, opposite direction, because they are never called for
169
+ * the same reason.
170
+ *
171
+ * Returns false on anything unparseable rather than throwing — this runs inside
172
+ * an error path, and an error path that can itself throw turns a diagnosable
173
+ * failure into a mystery.
174
+ */
175
+ function isPipeworxOrigin(url: string | URL | undefined | null): boolean {
176
+ if (!url) return false;
177
+ let host: string;
178
+ try {
179
+ host = new URL(url instanceof URL ? url.href : url).hostname.toLowerCase();
180
+ } catch {
181
+ return false;
182
+ }
183
+ if (host === 'pipeworx.io' || host.endsWith('.pipeworx.io')) return true;
184
+ return SUPABASE_PROJECT_HOST.test(host);
185
+ }
186
+
187
+ /**
188
+ * Append the marker when this failure was OUR origin failing to answer.
189
+ *
190
+ * `status` is the HTTP status when there is one, and omitted for a timeout —
191
+ * where there is no response at all, and "the origin did not answer" is the
192
+ * whole observation. Statuses below 500 are left alone: a 404 from our own
193
+ * registry for a slug that does not exist is the caller's argument, not our
194
+ * outage, and marking it would put ordinary 404s on the incident dashboard.
195
+ *
196
+ * Idempotent, so a message that is wrapped and re-marked on the way up (the
197
+ * fleet pack's retry loop re-throws through two layers) carries the marker once.
198
+ */
199
+ function markInternalOrigin(
200
+ message: string,
201
+ url: string | URL | undefined | null,
202
+ status?: number,
203
+ ): string {
204
+ if (status !== undefined && status < 500) return message;
205
+ if (!isPipeworxOrigin(url)) return message;
206
+ if (message.includes(INTERNAL_ORIGIN_MARKER)) return message;
207
+ return message + INTERNAL_ORIGIN_MARKER;
208
+ }
209
+
210
+ /**
211
+ * Which blob4 value a failure from our own web services books as, or undefined
212
+ * if this is not one.
213
+ *
214
+ * Ordered AFTER `internalDbMetricsClass` at the call site: a PostgREST envelope
215
+ * from our own Supabase is a strictly more specific statement about the same
216
+ * row (which of our services, and why), and the two cannot disagree about
217
+ * whether the failure is ours.
218
+ */
219
+ function internalHostMetricsClass(error: string): string | undefined {
220
+ return error.includes(INTERNAL_ORIGIN_MARKER) ? INTERNAL_SERVICE_UNREACHABLE_CLASS : undefined;
221
+ }
222
+
223
+
224
+ /**
225
+ * One place to turn a failed `fetch` into an error a caller can act on.
226
+ *
227
+ * Nearly every pack was written the same way:
228
+ *
229
+ * if (!res.ok) throw new Error(`Unsplash: ${res.status}`);
230
+ *
231
+ * which discards the response body — and the body is usually where the upstream
232
+ * says what was actually wrong ("**symbol** not found: GBP", "parameter `year`
233
+ * out of range", "unknown taxonomy id"). The caller gets a number, cannot
234
+ * self-correct, and retries the same broken call. A 2026-07-31 sweep found this
235
+ * shape in 481 of 1,400 packs, 47 of them PLATFORM-keyed.
236
+ *
237
+ * It also hides bugs one level down. Two of the first three packs audited had a
238
+ * second defect that only existed because of this line: unsplash's rate-limit
239
+ * branch sat BELOW a catch-all and was unreachable, and bea-gov parsed
240
+ * `BEAAPI.Error.APIErrorDescription` below a `!res.ok` throw that made the
241
+ * parsing dead code for every non-200.
242
+ *
243
+ * DELIBERATELY NOT A CLASSIFIER. It does not add `user_error:` /
244
+ * `upstream_down:` prefixes. Those decide which tier a failure lands in, and the
245
+ * `error` tier is what the daily problem-tools list is built from — it means
246
+ * "Pipeworx has a defect". A 400 is genuinely ambiguous: often a caller's bad
247
+ * argument, but sometimes a query WE built wrong (ted-eu comma-joined its CPV
248
+ * values into something TED rejected, and that bug was found only because it sat
249
+ * in `error`). Blanket-classifying 400s as caller mistakes would have hidden it.
250
+ * A pack that KNOWS which it is should keep saying so explicitly; this helper is
251
+ * for the 481 that say nothing at all.
252
+ */
253
+
254
+ /** Longest upstream explanation we'll pass through. Enough for a real message,
255
+ * short enough that an HTML page or a stack trace can't swamp the error. */
256
+
257
+ const MAX_DETAIL = 300;
258
+
259
+ /**
260
+ * Default bound for `fetchWithTimeout` when a pack doesn't state its own.
261
+ *
262
+ * 25s mirrors the number `epo-ops` landed on after measuring the real failure:
263
+ * a degraded upstream that doesn't error, it just never answers, and a Worker
264
+ * sits in `await fetch()` until ITS OWN execution budget kills the request —
265
+ * which can take minutes, not seconds (epo_ops_search_patents measured 4-8
266
+ * MINUTE hangs before this existed). 25s is short enough that a caller gets a
267
+ * fast, actionable error instead of holding the connection, and long enough
268
+ * that it doesn't false-trip on a merely-slow-but-alive upstream.
269
+ */
270
+ const DEFAULT_FETCH_TIMEOUT_MS = 25_000;
271
+
272
+ /**
273
+ * Read the body of a failed response and fold it into a throwable Error.
274
+ *
275
+ * Usage — note the `await`, which is the one thing that makes this a mechanical
276
+ * change rather than a drop-in:
277
+ *
278
+ * if (!res.ok) throw await httpError(res, 'Unsplash');
279
+ *
280
+ * Safe to call on any non-ok response: a body that is missing, empty, unreadable
281
+ * or HTML degrades to exactly the old `Name: 404` string rather than throwing
282
+ * something new from inside the error path.
283
+ */
284
+ async function httpError(res: Response, name: string): Promise<Error> {
285
+ return new Error(await httpErrorMessage(res, name));
286
+ }
287
+
288
+ /** The message text without constructing an Error — for packs that need to wrap
289
+ * it in their own envelope or add an explicit classification prefix. */
290
+ async function httpErrorMessage(res: Response, name: string): Promise<string> {
291
+ // The one place a 5xx from a host WE run gets stamped as ours. `res.url` is
292
+ // the URL the fetch actually resolved to (after redirects), so this is a fact
293
+ // about the call rather than a guess from the `name` the pack passed in —
294
+ // reword that label freely, the class does not move. See
295
+ // internal-host-class.ts; no-op for every third-party upstream, which is why
296
+ // this touches 481 packs' error text and changes none of it.
297
+ return markInternalOrigin(
298
+ `${name}: ${res.status}${detailSuffix(await readDetail(res))}`,
299
+ res.url,
300
+ res.status,
301
+ );
302
+ }
303
+
304
+ /**
305
+ * Just the upstream's own explanation — no name, no status.
306
+ *
307
+ * For a pack that has already said both in its own sentence. epo-ops reads
308
+ * `EPO rejected this search as too large (HTTP 413) — ${httpErrorMessage(…)}`,
309
+ * which rendered as `… (HTTP 413) — EPO: 413.` once the XML detail was being
310
+ * dropped: the upstream named twice, the status twice, and the one thing EPO
311
+ * actually said ("Not enough characters before truncation character") nowhere
312
+ * (fleet #712). Returns '' when the body carries nothing readable, so a caller
313
+ * can fall back to its own wording.
314
+ */
315
+ async function upstreamDetail(res: Response): Promise<string> {
316
+ return readDetail(res);
317
+ }
318
+
319
+ /**
320
+ * Read a SUCCESSFUL response as JSON, failing loudly when it isn't JSON.
321
+ *
322
+ * `httpError` above only ever runs on `!res.ok`, which leaves the nastier half
323
+ * of the problem unhandled: an upstream that answers **HTTP 200 with an HTML
324
+ * page**. A bot wall, a login redirect, a maintenance interstitial and a CDN
325
+ * error page are all 200s, so `res.ok` is true, and `res.json()` then throws
326
+ * `Unexpected token '<', "<!DOCTYPE "... is not valid JSON`.
327
+ *
328
+ * That string is the problem. It names no upstream, carries no status, and
329
+ * reads like a parser bug in Pipeworx — so it lands in the `error` tier, which
330
+ * means "we have a defect", and the caller is told nothing they can act on.
331
+ * data.govt.nz sat dead behind an Imperva challenge this way and every
332
+ * status-code health check we own reported it green (7889a845). A zero-length
333
+ * body has the same shape: `Unexpected end of JSON input`, seen this week on
334
+ * uk-gazette (83% of external calls) and census.
335
+ *
336
+ * UNLIKE `httpError`, this one DOES classify, and the asymmetry is deliberate.
337
+ * A 400 is genuinely ambiguous — often the caller's bad argument, sometimes a
338
+ * query we built wrong — so blanket-classifying it would hide our own bugs.
339
+ * There is no such ambiguity here: **no argument a caller can pass makes a JSON
340
+ * API return an HTML page.** It is always the upstream, so `upstream_down:` is
341
+ * a statement of fact rather than a guess, and it keeps these out of the
342
+ * problem-tools list where they crowd out real defects.
343
+ *
344
+ * const data = await parseJson<Feed>(res, 'UK Gazette');
345
+ *
346
+ * Call it only after the `!res.ok` check — on a failed response you want
347
+ * `httpError`, which mines the body for the upstream's own explanation.
348
+ */
349
+ async function parseJson<T>(res: Response, name: string): Promise<T> {
350
+ let raw: string;
351
+ try {
352
+ raw = await res.text();
353
+ } catch {
354
+ throw new Error(
355
+ `upstream_down: ${name} returned a body that could not be read (HTTP ${res.status}). ` +
356
+ 'The connection most likely dropped mid-response; retrying is reasonable.',
357
+ );
358
+ }
359
+
360
+ const type = res.headers.get('content-type') ?? 'no content-type';
361
+
362
+ if (!raw.trim()) {
363
+ throw new Error(
364
+ `upstream_down: ${name} answered HTTP ${res.status} with an EMPTY body where JSON was expected (${type}). ` +
365
+ 'Nothing about the request can cause this — it is an upstream fault, and the same call may well work on retry.',
366
+ );
367
+ }
368
+
369
+ // Checked before parsing rather than in the catch, because knowing it is
370
+ // markup is what turns "we failed to parse something" into "they served a
371
+ // web page" — the second is diagnosable, the first is not.
372
+ const head = raw.slice(0, 200).trimStart().toLowerCase();
373
+ if (head.startsWith('<!doctype') || head.startsWith('<html') || head.startsWith('<?xml')) {
374
+ const kind = head.startsWith('<?xml') ? 'an XML document' : 'an HTML page';
375
+ // The summary, not the source. Pasting the first 120 characters of a web
376
+ // page handed the agent `<!DOCTYPE html><html lang="en"…` — the same leak
377
+ // this branch exists to describe (fleet #712).
378
+ throw new Error(
379
+ `upstream_down: ${name} answered HTTP ${res.status} with ${kind} instead of JSON (${type}). ` +
380
+ 'That is typically a bot wall, a login redirect or a maintenance page — it is returned as a SUCCESS, ' +
381
+ `so status-code health checks read it as fine. No argument change will get past it. ` +
382
+ `The page says: ${summarizeErrorBody(raw) || 'nothing readable'}`,
383
+ );
384
+ }
385
+
386
+ try {
387
+ return JSON.parse(raw) as T;
388
+ } catch {
389
+ throw new Error(
390
+ `upstream_down: ${name} answered HTTP ${res.status} with a body that is not valid JSON (${type}). ` +
391
+ `It begins: ${stripMarkup(raw).slice(0, 120) || '(unreadable)'}`,
392
+ );
393
+ }
394
+ }
395
+
396
+ /**
397
+ * `fetch`, but bounded — the fix for a systemic gap found 2026-08-30: a grep
398
+ * audit of every pack's `mcps/*\/src/index.ts` found 1,339 of ~1,500 call
399
+ * `fetch()` with NO timeout guard anywhere in the file. Two of those
400
+ * (epo-ops, statcan) were confirmed live-hanging for 4-8 minutes before this
401
+ * existed — every unguarded call carries the same risk, just unconfirmed.
402
+ *
403
+ * Mirrors the `epoFetch` wrapper `mcps/epo-ops/src/index.ts` shipped first:
404
+ * bound the request with `AbortSignal.timeout`, and on a timeout/abort throw
405
+ * an `upstream_down:` error that names the upstream and the bound rather than
406
+ * letting the raw `TimeoutError`/`AbortError` (which names neither) propagate.
407
+ * `upstream_down:` is deliberate, same reasoning as `parseJson` above — no
408
+ * argument a caller passes can make an upstream hang, so it is always the
409
+ * upstream's fault, and marking it that way keeps a slow API off the
410
+ * problem-tools list where it would crowd out our own defects.
411
+ *
412
+ * Usage — a mechanical swap for a bare `fetch(url, init)`:
413
+ *
414
+ * const res = await fetchWithTimeout(url, init, 'Some API');
415
+ *
416
+ * Pass `timeoutMs` as a fourth argument to override the default for a pack
417
+ * with a known-slower upstream; the label should be the same short name you'd
418
+ * pass to `httpError`/`httpErrorMessage` for that call.
419
+ */
420
+ async function fetchWithTimeout(
421
+ url: string | URL,
422
+ init: RequestInit = {},
423
+ name: string,
424
+ timeoutMs: number = DEFAULT_FETCH_TIMEOUT_MS,
425
+ ): Promise<Response> {
426
+ try {
427
+ return await fetch(url, { ...init, signal: AbortSignal.timeout(timeoutMs) });
428
+ } catch (err) {
429
+ if (err instanceof Error && (err.name === 'TimeoutError' || err.name === 'AbortError')) {
430
+ // States the OBSERVATION (no response in N seconds), not a diagnosis.
431
+ // "appears to be degraded" is an inference about the vendor that we have
432
+ // not checked, and it is wrong in a way that misdirects whoever reads it:
433
+ // a timeout from a Worker can equally mean OUR egress is blocked.
434
+ //
435
+ // Measured today (2026-09-01, fleet #1047): every call to
436
+ // mainnet.base.org failed from the x402 facilitator while the identical
437
+ // request from a laptop returned 200. Base was entirely healthy; the
438
+ // public RPC refuses Cloudflare Worker egress. Had this message fired
439
+ // there it would have blamed Base by name, and the next person would have
440
+ // waited for a vendor outage to clear that did not exist.
441
+ // A timeout has no status to test — there is no response at all — so
442
+ // `markInternalOrigin` is called without one: an origin we run that never
443
+ // answered is an availability failure by definition. This is the half of
444
+ // fleet #1096 with neither a SQLSTATE nor a status code to key on.
445
+ throw new Error(
446
+ markInternalOrigin(
447
+ `upstream_down: ${name} did not respond within ${timeoutMs / 1000}s. ` +
448
+ `That can be ${name} being slow or down, or this environment being unable to reach it ` +
449
+ `(some hosts refuse datacenter/Worker egress) — retry shortly, and check reachability ` +
450
+ `from elsewhere before concluding ${name} is down.`,
451
+ url,
452
+ ),
453
+ );
454
+ }
455
+ // Fleet #2382. Everything that isn't a timeout/abort here is a genuine
456
+ // NETWORK-LEVEL failure — DNS resolution, connection refused, TLS handshake,
457
+ // Cloudflare's own "Network connection lost." — meaning `fetch()` itself
458
+ // threw and no HTTP response of any kind was ever received. Until this fix
459
+ // that raw exception was rethrown VERBATIM: a bare `TypeError: fetch failed`
460
+ // (or the Workers-runtime equivalent) names no upstream, carries no class
461
+ // token, and reads exactly like a defect in OUR code — because it says
462
+ // nothing about the call at all. It landed in `error`, the tier that means
463
+ // "Pipeworx has a defect", for every one of the (at the time of writing)
464
+ // ~470 packs that call this helper directly with no wrapper of their own.
465
+ //
466
+ // `dexscreener` hit this independently (fleet #1579) and fixed it with a
467
+ // bespoke per-pack try/catch around `fetchWithTimeout`. That fix is correct
468
+ // but only covers one pack; every other caller of this shared helper still
469
+ // leaked the raw exception. Moving the same fix HERE — the one place that
470
+ // already carries the timeout case — covers every pack that uses
471
+ // `fetchWithTimeout` without a wrapper, for free, and without widening
472
+ // `classifyToolError`'s regex list: the fix is giving the message a proper
473
+ // `upstream_down:` token at the point the two facts (no response was ever
474
+ // received, and which host we were trying to reach) are actually in hand,
475
+ // not teaching the classifier to guess from prose after the fact.
476
+ //
477
+ // Safe on the same grounds as the timeout branch above: no argument a
478
+ // caller passes can make `fetch()` itself throw a connection-level error,
479
+ // so this is always an availability failure, never a caller mistake. Same
480
+ // `markInternalOrigin` treatment — an origin we run that never answered is
481
+ // still ours, not a third party's outage.
482
+ const raw = err instanceof Error ? err.message : String(err);
483
+ throw new Error(
484
+ markInternalOrigin(
485
+ `upstream_down: could not reach ${name} at all (${raw.slice(0, 160)}). ` +
486
+ `No request reached ${name}, so this says NOTHING about whether the arguments you passed ` +
487
+ 'are valid — do not re-check them on the strength of this error. Retry shortly.',
488
+ url,
489
+ ),
490
+ );
491
+ }
492
+ }
493
+
494
+ function detailSuffix(detail: string): string {
495
+ return detail ? ` — ${detail}` : '';
496
+ }
497
+
498
+ async function readDetail(res: Response): Promise<string> {
499
+ let raw: string;
500
+ try {
501
+ raw = await res.text();
502
+ } catch {
503
+ // Body already consumed, or the connection died mid-read. The status alone
504
+ // is still worth throwing — never let the error path throw its own error.
505
+ return '';
506
+ }
507
+ return summarizeErrorBody(raw);
508
+ }
509
+
510
+ /**
511
+ * Turn ANY error body — JSON, HTML, XML or plain text — into one short phrase
512
+ * that never contains markup.
513
+ *
514
+ * This used to just drop an HTML or XML body on the floor, on the reasoning
515
+ * that markup crowds out the status. That was half right. Dropping it loses the
516
+ * one sentence a caller could have acted on: an `Access Denied` title, an SDMX
517
+ * `<message:Error>` text, an OPS fault string. A 2026-08-30 support sweep
518
+ * measured 13 of 291 caller-facing error rows carrying a raw page or document
519
+ * verbatim, across 11 packs, and in every one of them the useful content —
520
+ * "Access Denied", "Invalid country code", "SCRAPE_TIMEOUT" — was in there,
521
+ * buried in markup the agent had to parse out of a string (fleet #712).
522
+ *
523
+ * So: extract the meaning, discard the markup. The output is passed through
524
+ * `stripMarkup` unconditionally, which is what lets `check:error-body-leak`
525
+ * assert mechanically that no caller-facing message can contain `<?xml`,
526
+ * `<!DOCTYPE` or `<html`.
527
+ */
528
+ function summarizeErrorBody(raw: string): string {
529
+ if (!raw || !raw.trim()) return '';
530
+
531
+ const head = raw.slice(0, 400).trimStart().toLowerCase();
532
+
533
+ // An HTML error page (Cloudflare interstitial, nginx default, a login
534
+ // redirect) says what it is in its <title>, and almost nowhere else.
535
+ if (head.startsWith('<!doctype') || head.startsWith('<html')) {
536
+ const title = htmlTitle(raw);
537
+ return title
538
+ ? `${title} (upstream returned an HTML error page, not an API response)`
539
+ : 'upstream returned an HTML error page, not an API response';
540
+ }
541
+
542
+ // XML fault documents — EPO OPS, SDMX (`<message:Error>`), SOAP faults. The
543
+ // human sentence sits in a child element whose tag name says what it is.
544
+ if (head.startsWith('<?xml') || head.startsWith('<')) {
545
+ const fault = xmlFaultText(raw);
546
+ return fault
547
+ ? `${stripMarkup(fault).slice(0, MAX_DETAIL)} (from the upstream's XML error document)`
548
+ : 'upstream returned an XML error document with no readable message';
549
+ }
550
+
551
+ // Most JSON error bodies bury one human sentence among ids and echoed request
552
+ // params. Prefer that sentence; fall back to the whole body when the shape is
553
+ // unfamiliar, since an unfamiliar shape is exactly when we can least afford to
554
+ // guess wrong and show nothing.
555
+ const fromJson = messageFromJson(raw);
556
+ return stripMarkup(fromJson ?? raw).slice(0, MAX_DETAIL);
557
+ }
558
+
559
+ /** The `<title>` of an HTML error page, or its first `<h1>` — the two places a
560
+ * bot wall, a 502 and an "Access Denied" all state what happened. */
561
+ function htmlTitle(raw: string): string | null {
562
+ const head = raw.slice(0, 4000);
563
+ for (const re of [/<title[^>]*>([\s\S]*?)<\/title>/i, /<h1[^>]*>([\s\S]*?)<\/h1>/i]) {
564
+ const m = re.exec(head);
565
+ const text = m ? stripMarkup(m[1]) : '';
566
+ if (text) return text.slice(0, 160);
567
+ }
568
+ return null;
569
+ }
570
+
571
+ /** Tag names that carry the explanation in an XML fault document, namespace
572
+ * prefix optional (`<message:Error>`, `<com:Text>`, `<faultstring>`). */
573
+ const XML_FAULT_TAG_RE =
574
+ /<(?:[A-Za-z0-9_.-]+:)?(?:text|message|description|faultstring|reason|detail|title|errormessage|error)\b[^>]*>([^<]{2,400})</i;
575
+
576
+ function xmlFaultText(raw: string): string | null {
577
+ const head = raw.slice(0, 8000);
578
+ const tagged = XML_FAULT_TAG_RE.exec(head);
579
+ if (tagged && tagged[1].trim()) return tagged[1];
580
+
581
+ // Nothing conventionally named — take the longest text node instead. A fault
582
+ // document with one sentence in an oddly named element is still readable;
583
+ // returning nothing at all is not.
584
+ let best = '';
585
+ for (const m of head.matchAll(/>([^<>]{8,400})</g)) {
586
+ const text = m[1].trim();
587
+ if (text.length > best.length) best = text;
588
+ }
589
+ return best || null;
590
+ }
591
+
592
+ /**
593
+ * Remove every tag and stray angle bracket, then collapse whitespace.
594
+ *
595
+ * Applied to everything on the way out, including the JSON and plain-text
596
+ * paths, because an upstream is free to embed markup in a JSON string field —
597
+ * and a leak is a leak regardless of which branch produced it.
598
+ */
599
+ function stripMarkup(s: string): string {
600
+ return collapse(decodeEntities(s.replace(/<[^>]*>/g, ' ')).replace(/[<>]/g, ' '));
601
+ }
602
+
603
+ /** The handful of entities that show up in error-page titles. Decoded AFTER
604
+ * tags are stripped and BEFORE the angle-bracket sweep, so `&lt;script&gt;`
605
+ * in a title cannot decode into markup that survives — EMBL-EBI's ChEMBL 500
606
+ * page renders as `500 Internal Server Error &lt; EMBL-EBI` otherwise. */
607
+ function decodeEntities(s: string): string {
608
+ return s
609
+ .replace(/&(?:amp|#0*38);/gi, '&')
610
+ .replace(/&(?:lt|#0*60);/gi, '<')
611
+ .replace(/&(?:gt|#0*62);/gi, '>')
612
+ .replace(/&(?:quot|#0*34);/gi, '"')
613
+ .replace(/&(?:#0*39|apos|#x0*27);/gi, "'")
614
+ .replace(/&nbsp;/gi, ' ');
615
+ }
616
+
617
+ /** The conventional "what went wrong" field, under any of the names upstreams
618
+ * actually use. Checked in order; first non-empty string wins. */
619
+ const MESSAGE_KEYS = [
620
+ 'message', 'error_message', 'errorMessage', 'detail', 'details',
621
+ 'description', 'error_description', 'reason', 'title', 'fault',
622
+ ];
623
+
624
+ function messageFromJson(raw: string): string | null {
625
+ let parsed: unknown;
626
+ try {
627
+ parsed = JSON.parse(raw);
628
+ } catch {
629
+ return null;
630
+ }
631
+ return pickMessage(parsed, 0);
632
+ }
633
+
634
+ function pickMessage(node: unknown, depth: number): string | null {
635
+ // Two levels covers `{error: {message}}` and `{errors: [{detail}]}`, the two
636
+ // shapes that account for nearly all of them, without walking a large payload.
637
+ if (depth > 2 || node == null) return null;
638
+
639
+ if (typeof node === 'string') return node.trim() || null;
640
+
641
+ if (Array.isArray(node)) {
642
+ for (const item of node) {
643
+ const found = pickMessage(item, depth + 1);
644
+ if (found) return found;
645
+ }
646
+ return null;
647
+ }
648
+
649
+ if (typeof node !== 'object') return null;
650
+ const obj = node as Record<string, unknown>;
651
+
652
+ for (const key of MESSAGE_KEYS) {
653
+ const v = obj[key];
654
+ if (typeof v === 'string' && v.trim()) return v.trim();
655
+ }
656
+ // `{error: …}` where error is itself an object or a string — the single most
657
+ // common wrapper, so it is worth descending into by name rather than scanning
658
+ // every key and risking picking up an echoed request parameter.
659
+ for (const key of ['error', 'errors', 'fault', 'Error', 'data']) {
660
+ if (key in obj) {
661
+ const found = pickMessage(obj[key], depth + 1);
662
+ if (found) return found;
663
+ }
664
+ }
665
+ return null;
666
+ }
667
+
668
+ /** Errors are read in a single line of log output; newlines and runs of
669
+ * whitespace make a multi-line body unreadable there. */
670
+ function collapse(s: string): string {
671
+ return s.replace(/\s+/g, ' ').trim();
672
+ }
673
+
674
+
675
+ /**
676
+ * Minimal text extractor for simple, text-based PDFs — the kind a
677
+ * mail-merge/print pipeline produces (Word or LibreOffice "export to PDF"),
678
+ * not a scanned image. Built for Kentucky's per-statute PDFs
679
+ * (apps.legislature.ky.gov/law/statutes/statute.aspx?id=...), which ARE
680
+ * exactly that shape: one object tree, FlateDecode content streams, standard
681
+ * TrueType fonts. This is the capability the state-law survey flagged as
682
+ * "the largest single win available" (docs/state-law-probe.md) — several
683
+ * state statute sites serve text ONLY as a per-section PDF, with no HTML
684
+ * fallback, and until now this repo had no way to read one.
685
+ *
686
+ * WHAT THIS DOES NOT DO: parse a general PDF. No encryption, no cross-
687
+ * reference recovery for a damaged file, no image/OCR, no custom
688
+ * `/Differences` glyph encodings, no LZW or RunLength filters, no multi-byte
689
+ * codespaces other than the 2-byte one every sampled font used. Most objects
690
+ * this needs — Catalog, Pages, a leaf Page, its Resources/Font dict, its
691
+ * Contents stream(s), a Type0 font's ToUnicode CMap — are found by scanning
692
+ * for `N 0 obj ... endobj` directly in the file bytes. If a producer moves
693
+ * one into a compressed object stream and this extractor still cannot find
694
+ * it (see the next paragraph for what IS handled), extraction degrades to
695
+ * `pages_found: 0` / empty text rather than silently returning garbage —
696
+ * check `warnings` before trusting a thin result.
697
+ *
698
+ * COMPRESSED OBJECT STREAMS (`/Type/ObjStm`) ARE SUPPORTED (fleet #2744,
699
+ * added for Wyoming's statute PDFs, which — unlike Kentucky's — are
700
+ * cross-reference-STREAM files whose Catalog and Pages root are themselves
701
+ * compressed into an ObjStm: `5205 0 obj` never appears literally in the
702
+ * file; it exists only as entry in a decompressed `/Type/ObjStm` stream).
703
+ * Every direct `/Type/ObjStm` object found in the file is inflated up front;
704
+ * its header (`/N` object pairs starting at byte `/First`) is parsed into a
705
+ * `(object number -> dict text)` map, and `getObject` falls back to that map
706
+ * whenever the direct byte-scan finds nothing. Per the PDF spec a compressed
707
+ * object can never itself contain a stream (Contents, ToUnicode CMaps, and
708
+ * ObjStm/XRef streams are therefore always direct objects, found the
709
+ * original way) — so this fallback only ever needs to resolve plain
710
+ * dictionaries/arrays (Catalog, Pages, Font, sometimes Page), which is
711
+ * exactly what Wyoming's shape needs. An ObjStm nested inside another
712
+ * ObjStm via `/Extends` is NOT walked — not observed in any file tested.
713
+ *
714
+ * ALSO FIXED HERE: the Catalog/Pages discovery no longer requires `/Type`
715
+ * to be the first thing found after `<<` in a Catalog's own dictionary text
716
+ * — the original `<<[^>]*\/Type\s*\/Catalog` regex broke the moment any
717
+ * NESTED dict (e.g. `/MarkInfo<</Marked true>>`) appeared earlier in the
718
+ * same object, because `[^>]*` cannot cross that inner `>>`. Wyoming's real
719
+ * Catalog objects are exactly this shape (`/Lang(...)/MarkInfo<<...>>
720
+ * /Metadata .../Pages .../Type/Catalog/ViewerPreferences<<...>>`, `/Type`
721
+ * nowhere near the start). Discovery now finds `/Type/Catalog` (or
722
+ * `/Type/Pages`) as plain text anywhere — direct or inside a decompressed
723
+ * ObjStm — and locates its OWN object number by nearest preceding
724
+ * `N 0 obj` marker, which works regardless of what else is in the dict.
725
+ *
726
+ * TWO FONT SHAPES, BOTH NEEDED (confirmed against real KRS statute PDFs,
727
+ * fleet #2734):
728
+ * - Simple TrueType/Type1 with `/Encoding/WinAnsiEncoding` — the common
729
+ * case. WinAnsiEncoding is BY DESIGN identical to Windows code page 1252
730
+ * (that is what "WinAnsi" means), so the raw string-literal bytes decode
731
+ * directly via `TextDecoder('windows-1252')` with no per-glyph table of
732
+ * our own to get wrong.
733
+ * - Type0/CIDFontType2 with `/Encoding/Identity-H` — seen even inside an
734
+ * otherwise-simple statute, apparently whenever the PDF producer's text
735
+ * shaping fell back to a subset-embedded font for one run (KRS 532.025's
736
+ * closing sentence, which includes a curly apostrophe, was entirely in
737
+ * this font while the rest of the document used WinAnsi). Each character
738
+ * is a 2-byte CID with NO inherent meaning; the `/ToUnicode` CMap
739
+ * attached to the font is the only place the real Unicode value lives
740
+ * (`beginbfchar`/`beginbfrange` blocks). Skipping this font shape would
741
+ * have truncated or corrupted exactly the statutes that happen to use a
742
+ * special character — not a rare edge case, a silent one.
743
+ *
744
+ * Verified end-to-end against two real KRS statutes during this pack's
745
+ * build: a single-page WinAnsi-only PDF (507.020, Murder) and a four-page PDF
746
+ * mixing both font shapes (532.025, including the CID-encoded sentence
747
+ * naming "Kimber's Law") — both reproduced the statute's true text exactly,
748
+ * eyeballed against the rendered PDF.
749
+ *
750
+ * INDIRECT /Length (fleet #2743): extended for North Dakota's Century Code
751
+ * chapter PDFs (ndlegis.gov/cencode/t*.pdf), whose content streams all write
752
+ * `/Length` as an indirect reference ("/Length 3 0 R") rather than inlining
753
+ * the literal integer the way every sampled KRS PDF did. The referenced
754
+ * object is resolved (it is always a bare integer object, never itself a
755
+ * stream) rather than falling back to scanning for the "endstream" keyword —
756
+ * that fallback slices in the EOL bytes between the compressed data and the
757
+ * keyword, which this repo's own Node zlib rejects as "trailing junk" even
758
+ * though the compressed payload is intact (Workers' DecompressionStream may
759
+ * be more lenient, but Node is what `prepush-tests.mjs` runs, so the bug was
760
+ * real either way). Verified against ND Century Code chapter 12.1-16
761
+ * (Homicide, t12-1c16.pdf) — every content stream in that 9-page PDF uses
762
+ * the indirect form.
763
+ *
764
+ * THREE FIXES PORTED FROM mcps/guam-code's PACK-LOCAL FORK (fleet #2904,
765
+ * folding in #2888's src/gu-pdf-extract.ts -- nothing about them was
766
+ * actually Guam-specific, which is why they live here now instead of only
767
+ * in that fork):
768
+ * 1. inflate()'s two stream writes were fire-and-forget
769
+ * (void writer.write(...) / void writer.close()). A malformed deflate
770
+ * stream (real, not hypothetical -- seen across several Guam chapter
771
+ * PDFs) rejects that detached promise UNHANDLED, invisible to any
772
+ * try/catch wrapped around this function's caller and fatal to the
773
+ * whole Node process; in a Worker it is an unhandled rejection on a
774
+ * live request. Both writes now carry a no-op .catch() -- the real
775
+ * failure still surfaces through reader.read() below, which every
776
+ * caller already handles.
777
+ * 2. A literal (parenthesized) string run was decoded by passing its raw
778
+ * windows-1252-decoded bytes through unchanged, which is correct for a
779
+ * WinAnsi font but wrong for a mapped1/cid font whose code space only
780
+ * means anything via its own /ToUnicode CMap -- the same lookup a hex
781
+ * string run already gets. Guam's PDF producer writes several of its
782
+ * non-WinAnsi subset-embedded fonts' text as literal strings rather
783
+ * than hex, which this extractor previously decoded as line-noise
784
+ * control characters instead of the real text.
785
+ * 3. decompressedStream now retries a /FlateDecode stream with its tail
786
+ * trimmed 1-4 bytes before giving up. Several real PDFs declare a
787
+ * /Length a few bytes LONGER than the true deflate stream (confirmed
788
+ * on Guam's 21gc033.pdf object 2: /Length says 96, the deflate stream
789
+ * completes at byte 95, byte 96 is one stray trailing byte). Python's
790
+ * zlib tolerates trailing garbage by default; the Workers/Node
791
+ * DecompressionStream used here does not and throws "Trailing junk
792
+ * found after the end of the compressed stream" -- the true stream
793
+ * length is always <= the declared one in every sample seen, never
794
+ * longer, so trimming is safe.
795
+ */
796
+
797
+ interface PdfExtractResult {
798
+ /** Extracted text, pages joined with a blank line. Empty string if nothing
799
+ * could be read — check `warnings`, not just truthiness, before deciding
800
+ * that means the document has no text. */
801
+ text: string;
802
+ /** Number of leaf /Type/Page objects found via the Pages tree (the WHOLE
803
+ * document's page count, even when `pageRange` restricted extraction to
804
+ * a subset — see `extractedPages` for how many were actually read). */
805
+ pages: number;
806
+ /** Number of pages actually extracted into `text`. Equal to `pages`
807
+ * unless a `pageRange` option restricted the run. */
808
+ extractedPages: number;
809
+ /** Non-fatal problems found while extracting (unknown font, missing
810
+ * ToUnicode, etc). A non-empty array does not mean the text is wrong, but
811
+ * it means something was guessed rather than read. */
812
+ warnings: string[];
813
+ }
814
+
815
+ interface PdfExtractOptions {
816
+ /** Restrict extraction to a slice of the document's leaf pages, bounded by
817
+ * PAGE OBJECT NUMBERS rather than page indexes — built for sources (e.g.
818
+ * New Mexico's NMOneSource chapter PDFs, fleet #2884) whose per-section
819
+ * bookmarks in `extractPdfOutline`'s output give a /Dest page object
820
+ * directly, with no indexing step of their own needed. `fromPageObj` is
821
+ * resolved to its position in document order and extraction starts
822
+ * there; `toPageObj` (also resolved to its position) is the LAST page
823
+ * included — pass the object a FOLLOWING bookmark points to and extract
824
+ * up to (not including) it by resolving that page's own PRECEDING page
825
+ * instead, since this is page-granularity, not byte-granularity. If
826
+ * `fromPageObj` is not found among the document's pages, this option is
827
+ * ignored and the whole document is extracted (a safe fallback, not a
828
+ * silent wrong answer — a warning is added). */
829
+ pageRange?: { fromPageObj: number; toPageObj?: number };
830
+
831
+ /** Called right after a font's own /ToUnicode CMap is parsed (for either
832
+ * a Type0 font's CID map or a simple font's 1-byte map), before it is
833
+ * used to decode any text. Return a REPLACEMENT map to override specific
834
+ * codes, or return nothing (undefined) to use the parsed map unchanged.
835
+ * Built for mcps/guam-code (fleet #2888/#2904): one embedded symbol font
836
+ * per chapter PDF carries a /ToUnicode CMap that is itself wrong in the
837
+ * SOURCE file (confirmed by eye against the rendered PDF — not a general
838
+ * parsing defect this shared extractor could detect), so the fix is
839
+ * keyed by matching `fontDict` against something only that caller's PDFs
840
+ * produce (there, a font name) rather than built into this file. */
841
+ fontCMapOverride?: (fontDict: string, map: Map<number, number>) => Map<number, number> | void;
842
+ }
843
+
844
+ interface PdfOutlineEntry {
845
+ /** The bookmark's own text, verbatim (e.g. "30-2-1. Murder." or
846
+ * "ARTICLE 1 General Provisions" or "ANNOTATIONS"). */
847
+ title: string;
848
+ /** The PDF page OBJECT NUMBER this bookmark's /Dest (or /A GoTo action)
849
+ * points to, or null if neither was found on this node. Pass this
850
+ * straight into `extractPdfText`'s `pageRange` option — no index lookup
851
+ * needed on the caller's side. */
852
+ pageObjNum: number | null;
853
+ /** Nesting depth in the outline tree, root's children = 0. A section's
854
+ * own "ANNOTATIONS" child (depth = section's depth + 1) is how a caller
855
+ * distinguishes statutory text from case annotations that start on the
856
+ * same page — see mcps/new-mexico-code for the worked example. */
857
+ depth: number;
858
+ }
859
+
860
+ type FontDecoder =
861
+ | { kind: 'winansi' }
862
+ | { kind: 'cid'; map: Map<number, number> }
863
+ // Simple (1-byte-code) font decoded via its own /ToUnicode CMap rather than
864
+ // WinAnsi — see fontDecoderFor's comment on North Dakota's subsetted fonts.
865
+ | { kind: 'mapped1'; map: Map<number, number> };
866
+
867
+ /** Inflate one `/FlateDecode` stream using the platform's own zlib — no
868
+ * dependency, works identically in the Workers runtime and in Node (used by
869
+ * this repo's prepush tests), because both implement the standard
870
+ * CompressionStream/DecompressionStream API over the zlib wire format PDF's
871
+ * FlateDecode filter already uses. */
872
+ async function inflate(bytes: Uint8Array): Promise<Uint8Array> {
873
+ const ds = new DecompressionStream('deflate');
874
+ const writer = ds.writable.getWriter();
875
+ // Fire-and-forget on purpose: the actual decompression result/error comes
876
+ // through reader.read() below, which every caller already wraps in a
877
+ // try/catch. Without a .catch() here, a malformed deflate stream rejects
878
+ // this DETACHED write/close promise UNHANDLED -- invisible to that
879
+ // try/catch and fatal to the whole process (ported from
880
+ // mcps/guam-code's gu-pdf-extract.ts FORK FIX 3/3, fleet #2888/#2904;
881
+ // confirmed crashing scripts/bake-sections.mjs outright on real Guam
882
+ // chapter PDFs). The no-op .catch() here does not hide a real failure --
883
+ // it still surfaces through reader.read() for the caller to see.
884
+ const writeDone = writer.write(bytes).catch(() => {});
885
+ const closeDone = writer.close().catch(() => {});
886
+ void writeDone;
887
+ void closeDone;
888
+ const chunks: Uint8Array[] = [];
889
+ const reader = ds.readable.getReader();
890
+ for (;;) {
891
+ const { done, value } = await reader.read();
892
+ if (done) break;
893
+ chunks.push(value);
894
+ }
895
+ const total = chunks.reduce((n, c) => n + c.length, 0);
896
+ const out = new Uint8Array(total);
897
+ let offset = 0;
898
+ for (const c of chunks) {
899
+ out.set(c, offset);
900
+ offset += c.length;
901
+ }
902
+ return out;
903
+ }
904
+
905
+ /**
906
+ * Undo PDF string-literal escaping — `\(`, `\)`, `\\`, `\n`/`\r`/`\t`,
907
+ * octal `\ddd`, and a trailing backslash-newline line continuation — on a
908
+ * windows-1252-decoded slice. Safe to run on the decoded string rather than
909
+ * raw bytes: every character this function inspects or produces (backslash,
910
+ * parens, digits, the letters n/r/t/b/f) is plain ASCII, which windows-1252
911
+ * never remaps, so the 1-byte-per-character alignment this relies on holds
912
+ * throughout.
913
+ */
914
+ function unescapePdfLiteral(s: string): string {
915
+ let out = '';
916
+ for (let i = 0; i < s.length; i++) {
917
+ const c = s[i];
918
+ if (c !== '\\') { out += c; continue; }
919
+ const next = s[i + 1];
920
+ if (next === undefined) continue;
921
+ if (next === 'n') { out += '\n'; i++; }
922
+ else if (next === 'r') { out += '\r'; i++; }
923
+ else if (next === 't') { out += '\t'; i++; }
924
+ else if (next === 'b' || next === 'f') { i++; }
925
+ else if (next === '(' || next === ')' || next === '\\') { out += next; i++; }
926
+ else if (next >= '0' && next <= '7') {
927
+ let oct = next, j = i + 1, k = 0;
928
+ while (k < 2 && s[j + 1] >= '0' && s[j + 1] <= '7') { j++; oct += s[j]; k++; }
929
+ out += String.fromCharCode(parseInt(oct, 8) & 0xff);
930
+ i = j;
931
+ } else if (next === '\n') { i++; }
932
+ else if (next === '\r') { i++; if (s[i + 1] === '\n') i++; }
933
+ else { out += next; i++; }
934
+ }
935
+ return out;
936
+ }
937
+
938
+ interface RawObject {
939
+ dict: string;
940
+ streamBytes: Uint8Array | null;
941
+ }
942
+
943
+ /** Extract readable text from a simple, non-encrypted, non-scanned PDF. */
944
+ async function extractPdfText(buf: ArrayBuffer, options?: PdfExtractOptions): Promise<PdfExtractResult> {
945
+ const bytes = new Uint8Array(buf);
946
+ // windows-1252 decode is total (every byte maps to exactly one UTF-16 code
947
+ // unit) and length-preserving, so a match index found in `scan` is also a
948
+ // valid BYTE offset into `bytes` — used below to slice stream data without
949
+ // ever routing binary bytes through a lossy string round-trip.
950
+ const scan = new TextDecoder('windows-1252').decode(bytes);
951
+ const warnings: string[] = [];
952
+ const objCache = new Map<number, RawObject | null>();
953
+
954
+ // Byte offset of every DIRECT "N 0 obj" marker, sorted, for two uses: (1)
955
+ // the object-lookup fallback stays regex-per-object the same as before;
956
+ // (2) `objNumBefore` below answers "which object's text is this index
957
+ // inside", which is how Catalog/Pages/ObjStm discovery works regardless of
958
+ // what else that object's dictionary contains — see the file header.
959
+ const objStarts: { index: number; num: number }[] = [];
960
+ // Object number -> index right after its "N 0 obj" marker (i.e. where the
961
+ // dictionary text begins), for every object found by the SAME forward
962
+ // scan that builds objStarts — first occurrence wins, matching what a
963
+ // one-shot `.exec` would find. This turns getDirectObject's lookup from
964
+ // one full-document regex scan PER DISTINCT OBJECT NUMBER into one O(1)
965
+ // map read: with a large page-tree walk (New Mexico's chapter PDFs run
966
+ // 657-1,138+ pages, fleet #2884) the old per-call scan made even a
967
+ // `pageRange`-restricted extraction cost nearly as much as extracting the
968
+ // whole document, because walking the Pages tree alone touches every
969
+ // page's object once regardless of how many are actually rendered —
970
+ // measured 1,611ms for a 56-of-1,138-page extraction before this fix vs.
971
+ // 1,699ms for the full document. The fallback regex scan below is kept
972
+ // for any object this forward pass didn't catch (none observed, but
973
+ // cheaper to keep than to prove impossible).
974
+ const objEndByNum = new Map<number, number>();
975
+ {
976
+ const re = /(\d+)\s+0\s+obj\b/g;
977
+ let m: RegExpExecArray | null;
978
+ while ((m = re.exec(scan))) {
979
+ objStarts.push({ index: m.index, num: parseInt(m[1], 10) });
980
+ const num = parseInt(m[1], 10);
981
+ if (!objEndByNum.has(num)) objEndByNum.set(num, m.index + m[0].length);
982
+ }
983
+ }
984
+ function objNumBefore(markerIndex: number): number | null {
985
+ let lo = 0, hi = objStarts.length - 1, ans: number | null = null;
986
+ while (lo <= hi) {
987
+ const mid = (lo + hi) >> 1;
988
+ if (objStarts[mid].index <= markerIndex) { ans = objStarts[mid].num; lo = mid + 1; }
989
+ else hi = mid - 1;
990
+ }
991
+ return ans;
992
+ }
993
+
994
+ function getDirectObject(n: number): RawObject | null {
995
+ if (objCache.has(n)) return objCache.get(n) ?? null;
996
+ const fastEnd = objEndByNum.get(n);
997
+ let objStart: number;
998
+ if (fastEnd !== undefined) {
999
+ objStart = fastEnd;
1000
+ } else {
1001
+ const re = new RegExp(`(?:^|\\D)${n}\\s+0\\s+obj`);
1002
+ const m = re.exec(scan);
1003
+ if (!m) { objCache.set(n, null); return null; }
1004
+ objStart = m.index + m[0].length;
1005
+ }
1006
+ const endobjIdx = scan.indexOf('endobj', objStart);
1007
+ const streamIdx = scan.indexOf('stream', objStart);
1008
+ let dict: string;
1009
+ let streamBytes: Uint8Array | null = null;
1010
+ if (streamIdx !== -1 && (endobjIdx === -1 || streamIdx < endobjIdx)) {
1011
+ dict = scan.slice(objStart, streamIdx);
1012
+ let dataStart = streamIdx + 'stream'.length;
1013
+ if (bytes[dataStart] === 0x0d) dataStart++;
1014
+ if (bytes[dataStart] === 0x0a) dataStart++;
1015
+ // Matches EITHER "/Length 428" (literal) or "/Length 12 0 R"
1016
+ // (indirect) — group 2 is present only for the indirect form. Written
1017
+ // this way ON PURPOSE instead of the more obvious
1018
+ // `/\/Length\s+(\d+)(?!\s+0\s+R)/` for a literal: that negative
1019
+ // lookahead is a backtracking trap for any MULTI-DIGIT indirect
1020
+ // reference. On "/Length 12 0 R", a greedy `(\d+)` first tries "12",
1021
+ // the lookahead correctly forbids it (followed by " 0 R") — but the
1022
+ // engine then backtracks `\d+` down to "1", at which point the
1023
+ // lookahead is checked against "2 0 R", which does NOT start with
1024
+ // whitespace, so the forbidden pattern no longer matches and the
1025
+ // lookahead (wrongly) PASSES. The match silently becomes "1" instead
1026
+ // of failing, so this object's /Length reads as the literal integer 1
1027
+ // instead of as a reference to resolve — exactly the shape of North
1028
+ // Dakota's Century Code chapter PDFs (fleet #2743), whose indirect
1029
+ // Length refs are almost all two or more digits. An explicit optional
1030
+ // group has nothing to backtrack: `(\d+)` greedily keeps "12" and the
1031
+ // optional `(\s+0\s+R)?` either matches right after it or doesn't.
1032
+ const lenDictMatch = /\/Length\s+(\d+)(\s+0\s+R)?/.exec(dict);
1033
+ let dataEnd = -1;
1034
+ if (lenDictMatch) {
1035
+ if (!lenDictMatch[2]) {
1036
+ dataEnd = dataStart + parseInt(lenDictMatch[1], 10);
1037
+ } else {
1038
+ // Indirect reference. The referenced object is a bare integer
1039
+ // ("12 0 obj\n4541\nendobj"), never itself a stream, so resolving
1040
+ // it through the same getObject() is safe (no recursion risk back
1041
+ // onto object n). Falling back to scanning for the "endstream"
1042
+ // keyword instead — which the pre-fix code did unconditionally
1043
+ // here — slices in the EOL bytes PDF producers place between the
1044
+ // compressed data and that keyword, which Node's strict zlib
1045
+ // rejects as "trailing junk" even though the compressed payload
1046
+ // itself is intact.
1047
+ const lenObj = getObject(parseInt(lenDictMatch[1], 10));
1048
+ const lenVal = lenObj ? parseInt(lenObj.dict.trim(), 10) : NaN;
1049
+ if (Number.isFinite(lenVal) && lenVal >= 0) dataEnd = dataStart + lenVal;
1050
+ }
1051
+ }
1052
+ if (dataEnd === -1) dataEnd = scan.indexOf('endstream', dataStart);
1053
+ streamBytes = bytes.slice(dataStart, dataEnd);
1054
+ } else {
1055
+ dict = scan.slice(objStart, endobjIdx === -1 ? scan.length : endobjIdx);
1056
+ }
1057
+ const result = { dict, streamBytes };
1058
+ objCache.set(n, result);
1059
+ return result;
1060
+ }
1061
+
1062
+ // Compressed objects (inside a /Type/ObjStm) — object number -> dict text.
1063
+ // Populated below, before any Catalog/Pages/Font lookup runs, because a
1064
+ // compressed Catalog or Pages root (Wyoming's shape) must resolve exactly
1065
+ // like a direct one from that point on. Never holds a stream: the PDF spec
1066
+ // forbids a compressed object from containing one.
1067
+ const compressedObjects = new Map<number, string>();
1068
+ {
1069
+ const objStmNums: number[] = [];
1070
+ const re = /\/Type\s*\/ObjStm\b/g;
1071
+ let m: RegExpExecArray | null;
1072
+ while ((m = re.exec(scan))) {
1073
+ const n = objNumBefore(m.index);
1074
+ if (n !== null) objStmNums.push(n);
1075
+ }
1076
+ for (const n of objStmNums) {
1077
+ const obj = getDirectObject(n);
1078
+ if (!obj || !obj.streamBytes) { warnings.push(`ObjStm ${n} has no stream data`); continue; }
1079
+ let data: Uint8Array;
1080
+ if (/\/Filter\s*\/FlateDecode/.test(obj.dict)) {
1081
+ try {
1082
+ data = await inflate(obj.streamBytes);
1083
+ } catch (e) {
1084
+ warnings.push(`inflate failed for ObjStm ${n}: ${e instanceof Error ? e.message : String(e)}`);
1085
+ continue;
1086
+ }
1087
+ } else {
1088
+ data = obj.streamBytes;
1089
+ }
1090
+ const nMatch = /\/N\s+(\d+)/.exec(obj.dict);
1091
+ const firstMatch = /\/First\s+(\d+)/.exec(obj.dict);
1092
+ if (!nMatch || !firstMatch) { warnings.push(`ObjStm ${n} missing /N or /First`); continue; }
1093
+ const count = parseInt(nMatch[1], 10);
1094
+ const first = parseInt(firstMatch[1], 10);
1095
+ const header = new TextDecoder('latin1').decode(data.slice(0, first)).trim().split(/\s+/).map((x) => parseInt(x, 10));
1096
+ const entries: { num: number; offset: number }[] = [];
1097
+ for (let i = 0; i + 1 < header.length && entries.length < count; i += 2) {
1098
+ entries.push({ num: header[i], offset: header[i + 1] });
1099
+ }
1100
+ const body = new TextDecoder('windows-1252').decode(data);
1101
+ for (let i = 0; i < entries.length; i++) {
1102
+ const start = first + entries[i].offset;
1103
+ const end = i + 1 < entries.length ? first + entries[i + 1].offset : data.length;
1104
+ if (start < 0 || end > data.length || start > end) {
1105
+ warnings.push(`ObjStm ${n} entry ${entries[i].num} has an out-of-range offset, skipped`);
1106
+ continue;
1107
+ }
1108
+ compressedObjects.set(entries[i].num, body.slice(start, end));
1109
+ }
1110
+ }
1111
+ }
1112
+
1113
+ // The lookup every caller below actually uses: direct object text, falling
1114
+ // back to a compressed one. Never both — a given object number is either
1115
+ // found literally in the file or inside exactly one ObjStm, not both.
1116
+ function getObject(n: number): RawObject | null {
1117
+ const direct = getDirectObject(n);
1118
+ if (direct) return direct;
1119
+ const compressed = compressedObjects.get(n);
1120
+ if (compressed !== undefined) return { dict: compressed, streamBytes: null };
1121
+ return null;
1122
+ }
1123
+
1124
+ async function decompressedStream(n: number): Promise<Uint8Array | null> {
1125
+ const obj = getObject(n);
1126
+ if (!obj || !obj.streamBytes) return null;
1127
+ if (/\/FlateDecode/.test(obj.dict)) {
1128
+ // Ported from mcps/guam-code's gu-pdf-extract.ts FORK FIX (fleet
1129
+ // #2888/#2904): several real PDFs store a /Length a small number of
1130
+ // bytes LONGER than the TRUE deflate stream length (confirmed on
1131
+ // Guam's 21gc033.pdf object 2: /Length says 96, the deflate stream is
1132
+ // complete at byte 95, and the 96th byte is one stray trailing byte).
1133
+ // Python's zlib tolerates trailing garbage by default; the
1134
+ // Workers/Node DecompressionStream used here does not and throws
1135
+ // "Trailing junk found after the end of the compressed stream".
1136
+ // Rather than fail the whole page, retry with the slice trimmed one
1137
+ // byte shorter a few times -- the true stream length is always <= the
1138
+ // declared one in every sample seen, never longer.
1139
+ for (let trim = 0; trim <= 4 && trim < obj.streamBytes.length; trim++) {
1140
+ try {
1141
+ return await inflate(trim === 0 ? obj.streamBytes : obj.streamBytes.subarray(0, obj.streamBytes.length - trim));
1142
+ } catch (e) {
1143
+ if (trim === 4) {
1144
+ warnings.push(`inflate failed for object ${n} even after trimming trailing bytes: ${e instanceof Error ? e.message : String(e)}`);
1145
+ return null;
1146
+ }
1147
+ }
1148
+ }
1149
+ return null;
1150
+ }
1151
+ return obj.streamBytes;
1152
+ }
1153
+
1154
+ // Catalog -> Pages -> Kids, walked in document order. Falls back to the
1155
+ // first /Type/Pages object if no /Type/Catalog is found. Both searches
1156
+ // look for the TYPE MARKER as plain text first (direct text, or inside any
1157
+ // decompressed ObjStm body) and then find which object that text belongs
1158
+ // to — see the file header for why this replaced a single greedy regex.
1159
+ let pagesRef: number | null = null;
1160
+ function findObjWithType(typeName: string): number | null {
1161
+ const directIdx = scan.search(new RegExp(`/Type\\s*/${typeName}\\b`));
1162
+ if (directIdx !== -1) {
1163
+ const n = objNumBefore(directIdx);
1164
+ if (n !== null) return n;
1165
+ }
1166
+ const typeRe = new RegExp(`/Type\\s*/${typeName}\\b`);
1167
+ for (const [num, text] of compressedObjects) {
1168
+ if (typeRe.test(text)) return num;
1169
+ }
1170
+ return null;
1171
+ }
1172
+ const catNum = findObjWithType('Catalog');
1173
+ if (catNum !== null) {
1174
+ const catObj = getObject(catNum);
1175
+ const pm = catObj ? /\/Pages\s+(\d+)\s+0\s+R/.exec(catObj.dict) : null;
1176
+ if (pm) pagesRef = parseInt(pm[1], 10);
1177
+ }
1178
+ if (pagesRef === null) {
1179
+ pagesRef = findObjWithType('Pages');
1180
+ }
1181
+
1182
+ const leafPages: number[] = [];
1183
+ function walkPages(n: number, depth: number): void {
1184
+ if (depth > 12) { warnings.push('Pages tree exceeded depth 12, stopped walking'); return; }
1185
+ const obj = getObject(n);
1186
+ if (!obj) return;
1187
+ if (/\/Type\s*\/Page\b(?!s)/.test(obj.dict)) { leafPages.push(n); return; }
1188
+ const kidsMatch = /\/Kids\s*\[([^\]]*)\]/.exec(obj.dict);
1189
+ if (!kidsMatch) return;
1190
+ for (const km of kidsMatch[1].matchAll(/(\d+)\s+0\s+R/g)) walkPages(parseInt(km[1], 10), depth + 1);
1191
+ }
1192
+ if (pagesRef !== null) walkPages(pagesRef, 0);
1193
+ else warnings.push('no /Type/Catalog or /Type/Pages object found');
1194
+
1195
+ const totalPages = leafPages.length;
1196
+ let pagesToExtract = leafPages;
1197
+ if (options?.pageRange) {
1198
+ const { fromPageObj, toPageObj } = options.pageRange;
1199
+ const fromIdx = leafPages.indexOf(fromPageObj);
1200
+ if (fromIdx === -1) {
1201
+ warnings.push(
1202
+ `pageRange.fromPageObj ${fromPageObj} is not a leaf page in this document — extracting the whole document instead of the requested range.`,
1203
+ );
1204
+ } else {
1205
+ let toIdx = leafPages.length - 1;
1206
+ if (toPageObj !== undefined) {
1207
+ const foundToIdx = leafPages.indexOf(toPageObj);
1208
+ // toPageObj not found, or found strictly BEFORE fromIdx (a bad or
1209
+ // out-of-order bookmark), falls back to end-of-document rather than
1210
+ // producing a negative-length slice. Equal to fromIdx is normal and
1211
+ // valid — a section whose first bookmarked child (e.g. its own
1212
+ // "ANNOTATIONS") starts on the SAME page is a single-page range,
1213
+ // not an error.
1214
+ if (foundToIdx >= fromIdx) toIdx = foundToIdx;
1215
+ else if (foundToIdx !== -1) {
1216
+ warnings.push(
1217
+ `pageRange.toPageObj ${toPageObj} resolves to a page before fromPageObj ${fromPageObj} — extracting through end of document instead.`,
1218
+ );
1219
+ } else {
1220
+ warnings.push(`pageRange.toPageObj ${toPageObj} is not a leaf page in this document — extracting through end of document instead.`);
1221
+ }
1222
+ }
1223
+ pagesToExtract = leafPages.slice(fromIdx, toIdx + 1);
1224
+ }
1225
+ }
1226
+
1227
+ /** Parse a /ToUnicode CMap stream's `beginbfchar`/`beginbfrange` blocks into
1228
+ * a code -> Unicode-codepoint map. Shared between Type0's 2-byte CIDs and
1229
+ * a simple font's 1-byte codes — the CMap text format is identical either
1230
+ * way; only how the caller SLICES its hex string differs (see decodeRun). */
1231
+ async function parseToUnicodeCMap(cmapObjNum: number): Promise<Map<number, number>> {
1232
+ const cmapBytes = await decompressedStream(cmapObjNum);
1233
+ const cmapText = cmapBytes ? new TextDecoder('latin1').decode(cmapBytes) : '';
1234
+ const map = new Map<number, number>();
1235
+ for (const block of cmapText.matchAll(/beginbfchar([\s\S]*?)endbfchar/g)) {
1236
+ for (const pair of block[1].matchAll(/<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>/g)) {
1237
+ map.set(parseInt(pair[1], 16), parseInt(pair[2].slice(0, 4), 16));
1238
+ }
1239
+ }
1240
+ for (const block of cmapText.matchAll(/beginbfrange([\s\S]*?)endbfrange/g)) {
1241
+ for (const triple of block[1].matchAll(/<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>/g)) {
1242
+ const lo = parseInt(triple[1], 16), hi = parseInt(triple[2], 16), base = parseInt(triple[3].slice(0, 4), 16);
1243
+ for (let c = lo; c <= hi && c - lo < 65536; c++) map.set(c, base + (c - lo));
1244
+ }
1245
+ }
1246
+ return map;
1247
+ }
1248
+
1249
+ async function fontDecoderFor(fontObjNum: number): Promise<FontDecoder> {
1250
+ const obj = getObject(fontObjNum);
1251
+ if (!obj) return { kind: 'winansi' };
1252
+ if (/\/Subtype\s*\/Type0\b/.test(obj.dict)) {
1253
+ const tuMatch = /\/ToUnicode\s+(\d+)\s+0\s+R/.exec(obj.dict);
1254
+ if (!tuMatch) {
1255
+ warnings.push(`Type0 font object ${fontObjNum} has no /ToUnicode; its text will show as U+FFFD`);
1256
+ return { kind: 'cid', map: new Map() };
1257
+ }
1258
+ const map = await parseToUnicodeCMap(parseInt(tuMatch[1], 10));
1259
+ return { kind: 'cid', map: options?.fontCMapOverride?.(obj.dict, map) ?? map };
1260
+ }
1261
+ // Simple font. Every KRS sample used /WinAnsiEncoding, which decodes
1262
+ // directly as windows-1252 bytes below — unchanged. North Dakota's
1263
+ // Century Code PDFs (fleet #2743) use simple TrueType fonts that declare
1264
+ // NO /Encoding at all: their embedded subsets number glyphs 0, 1, 2... in
1265
+ // whatever order the subsetter emitted them, with no relationship to any
1266
+ // standard code page — reading code 0x01 as the WinAnsi byte 0x01 (a
1267
+ // control character) produces blank/garbled output, not a wrong letter.
1268
+ // Those fonts DO carry a /ToUnicode CMap (the same mechanism Type0 fonts
1269
+ // use above, just keyed by a 1-byte code instead of a 2-byte CID), which
1270
+ // is the only place the real character is recorded — use it whenever the
1271
+ // font isn't declared WinAnsi rather than guessing.
1272
+ if (!/\/Encoding\s*\/WinAnsiEncoding\b/.test(obj.dict)) {
1273
+ const tuMatch = /\/ToUnicode\s+(\d+)\s+0\s+R/.exec(obj.dict);
1274
+ if (tuMatch) {
1275
+ const parsedMap = await parseToUnicodeCMap(parseInt(tuMatch[1], 10));
1276
+ const map = options?.fontCMapOverride?.(obj.dict, parsedMap) ?? parsedMap;
1277
+ if (map.size) return { kind: 'mapped1', map };
1278
+ }
1279
+ warnings.push(`font object ${fontObjNum} has a non-WinAnsi simple encoding and no usable /ToUnicode; decoded as WinAnsi anyway`);
1280
+ }
1281
+ return { kind: 'winansi' };
1282
+ }
1283
+
1284
+ const TOKEN_RE =
1285
+ /\/(\S+)\s+[\d.]+\s+Tf|\[((?:\\.|[^\]])*)\]\s*TJ|<([0-9A-Fa-f\s]*)>\s*Tj|\(((?:\\.|[^()])*)\)\s*Tj|1\s+0\s+0\s+1\s+[\d.-]+\s+([\d.-]+)\s+Tm/g;
1286
+ const TJ_PIECE_RE = /\(((?:\\.|[^()])*)\)|<([0-9A-Fa-f\s]*)>/g;
1287
+
1288
+ function decodeRun(dec: FontDecoder, literal: string | undefined, hex: string | undefined): string {
1289
+ if (literal !== undefined) {
1290
+ const unescaped = unescapePdfLiteral(literal);
1291
+ // Ported from mcps/guam-code's gu-pdf-extract.ts FORK FIX 1/2 (fleet
1292
+ // #2888/#2904): a WinAnsi run's literal-string bytes ARE already the
1293
+ // correct windows-1252 text (unchanged below) -- but a non-WinAnsi
1294
+ // simple font ('mapped1') or a Type0 font ('cid') can ALSO show up as
1295
+ // a literal string rather than hex. Guam's PDF producer writes
1296
+ // several of its subset-embedded fonts' text this way; passing the
1297
+ // raw byte through unchanged (the old behavior here) skips that
1298
+ // font's /ToUnicode map entirely and produces line-noise control
1299
+ // characters instead of the real character -- route it through the
1300
+ // SAME decoder a hex run below already uses.
1301
+ if (dec.kind === 'winansi') return unescaped;
1302
+ let out = '';
1303
+ for (let i = 0; i < unescaped.length; i++) {
1304
+ const code = unescaped.charCodeAt(i) & 0xff;
1305
+ out += String.fromCodePoint(dec.map.get(code) ?? 0xfffd);
1306
+ }
1307
+ return out;
1308
+ }
1309
+ const cleaned = (hex ?? '').replace(/\s+/g, '');
1310
+ let out = '';
1311
+ if (dec.kind === 'mapped1') {
1312
+ // 1-byte code per glyph (a simple font's code space), unlike Type0's
1313
+ // 2-byte CIDs below — see fontDecoderFor.
1314
+ for (let i = 0; i + 2 <= cleaned.length; i += 2) {
1315
+ const code = parseInt(cleaned.slice(i, i + 2), 16);
1316
+ out += String.fromCodePoint(dec.map.get(code) ?? 0xfffd);
1317
+ }
1318
+ return out;
1319
+ }
1320
+ for (let i = 0; i + 4 <= cleaned.length; i += 4) {
1321
+ const cid = parseInt(cleaned.slice(i, i + 4), 16);
1322
+ if (dec.kind === 'cid') out += String.fromCodePoint(dec.map.get(cid) ?? 0xfffd);
1323
+ else out += String.fromCharCode(cid & 0xff);
1324
+ }
1325
+ return out;
1326
+ }
1327
+
1328
+ const pageTexts: string[] = [];
1329
+ for (const pn of pagesToExtract) {
1330
+ const page = getObject(pn);
1331
+ if (!page) continue;
1332
+
1333
+ let resourcesDict = page.dict;
1334
+ const resRef = /\/Resources\s+(\d+)\s+0\s+R/.exec(page.dict);
1335
+ if (resRef) {
1336
+ const ro = getObject(parseInt(resRef[1], 10));
1337
+ if (ro) resourcesDict = ro.dict;
1338
+ }
1339
+ // /Font is usually inline in Resources, but North Dakota's Century Code
1340
+ // PDFs (fleet #2743) write it as an indirect reference ("/Font 18 0 R")
1341
+ // to a separate dict object — resolve that one hop before giving up.
1342
+ let fontDictBody: string | null = null;
1343
+ const inlineFontMatch = /\/Font\s*<<([^>]*)>>/.exec(resourcesDict);
1344
+ if (inlineFontMatch) {
1345
+ fontDictBody = inlineFontMatch[1];
1346
+ } else {
1347
+ const fontRefMatch = /\/Font\s+(\d+)\s+0\s+R/.exec(resourcesDict);
1348
+ if (fontRefMatch) {
1349
+ const fontObj = getObject(parseInt(fontRefMatch[1], 10));
1350
+ if (fontObj) fontDictBody = fontObj.dict;
1351
+ }
1352
+ }
1353
+ const fontMap = new Map<string, FontDecoder>();
1354
+ if (fontDictBody) {
1355
+ for (const fm of fontDictBody.matchAll(/\/(\S+?)\s+(\d+)\s+0\s+R/g)) {
1356
+ fontMap.set(fm[1], await fontDecoderFor(parseInt(fm[2], 10)));
1357
+ }
1358
+ }
1359
+
1360
+ const contentsMatch = /\/Contents\s*(\[[^\]]*\]|\d+\s+0\s+R)/.exec(page.dict);
1361
+ const contentRefs = contentsMatch
1362
+ ? [...contentsMatch[1].matchAll(/(\d+)\s+0\s+R/g)].map((m) => parseInt(m[1], 10))
1363
+ : [];
1364
+ let raw = '';
1365
+ for (const cr of contentRefs) {
1366
+ const dec = await decompressedStream(cr);
1367
+ if (dec) raw += new TextDecoder('latin1').decode(dec) + '\n';
1368
+ }
1369
+
1370
+ let out = '';
1371
+ let currentFont: string | null = null;
1372
+ let lastY: number | null = null;
1373
+ for (const m of raw.matchAll(TOKEN_RE)) {
1374
+ if (m[1] !== undefined) { currentFont = m[1]; continue; }
1375
+ if (m[5] !== undefined) {
1376
+ const y = parseFloat(m[5]);
1377
+ if (lastY !== null && Math.abs(y - lastY) > 1) out += '\n';
1378
+ lastY = y;
1379
+ continue;
1380
+ }
1381
+ const dec = (currentFont && fontMap.get(currentFont)) || { kind: 'winansi' as const };
1382
+ if (m[2] !== undefined) {
1383
+ for (const piece of m[2].matchAll(TJ_PIECE_RE)) out += decodeRun(dec, piece[1], piece[2]);
1384
+ } else if (m[3] !== undefined) {
1385
+ out += decodeRun(dec, undefined, m[3]);
1386
+ } else if (m[4] !== undefined) {
1387
+ out += decodeRun(dec, m[4], undefined);
1388
+ }
1389
+ }
1390
+ pageTexts.push(out.trim());
1391
+ }
1392
+
1393
+ return { text: pageTexts.join('\n\n'), pages: totalPages, extractedPages: pagesToExtract.length, warnings };
1394
+ }
1395
+
1396
+ /**
1397
+ * Parse a PDF's outline (bookmark) tree into a flat, document-order list of
1398
+ * {title, pageObjNum, depth} entries — WITHOUT decompressing any content
1399
+ * stream, so this is orders of magnitude cheaper than `extractPdfText` and
1400
+ * safe to run just to decide WHERE to extract from. Built for
1401
+ * mcps/new-mexico-code (fleet #2884): NMOneSource publishes each NMSA 1978
1402
+ * CHAPTER as one PDF (657-1,138+ pages observed) but bookmarks every
1403
+ * ARTICLE heading (depth 0), every SECTION citation (depth 1, e.g.
1404
+ * "30-2-1. Murder."), and that section's own "ANNOTATIONS" sub-heading
1405
+ * (depth 2) — giving a caller everything needed to resolve a bare citation
1406
+ * straight to a page RANGE via `extractPdfText`'s `pageRange` option,
1407
+ * without ever extracting pages the caller didn't ask for.
1408
+ *
1409
+ * Assumes outline objects are DIRECT (never compressed into an /ObjStm) —
1410
+ * true of every source sampled while building this. A PDF whose outline IS
1411
+ * compressed, or that has no /Outlines at all, returns an empty array
1412
+ * rather than guessing; check for that before concluding a citation isn't
1413
+ * bookmarked.
1414
+ */
1415
+ function extractPdfOutline(buf: ArrayBuffer): PdfOutlineEntry[] {
1416
+ const bytes = new Uint8Array(buf);
1417
+ const scan = new TextDecoder('windows-1252').decode(bytes);
1418
+
1419
+ const objStarts: { index: number; num: number }[] = [];
1420
+ // See extractPdfText's identical map for why: one O(1) lookup per object
1421
+ // instead of one full-document regex scan — an outline tree with ~1,400
1422
+ // bookmarks (New Mexico's larger chapters) made this function itself the
1423
+ // bottleneck before this fix (measured 2.8s on Chapter 30's 5.8MB PDF).
1424
+ const objEndByNum = new Map<number, number>();
1425
+ {
1426
+ const re = /(\d+)\s+0\s+obj\b/g;
1427
+ let m: RegExpExecArray | null;
1428
+ while ((m = re.exec(scan))) {
1429
+ objStarts.push({ index: m.index, num: parseInt(m[1], 10) });
1430
+ const num = parseInt(m[1], 10);
1431
+ if (!objEndByNum.has(num)) objEndByNum.set(num, m.index + m[0].length);
1432
+ }
1433
+ }
1434
+ function objNumBefore(markerIndex: number): number | null {
1435
+ let lo = 0, hi = objStarts.length - 1, ans: number | null = null;
1436
+ while (lo <= hi) {
1437
+ const mid = (lo + hi) >> 1;
1438
+ if (objStarts[mid].index <= markerIndex) { ans = objStarts[mid].num; lo = mid + 1; }
1439
+ else hi = mid - 1;
1440
+ }
1441
+ return ans;
1442
+ }
1443
+
1444
+ const dictCache = new Map<number, string | null>();
1445
+ function dictOf(n: number): string | null {
1446
+ if (dictCache.has(n)) return dictCache.get(n) ?? null;
1447
+ let start: number;
1448
+ const fastEnd = objEndByNum.get(n);
1449
+ if (fastEnd !== undefined) {
1450
+ start = fastEnd;
1451
+ } else {
1452
+ const re = new RegExp(`(?:^|\\D)${n}\\s+0\\s+obj`);
1453
+ const m = re.exec(scan);
1454
+ if (!m) { dictCache.set(n, null); return null; }
1455
+ start = m.index + m[0].length;
1456
+ }
1457
+ const end = scan.indexOf('endobj', start);
1458
+ const dict = end === -1 ? scan.slice(start, Math.min(scan.length, start + 4000)) : scan.slice(start, end);
1459
+ dictCache.set(n, dict);
1460
+ return dict;
1461
+ }
1462
+
1463
+ // Prefer the Catalog's own /Outlines reference; fall back to the first
1464
+ // direct /Type/Outlines object found anywhere (same "search for the type
1465
+ // marker as plain text" approach extractPdfText uses for Catalog/Pages).
1466
+ let outlinesRoot: number | null = null;
1467
+ const catIdx = scan.search(/\/Type\s*\/Catalog\b/);
1468
+ if (catIdx !== -1) {
1469
+ const catNum = objNumBefore(catIdx);
1470
+ const catDict = catNum !== null ? dictOf(catNum) : null;
1471
+ const om = catDict ? /\/Outlines\s+(\d+)\s+0\s+R/.exec(catDict) : null;
1472
+ if (om) outlinesRoot = parseInt(om[1], 10);
1473
+ }
1474
+ if (outlinesRoot === null) {
1475
+ const outIdx = scan.search(/\/Type\s*\/Outlines\b/);
1476
+ if (outIdx !== -1) outlinesRoot = objNumBefore(outIdx);
1477
+ }
1478
+ if (outlinesRoot === null) return [];
1479
+
1480
+ function pageObjOf(dict: string): number | null {
1481
+ const destMatch = /\/Dest\s*\[\s*(\d+)\s+0\s+R/.exec(dict);
1482
+ if (destMatch) return parseInt(destMatch[1], 10);
1483
+ const actionMatch = /\/A\s*<<[^>]*\/D\s*\[\s*(\d+)\s+0\s+R/.exec(dict);
1484
+ if (actionMatch) return parseInt(actionMatch[1], 10);
1485
+ return null;
1486
+ }
1487
+
1488
+ const entries: PdfOutlineEntry[] = [];
1489
+ const seen = new Set<number>();
1490
+ function walk(n: number, depth: number): void {
1491
+ if (depth > 20 || seen.has(n)) return; // guards a malformed/cyclic tree
1492
+ seen.add(n);
1493
+ const dict = dictOf(n);
1494
+ if (!dict) return;
1495
+ const titleMatch = /\/Title\s*\(((?:\\.|[^()])*)\)/.exec(dict);
1496
+ if (titleMatch) {
1497
+ entries.push({ title: unescapePdfLiteral(titleMatch[1]).replace(/\s+/g, ' ').trim(), pageObjNum: pageObjOf(dict), depth });
1498
+ }
1499
+ const firstMatch = /\/First\s+(\d+)\s+0\s+R/.exec(dict);
1500
+ if (firstMatch) walk(parseInt(firstMatch[1], 10), titleMatch ? depth + 1 : depth);
1501
+ const nextMatch = /\/Next\s+(\d+)\s+0\s+R/.exec(dict);
1502
+ if (nextMatch) walk(parseInt(nextMatch[1], 10), depth);
1503
+ }
1504
+ const rootDict = dictOf(outlinesRoot);
1505
+ const rootFirst = rootDict ? /\/First\s+(\d+)\s+0\s+R/.exec(rootDict) : null;
1506
+ if (rootFirst) walk(parseInt(rootFirst[1], 10), 0);
1507
+
1508
+ return entries;
1509
+ }
1510
+ /**
1511
+ * Guam Code Annotated (GCA) — section lookup and search. Fleet #2888.
1512
+ *
1513
+ * SOURCE AND SHAPE. The Compiler of Laws (Judiciary of Guam,
1514
+ * col.guamcourts.gov) publishes the GCA as ~800 per-CHAPTER PDFs, one static
1515
+ * HTML page (22 collapsible Title panels, each listing its Chapters with a
1516
+ * link to that chapter's PDF) and NOTHING lighter — no per-title listing, no
1517
+ * search API, no site-wide section index (confirmed live). That page is
1518
+ * baked ONCE into `gu-index-data.ts` (title/chapter/catchline/PDF URL) by
1519
+ * `scripts/bake-index.mjs`, the same "baked table" shape
1520
+ * mcps/north-dakota-code uses for its own structurally-identical gap.
1521
+ *
1522
+ * UNLIKE North Dakota, the GCA index page does not also enumerate every
1523
+ * SECTION. But every sampled chapter PDF opens with its own embedded table
1524
+ * of contents — a run of "§ <chapter>.<nn>. Catchline." headings — so
1525
+ * `scripts/bake-sections.mjs` fetches all ~800 chapter PDFs ONCE and bakes a
1526
+ * section-level catchline index (`gu-section-data.ts`, 15,760 rows) from
1527
+ * that TOC block. gu_search reads it; gu_statute fetches the live chapter
1528
+ * PDF fresh on every call and re-derives the section boundaries itself
1529
+ * (never trusts the baked catchline for the text it returns) — see
1530
+ * `sliceGuamSection` below.
1531
+ *
1532
+ * TWO CITATION CONVENTIONS, BOTH REAL (not a bug in either): newer,
1533
+ * recodified titles (9 GCA, the Criminal & Correctional Code, adapted from
1534
+ * the 1977 Model Penal Code) cite "chapter.section" with a dot — "9 GCA §
1535
+ * 16.30" is Aggravated Murder. Older titles (e.g. 21 GCA Ch. 33, Proof and
1536
+ * Acknowledgments of Instruments) concatenate chapter and a zero-padded
1537
+ * section number with NO separator — "21 GCA § 33101" is chapter 33,
1538
+ * section 101. gu_statute accepts either form; see `parseGuCitation`.
1539
+ *
1540
+ * THE PDF EXTRACTOR WAS PACK-LOCALLY FORKED, THEN FOLDED BACK IN (fleet
1541
+ * #2904). `shared/src/pdf-text.ts` already read Kentucky's and North
1542
+ * Dakota's statute PDFs, but Guam's chapter PDFs defeated it in ways it did
1543
+ * not yet handle, and this pack was built the same week another pack
1544
+ * (#2884, New Mexico) was independently editing that shared file — so the
1545
+ * fixes initially lived in a pack-local fork (`./gu-pdf-extract.ts`), with a
1546
+ * loud note for whoever next touched the shared extractor to fold them in.
1547
+ * That fold-in happened in #2904: all three fixes were genuinely NOT
1548
+ * Guam-specific and now live in `shared/src/pdf-text.ts` itself — (1) a
1549
+ * literal (parenthesized) string run is routed through the current font's
1550
+ * ToUnicode map exactly like a hex run, instead of being passed through
1551
+ * unchanged (which produced line-noise control characters for Guam's
1552
+ * non-WinAnsi subset fonts); (2) a `/Length` a few bytes longer than the
1553
+ * true deflate stream is retried with the tail trimmed, rather than failing
1554
+ * outright on Node/Workers' strict trailing-junk check; (3) `inflate()`'s
1555
+ * two stream writes are no longer fire-and-forget, so a malformed deflate
1556
+ * stream can no longer reject an UNHANDLED promise (this one crashed this
1557
+ * pack's own bake script outright before the fix). The fork is deleted —
1558
+ * ONE delta stayed pack-local, in `./gu-font-cmap-fix.ts`, because it
1559
+ * genuinely is Guam-specific: one embedded symbol font per chapter PDF
1560
+ * (`...+WPTypographicSymbols`) carries a `/ToUnicode` CMap that is itself
1561
+ * WRONG in Guam's own published source file (its two curly-quote codes map
1562
+ * to 'A' and '@' instead of the real glyphs) — passed into `extractPdfText`
1563
+ * via its new `fontCMapOverride` option, a generic hook with nothing
1564
+ * Guam-specific about the mechanism itself.
1565
+ *
1566
+ * FOUR CAPABILITIES — two available, two are not:
1567
+ * - citation lookup -> gu_statute
1568
+ * - topic/keyword search -> gu_search (over official section and
1569
+ * chapter catchlines, not full statutory body text — see its own
1570
+ * description)
1571
+ * - amendments/enactment history -> embedded in gu_statute's `text` field
1572
+ * as the statute's own "SOURCE:" line (e.g. "Enacted 1977 as 16.30;
1573
+ * amended by P.L. 14-125..."), the same shape Kentucky's "History:" line
1574
+ * takes. Not split out as a separate field — the GCA does not mark it
1575
+ * up distinctly enough to parse reliably across both PDF eras.
1576
+ * - historical version (a section's text as it read before an amendment)
1577
+ * -> NOT AVAILABLE. col.guamcourts.gov serves the CURRENT text only; no
1578
+ * per-year or per-session archive was found.
1579
+ *
1580
+ * Keyless, no signup.
1581
+ */
1582
+ import { guamFontCMapOverride } from './gu-font-cmap-fix.js';
1583
+ import type { GuChapterRow, GuTitleRow } from './gu-index-data.js';
1584
+ import type { GuSectionRow } from './gu-section-data.js';
1585
+
1586
+ const UA = 'pipeworx-mcp-guam-code/1.0 (+https://pipeworx.io)';
1587
+ const UPSTREAM = 'Compiler of Laws, Judiciary of Guam (col.guamcourts.gov)';
1588
+
1589
+ async function pwFetch(url: string | URL, init?: RequestInit): Promise<Response> {
1590
+ const headers = { 'User-Agent': UA, ...(init?.headers ?? {}) };
1591
+ return fetchWithTimeout(url, { ...init, headers }, UPSTREAM);
1592
+ }
1593
+
1594
+ // fleet #2888, same shape as north-dakota-code (fleet #2754): the baked
1595
+ // index module is uploaded to KV at deploy and injected as args._bakedIndex
1596
+ // on the first call into this pack, rather than value-imported here — a
1597
+ // value import would put the whole ~900KB table into every gateway
1598
+ // isolate's retained heap whether or not it ever serves a guam-code call.
1599
+ // NEVER value-import gu-index-data.ts or gu-section-data.ts here — a
1600
+ // type-only import (above) is erased at build time and is fine.
1601
+ let GU_TITLES: GuTitleRow[] = [];
1602
+ let GU_CHAPTERS: GuChapterRow[] = [];
1603
+ let GU_SECTIONS: GuSectionRow[] = [];
1604
+ let GU_INDEX_CAPTURED_AT = '';
1605
+ let GU_SECTION_INDEX_CAPTURED_AT = '';
1606
+
1607
+ function ensureIndex(args: Record<string, unknown>): void {
1608
+ if (GU_CHAPTERS.length) return;
1609
+ const idx = args._bakedIndex as Record<string, unknown> | null | undefined;
1610
+ const titles = idx?.GU_TITLES;
1611
+ const chapters = idx?.GU_CHAPTERS;
1612
+ const sections = idx?.GU_SECTIONS;
1613
+ if (
1614
+ !Array.isArray(titles) ||
1615
+ !Array.isArray(chapters) ||
1616
+ chapters.length === 0 ||
1617
+ !Array.isArray(sections) ||
1618
+ typeof idx?.GU_INDEX_CAPTURED_AT !== 'string' ||
1619
+ typeof idx?.GU_SECTION_INDEX_CAPTURED_AT !== 'string'
1620
+ ) {
1621
+ // Loud on purpose: answering from an empty table would read as "no such
1622
+ // section" / "no match", a silent wrong answer.
1623
+ throw new Error(
1624
+ `guam-code: the baked GCA index is unavailable (${idx == null ? 'not injected' : 'malformed'}). ` +
1625
+ 'The gateway loads it from KV on the first call into this pack; this is a server-side fault, not a bad citation or query — retry shortly.',
1626
+ );
1627
+ }
1628
+ GU_TITLES = titles as GuTitleRow[];
1629
+ GU_CHAPTERS = chapters as GuChapterRow[];
1630
+ GU_SECTIONS = sections as GuSectionRow[];
1631
+ GU_INDEX_CAPTURED_AT = idx.GU_INDEX_CAPTURED_AT as string;
1632
+ GU_SECTION_INDEX_CAPTURED_AT = idx.GU_SECTION_INDEX_CAPTURED_AT as string;
1633
+ }
1634
+
1635
+ /** Strip the per-page running header ("9 GCA CRIMES AND CORRECTIONS CH. 16
1636
+ * CRIMINAL HOMICIDE <pagenum>") and the "COL<digits>" document-management
1637
+ * stamp that every chapter PDF repeats on every page — both are page
1638
+ * furniture, not statutory text, and would otherwise land mid-section
1639
+ * whenever a section spans a page break (confirmed live: 9gc016.pdf splits
1640
+ * "§ 16.30. Aggravated Murder Defined." from its own body text across
1641
+ * exactly this furniture). The header is self-derived from the chapter
1642
+ * text's own first occurrence rather than reconstructed from the title/
1643
+ * chapter catchline strings, which may not match the PDF's own casing or
1644
+ * punctuation exactly. */
1645
+ function stripChapterNoise(text: string): string {
1646
+ let out = text.replace(/COL\d{6,10}/g, ' ');
1647
+ const headerMatch = /^(\d+[A-Za-z]?\s+GCA\s+.+?\s+CH\.\s+\S+\s+.+?)\s+\d+\s+/.exec(out);
1648
+ if (headerMatch) {
1649
+ const prefix = headerMatch[1].replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
1650
+ out = out.replace(new RegExp(`${prefix}\\s+\\d+\\s*`, 'g'), ' ');
1651
+ }
1652
+ return out.replace(/\s+/g, ' ').trim();
1653
+ }
1654
+
1655
+ interface ParsedCitation {
1656
+ title: number | null;
1657
+ chapter: string;
1658
+ /** The citation's own numeric suffix, in whichever style the input used:
1659
+ * dotted ("16.30") keeps the dot; concatenated ("33101") does not. */
1660
+ full: string;
1661
+ dotted: boolean;
1662
+ }
1663
+
1664
+ /** Parse free-form Guam citation input. Accepts "9 GCA § 16.30", "9 GCA
1665
+ * 16.30", "16.30" (needs `titleArg`), "21 GCA § 33101", "33101" (needs
1666
+ * `titleArg`). Returns null if no digits were found at all. The
1667
+ * chapter/section SPLIT for the no-dot (concatenated) form is genuinely
1668
+ * ambiguous from the string alone (chapter numbers vary in length) — it is
1669
+ * resolved later, against the known chapter list for the given title, not
1670
+ * here. */
1671
+ function parseGuCitation(raw: string, titleArg: number | null): ParsedCitation | null {
1672
+ let s = raw.trim();
1673
+ let title = titleArg;
1674
+ const titleMatch = /^(?:title\s+)?(\d+)\s*GCA\b\s*/i.exec(s);
1675
+ if (titleMatch) {
1676
+ title = parseInt(titleMatch[1], 10);
1677
+ s = s.slice(titleMatch[0].length);
1678
+ }
1679
+ s = s.replace(/^§+\s*/, '').trim();
1680
+ const dottedMatch = /^(\d+[A-Za-z]?)\.(\d+[A-Za-z]?)$/.exec(s);
1681
+ if (dottedMatch) {
1682
+ return { title, chapter: dottedMatch[1], full: `${dottedMatch[1]}.${dottedMatch[2]}`, dotted: true };
1683
+ }
1684
+ const concatMatch = /^(\d+[A-Za-z]?)$/.exec(s);
1685
+ if (concatMatch) {
1686
+ // Chapter unknown yet — resolved against the title's chapter list by
1687
+ // the caller (longest-known-chapter-number-prefix match).
1688
+ return { title, chapter: '', full: concatMatch[1], dotted: false };
1689
+ }
1690
+ return null;
1691
+ }
1692
+
1693
+ /** For the concatenated (no-dot) citation style, the chapter/section
1694
+ * boundary is ambiguous from the digit string alone (e.g. is "33101"
1695
+ * chapter 3 + section 3101, or chapter 33 + section 101?). Resolve it by
1696
+ * picking the LONGEST known chapter number, for the given title, that is a
1697
+ * literal prefix of the citation with a non-empty remainder — the correct
1698
+ * answer in every case sampled, and the same disambiguation the baked
1699
+ * gu-section-data.ts table independently confirms for every chapter it
1700
+ * covers. */
1701
+ function resolveConcatenatedChapter(titleIdx: number, full: string): string | null {
1702
+ let best: string | null = null;
1703
+ for (const [tIdx, chapter] of GU_CHAPTERS) {
1704
+ if (tIdx !== titleIdx) continue;
1705
+ if (full.startsWith(chapter) && full.length > chapter.length && (best === null || chapter.length > best.length)) {
1706
+ best = chapter;
1707
+ }
1708
+ }
1709
+ return best;
1710
+ }
1711
+
1712
+ function findChapter(titleIdx: number, chapter: string): { idx: number; row: GuChapterRow } | null {
1713
+ for (let i = 0; i < GU_CHAPTERS.length; i++) {
1714
+ const row = GU_CHAPTERS[i]!;
1715
+ if (row[0] === titleIdx && row[1] === chapter) return { idx: i, row };
1716
+ }
1717
+ return null;
1718
+ }
1719
+
1720
+ /** Cut one section's text out of a whole-chapter extraction. Heading
1721
+ * candidates are built from the TARGET citation itself (both the exact
1722
+ * string the caller is asking for AND a chapter-local "any section in this
1723
+ * chapter" pattern, used only to find the END boundary) rather than a
1724
+ * generic number pattern — the same defense mcps/north-dakota-code's
1725
+ * sliceNdSection uses against a cross-reference impostor, adapted for
1726
+ * Guam's two citation styles (see parseGuCitation). Among occurrences of
1727
+ * the TARGET citation, picks whichever has the most text before the next
1728
+ * heading — the same most-text-wins tie-break ND and Iowa use. */
1729
+ function sliceGuamSection(chapterText: string, chapterNum: string, targetFull: string, dotted: boolean): string | null {
1730
+ const chapterEsc = chapterNum.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
1731
+ const anyHeadingRe = dotted
1732
+ ? new RegExp(`§\\s*${chapterEsc}\\.\\d+[A-Za-z]?\\.`, 'g')
1733
+ : new RegExp(`§\\s*${chapterEsc}\\d{2,4}[A-Za-z]?\\.`, 'g');
1734
+ const targetEsc = targetFull.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
1735
+ const targetRe = new RegExp(`§\\s*${targetEsc}\\.`, 'g');
1736
+
1737
+ const allHeadings: { at: number; end: number }[] = [];
1738
+ for (const m of chapterText.matchAll(anyHeadingRe)) allHeadings.push({ at: m.index, end: m.index + m[0].length });
1739
+ allHeadings.sort((a, b) => a.at - b.at);
1740
+
1741
+ let best: { at: number; end: number } | null = null;
1742
+ let bestLen = -1;
1743
+ for (const m of chapterText.matchAll(targetRe)) {
1744
+ const headingEnd = m.index + m[0].length;
1745
+ const next = allHeadings.find((h) => h.at > m.index);
1746
+ const bodyEnd = next ? next.at : chapterText.length;
1747
+ const len = bodyEnd - headingEnd;
1748
+ if (len > bestLen) {
1749
+ bestLen = len;
1750
+ best = { at: m.index, end: bodyEnd };
1751
+ }
1752
+ }
1753
+ if (!best) return null;
1754
+ return chapterText.slice(best.at, best.end).trim();
1755
+ }
1756
+
1757
+ const tools: McpToolExport['tools'] = [
1758
+ {
1759
+ name: 'gu_statute',
1760
+ description:
1761
+ 'Get the FULL TEXT of a section of the Guam Code Annotated (GCA) — Guam territorial law — by citation. "9 GCA 16.30" (or "9 GCA § 16.30") is Aggravated Murder Defined; "21 GCA 33101" is who may take an acknowledgment of a deed. Guam uses TWO citation styles depending on the title: newer titles write "chapter.section" with a dot (9 GCA, the Criminal Code); older titles concatenate chapter and a zero-padded section number with no separator (e.g. 21 GCA). Either form works. If the citation omits "N GCA", pass `title` separately (1-22). Returns the current statutory text, the official catchline, chapter/title context, source PDF URL, and data_as_of. Guam publishes the CURRENT text only — no per-year historical archive; this tool says so rather than guessing. Keyless. Use for "what does 9 GCA 16.30 say" or to check a Guam statute a case or contract relies on.',
1762
+ inputSchema: {
1763
+ type: 'object' as const,
1764
+ properties: {
1765
+ citation: {
1766
+ type: 'string',
1767
+ description: 'Section citation, e.g. "9 GCA 16.30", "9 GCA § 16.30", "21 GCA 33101", or a bare "16.30" / "33101" with `title` given separately.',
1768
+ },
1769
+ title: {
1770
+ type: 'number',
1771
+ description: 'GCA Title number (1-22), only needed if not already included in `citation` (e.g. citation="16.30", title=9).',
1772
+ },
1773
+ },
1774
+ required: ['citation'],
1775
+ },
1776
+ },
1777
+ {
1778
+ name: 'gu_search',
1779
+ description:
1780
+ 'Search the Guam Code Annotated BY TOPIC or keyword rather than citation — "murder", "security deposit", "concealed weapon permit", "homestead". Matches against official section and chapter CATCHLINES (not full statutory body text — Guam does not publish a full-text search, and this pack derives its section catchlines from each chapter PDF\'s own table of contents, not a separate official index, so coverage is best-effort). Optionally filter to one Title (1-22) with `title`. Returns matching citations, catchlines, and title/chapter context; call gu_statute with a citation for the full text. Keyless.',
1781
+ inputSchema: {
1782
+ type: 'object' as const,
1783
+ properties: {
1784
+ query: { type: 'string', description: 'Topic or keywords, e.g. "aggravated murder".' },
1785
+ title: { type: 'number', description: 'Restrict to one GCA Title (1-22), e.g. 9 for Crimes and Corrections.' },
1786
+ limit: { type: 'number', description: 'Max results to return, default 10, max 50.' },
1787
+ },
1788
+ required: ['query'],
1789
+ },
1790
+ },
1791
+ ];
1792
+
1793
+ async function guStatute(args: Record<string, unknown>) {
1794
+ const raw = String(args.citation ?? '').trim();
1795
+ if (!raw) {
1796
+ return { found: false, reason: 'empty_citation', message: 'gu_statute needs a citation, e.g. "9 GCA 16.30" or "21 GCA 33101".' };
1797
+ }
1798
+ const titleArg = args.title !== undefined && args.title !== null ? Number(args.title) : null;
1799
+ const parsed = parseGuCitation(raw, titleArg);
1800
+ if (!parsed || !parsed.title) {
1801
+ return {
1802
+ found: false,
1803
+ reason: 'unparseable_citation',
1804
+ citation: raw,
1805
+ message: `Could not parse "${raw}" as a GCA citation. Give it as "9 GCA 16.30" / "9 GCA § 16.30" / "21 GCA 33101", or pass a bare section with a separate title, e.g. citation="16.30", title=9.`,
1806
+ };
1807
+ }
1808
+
1809
+ const titleIdx = GU_TITLES.findIndex((t) => t.title === parsed.title);
1810
+ if (titleIdx === -1) {
1811
+ return {
1812
+ found: false,
1813
+ reason: 'title_not_found',
1814
+ citation: raw,
1815
+ message: `No GCA Title ${parsed.title} (valid range is 1-22).`,
1816
+ source: `${UPSTREAM}, keyless`,
1817
+ };
1818
+ }
1819
+ const titleRow = GU_TITLES[titleIdx]!;
1820
+
1821
+ let chapterNum = parsed.chapter;
1822
+ let fullCitation = parsed.full;
1823
+ if (!parsed.dotted && !chapterNum) {
1824
+ const resolved = resolveConcatenatedChapter(titleIdx, parsed.full);
1825
+ if (!resolved) {
1826
+ return {
1827
+ found: false,
1828
+ reason: 'citation_not_found',
1829
+ citation: raw,
1830
+ title: parsed.title,
1831
+ title_name: titleRow.name,
1832
+ message: `No chapter of ${parsed.title} GCA (${titleRow.name}) has a number that is a prefix of "${parsed.full}".`,
1833
+ hint: 'Call gu_search with a topic instead if you do not have an exact citation.',
1834
+ source: `${UPSTREAM}, keyless`,
1835
+ };
1836
+ }
1837
+ chapterNum = resolved;
1838
+ }
1839
+
1840
+ const chapterHit = findChapter(titleIdx, chapterNum);
1841
+ if (!chapterHit) {
1842
+ return {
1843
+ found: false,
1844
+ reason: 'chapter_not_found',
1845
+ citation: raw,
1846
+ title: parsed.title,
1847
+ title_name: titleRow.name,
1848
+ message: `${parsed.title} GCA has no Chapter ${chapterNum}.`,
1849
+ hint: 'Call gu_search with a topic instead if you do not have an exact citation.',
1850
+ source: `${UPSTREAM}, keyless`,
1851
+ };
1852
+ }
1853
+ const [, , chapterName, pdfUrl] = chapterHit.row;
1854
+
1855
+ const pdfRes = await pwFetch(pdfUrl);
1856
+ if (!pdfRes.ok) throw await httpError(pdfRes, 'Guam Compiler of Laws chapter PDF');
1857
+ const extracted = await extractPdfText(await pdfRes.arrayBuffer(), { fontCMapOverride: guamFontCMapOverride });
1858
+ if (!extracted.text.trim()) {
1859
+ return {
1860
+ found: true,
1861
+ citation: `${parsed.title} GCA § ${fullCitation}`,
1862
+ chapter: chapterNum,
1863
+ chapter_name: chapterName,
1864
+ title: parsed.title,
1865
+ title_name: titleRow.name,
1866
+ text: null,
1867
+ reason: 'upstream_parse',
1868
+ message: 'The chapter PDF was fetched but no text could be extracted from it — an extraction defect, not evidence the section does not exist.',
1869
+ warnings: extracted.warnings,
1870
+ pdf_url: pdfUrl,
1871
+ source: `${UPSTREAM}, keyless`,
1872
+ };
1873
+ }
1874
+
1875
+ const cleaned = stripChapterNoise(extracted.text);
1876
+ const sliced = sliceGuamSection(cleaned, chapterNum, fullCitation, parsed.dotted);
1877
+ if (!sliced) {
1878
+ return {
1879
+ found: false,
1880
+ reason: 'section_not_found_in_pdf',
1881
+ citation: `${parsed.title} GCA § ${fullCitation}`,
1882
+ chapter: chapterNum,
1883
+ chapter_name: chapterName,
1884
+ message: `"${fullCitation}" was not located as a heading in Chapter ${chapterNum}'s own PDF text — it may be vacant, repealed, or renumbered.`,
1885
+ hint: 'Call gu_search with a topic, or gu_statute with a nearby citation in the same chapter, to check.',
1886
+ pdf_url: pdfUrl,
1887
+ extraction_warnings: extracted.warnings.length ? extracted.warnings : undefined,
1888
+ source: `${UPSTREAM}, keyless`,
1889
+ };
1890
+ }
1891
+
1892
+ // The catchline is the heading text itself, up to its own trailing period
1893
+ // — re-derived from THIS extraction rather than trusted from the baked
1894
+ // gu-section-data.ts table, so a stale bake never produces a mismatched
1895
+ // catchline next to fresh text.
1896
+ const catchlineMatch = /^\s*([^.]+)\./.exec(sliced.replace(/^§\s*[\d.A-Za-z]+\.\s*/, ''));
1897
+
1898
+ return {
1899
+ found: true,
1900
+ citation: `${parsed.title} GCA § ${fullCitation}`,
1901
+ catchline: catchlineMatch ? catchlineMatch[1].trim() : undefined,
1902
+ chapter: chapterNum,
1903
+ chapter_name: chapterName,
1904
+ title: parsed.title,
1905
+ title_name: titleRow.name,
1906
+ text: sliced,
1907
+ text_chars: sliced.length,
1908
+ chapter_index_captured_at: GU_INDEX_CAPTURED_AT,
1909
+ historical_versions: 'not available — col.guamcourts.gov publishes only the current online edition; no per-year or per-session archive was found.',
1910
+ extraction_warnings: extracted.warnings.length ? extracted.warnings : undefined,
1911
+ pdf_url: pdfUrl,
1912
+ data_as_of: new Date().toISOString(),
1913
+ source: `${UPSTREAM}, keyless`,
1914
+ attribution: 'Guam statutes are public record.',
1915
+ };
1916
+ }
1917
+
1918
+ interface SearchHit { citation: string; catchline: string; chapterIdx: number; score: number }
1919
+
1920
+ /** IDF-weighted word overlap over section catchlines (+ their chapter's own
1921
+ * caption, for recall when the word lives in the chapter name rather than
1922
+ * any one section's) — same scoring shape as mcps/north-dakota-code's
1923
+ * nd_search / mcps/iowa-code's ia_code_search. */
1924
+ function wordWeights(words: string[], haystacks: string[]): Map<string, number> {
1925
+ const weight = new Map<string, number>();
1926
+ for (const w of words) {
1927
+ let df = 0;
1928
+ for (const h of haystacks) if (h.includes(w)) df++;
1929
+ weight.set(w, 1 / Math.max(1, df));
1930
+ }
1931
+ return weight;
1932
+ }
1933
+
1934
+ async function guSearch(args: Record<string, unknown>) {
1935
+ const q = String(args.query ?? '').trim();
1936
+ if (!q) {
1937
+ return {
1938
+ found: false,
1939
+ reason: 'empty_query',
1940
+ message: 'gu_search needs a "query" — a topic or a few keywords, e.g. "aggravated murder" or "security deposit".',
1941
+ hint: 'To look up a known citation directly, use gu_statute instead.',
1942
+ };
1943
+ }
1944
+ const limit = Math.min(50, Math.max(1, Number(args.limit) || 10));
1945
+ const titleFilter = args.title !== undefined && args.title !== null ? Number(args.title) : null;
1946
+ const titleIdxFilter = titleFilter !== null ? GU_TITLES.findIndex((t) => t.title === titleFilter) : null;
1947
+ if (titleFilter !== null && titleIdxFilter === -1) {
1948
+ return { found: false, reason: 'title_not_found', title: titleFilter, message: `No GCA Title ${titleFilter} (valid range is 1-22).` };
1949
+ }
1950
+
1951
+ const words = q.toLowerCase().split(/\s+/).filter((w) => w.length >= 3);
1952
+ if (!words.length) {
1953
+ return { found: false, reason: 'query_too_short', query: q, message: `"${q}" has no word of 3+ characters to match against section catchlines.` };
1954
+ }
1955
+
1956
+ const candidateRows = titleIdxFilter !== null ? GU_SECTIONS.filter((r) => GU_CHAPTERS[r[2]]![0] === titleIdxFilter) : GU_SECTIONS;
1957
+ const haystacks = candidateRows.map((r) => `${r[1]} ${GU_CHAPTERS[r[2]]![2]}`.toLowerCase());
1958
+ const weight = wordWeights(words, haystacks);
1959
+
1960
+ const hits: SearchHit[] = [];
1961
+ for (let i = 0; i < candidateRows.length; i++) {
1962
+ const h = haystacks[i]!;
1963
+ let score = 0;
1964
+ for (const w of words) if (h.includes(w)) score += weight.get(w) ?? 1;
1965
+ if (score > 0) hits.push({ citation: candidateRows[i]![0], catchline: candidateRows[i]![1], chapterIdx: candidateRows[i]![2], score });
1966
+ }
1967
+ hits.sort((a, b) => b.score - a.score);
1968
+
1969
+ if (!hits.length) {
1970
+ return {
1971
+ found: false,
1972
+ query: q,
1973
+ title: titleFilter ?? undefined,
1974
+ message: `No section or chapter catchline in the Guam Code Annotated${titleFilter ? ` Title ${titleFilter}` : ''} matched "${q}".`,
1975
+ hint: 'This matches official CATCHLINES (section and chapter names), not full statutory text — try shorter or more literal terms, or drop the `title` filter.',
1976
+ sections_scanned: candidateRows.length,
1977
+ data_as_of: GU_SECTION_INDEX_CAPTURED_AT,
1978
+ };
1979
+ }
1980
+
1981
+ const results = hits.slice(0, limit).map((h) => {
1982
+ const chapterRow = GU_CHAPTERS[h.chapterIdx]!;
1983
+ const titleRow = GU_TITLES[chapterRow[0]]!;
1984
+ const dotted = h.citation.includes('.');
1985
+ return {
1986
+ citation: `${titleRow.title} GCA § ${h.citation}`,
1987
+ section: h.citation,
1988
+ catchline: h.catchline,
1989
+ chapter: chapterRow[1],
1990
+ chapter_name: chapterRow[2],
1991
+ title: titleRow.title,
1992
+ title_name: titleRow.name,
1993
+ citation_style: dotted ? 'chapter.section' : 'concatenated',
1994
+ };
1995
+ });
1996
+
1997
+ return {
1998
+ found: true,
1999
+ query: q,
2000
+ title: titleFilter ?? undefined,
2001
+ count: results.length,
2002
+ total_matched: hits.length,
2003
+ results,
2004
+ hint: 'Call gu_statute with a citation above for the full text.',
2005
+ match_level: 'official_catchline',
2006
+ source: `${UPSTREAM} GCA chapter table-of-contents index, keyless`,
2007
+ data_as_of: GU_SECTION_INDEX_CAPTURED_AT,
2008
+ };
2009
+ }
2010
+
2011
+ async function callTool(name: string, args: Record<string, unknown>): Promise<unknown> {
2012
+ ensureIndex(args);
2013
+ switch (name) {
2014
+ case 'gu_statute':
2015
+ return guStatute(args);
2016
+ case 'gu_search':
2017
+ return guSearch(args);
2018
+ default:
2019
+ throw new Error(`Unknown tool: ${name}`);
2020
+ }
2021
+ }
2022
+
2023
+ export default { tools, callTool, meter: { credits: 1 } } satisfies McpToolExport;