@pipeworx/mcp-openaq 0.1.3 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/index.ts DELETED
@@ -1,1646 +0,0 @@
1
- interface McpToolDefinition {
2
- name: string;
3
- description: string;
4
- /** Human-facing one-liner (fleet #1967). Optional; consumers fall back to
5
- * description. Kept in step with shared/src/types.ts — scripts/lib/
6
- * check-inlined-types.mjs reports drift at publish time. */
7
- summary?: string;
8
- inputSchema: {
9
- type: 'object';
10
- properties: Record<string, unknown>;
11
- required?: string[];
12
- anyOf?: Array<{ required: string[] }>;
13
- oneOf?: Array<{ required: string[] }>;
14
- allOf?: Array<{ required: string[] }>;
15
- };
16
- outputSchema?: Record<string, unknown>;
17
- }
18
-
19
- interface McpToolExport {
20
- tools: McpToolDefinition[];
21
- callTool: (name: string, args: Record<string, unknown>) => Promise<unknown>;
22
- meter?: { credits: number };
23
- cost?: Record<string, unknown>;
24
- provider?: string;
25
- }
26
-
27
- /**
28
- * Place name -> coordinates, ranked so the answer is the city the asker meant.
29
- *
30
- * Extracted 2026-08-04 after the SAME defect was found in four packs in three
31
- * days (weather, open-meteo, openaq, openchargemap). Every one of them called
32
- * Open-Meteo's geocoder with `count=1` (or took `results[0]`) under a comment
33
- * asserting the upstream ranks by relevance and population. **That comment is
34
- * false.** Open-Meteo returns exact-name matches in no useful order, so a
35
- * 3,218-person Romanian village named Roma outranks Rome.
36
- *
37
- * Two things make this correct, and the second is the one that surprises people:
38
- *
39
- * 1. RANK BY POPULATION, with a feature-code boost to break ties when the
40
- * population field is thin or zero.
41
- * 2. RETRY IN OTHER LANGUAGES. Sorting alone is NOT enough. Verified live:
42
- * for `name=Roma&language=en`, Rome is ABSENT from the result set entirely
43
- * (the two Roma/IT rows are population 87 and 33), so a population sort
44
- * moves the answer from Roma, Romania to Roma, LESOTHO — still wrong.
45
- * `language=it` returns Roma, IT at 2,318,895. Endonyms live under the
46
- * local language, so when the English answer looks like nowhere we widen.
47
- *
48
- * The extra requests only happen when the first answer looks weak, so the
49
- * common case ("Chicago") still costs exactly one call.
50
- */
51
-
52
- const GEOCODE_URL$shared = 'https://geocoding-api.open-meteo.com/v1/search';
53
-
54
- /** Languages tried when the English answer looks like nowhere. Ordered by how
55
- * often a major world city hides behind its endonym (Roma, München, Köln,
56
- * Sevilla, Lisboa). */
57
- const EXONYM_LANGS = ['it', 'es', 'de', 'fr', 'pt'] as const;
58
-
59
- /** Population is the primary signal. These only lift real administrative places
60
- * above same-named villages when population data is thin or zero.
61
- * PPLC = national capital, PPLA = first-order admin seat, and so on.
62
- *
63
- * Fleet #111 (2026-08-06): these were 50x too big — PPLA's flat +1,000,000
64
- * outweighed San Francisco CA's real 811K-person lead over San Francisco,
65
- * Morazan, El Salvador (pop 16,152), so the department capital won on
66
- * feature-code alone. Scaled down so the boost can still break a genuine
67
- * thin/zero-population tie (a real capital vs an unrelated small hamlet)
68
- * but can never overturn a population gap in the tens-of-thousands or up —
69
- * which is every case that actually distinguishes a major city from a
70
- * same-named foreign administrative seat. */
71
- const FEATURE_BOOST: Record<string, number> = {
72
- PPLC: 100_000, PPLA: 20_000, PPLA2: 4_000, PPLA3: 800,
73
- };
74
-
75
- interface GeoHit {
76
- name?: string;
77
- admin1?: string;
78
- country?: string;
79
- country_code?: string;
80
- latitude: number;
81
- longitude: number;
82
- population?: number;
83
- feature_code?: string;
84
- }
85
-
86
- interface ResolvedPlace$shared {
87
- name: string;
88
- admin1: string | null;
89
- /** Full country name (e.g. "United States") as Open-Meteo returns it —
90
- * added migrating mcps/weather onto this module, which surfaces the full
91
- * name in tool output and keys a US-vs-not check off it (fleet #74). */
92
- country: string | null;
93
- country_code: string | null;
94
- latitude: number;
95
- longitude: number;
96
- population: number | null;
97
- /** What we searched to find it — surfaced so a caller can SEE that "Roma"
98
- * was resolved via Italian, rather than wonder why it worked. */
99
- matched_language: string;
100
- }
101
-
102
- function geoScore(r: GeoHit): number {
103
- return (r.population ?? 0) + (FEATURE_BOOST[r.feature_code ?? ''] ?? 0);
104
- }
105
-
106
- function bestOf(results: GeoHit[]): GeoHit | undefined {
107
- return results.length ? results.reduce((a, b) => (geoScore(b) > geoScore(a) ? b : a)) : undefined;
108
- }
109
-
110
- /**
111
- * "No real city found here."
112
- *
113
- * Deliberately reads RAW population, not geoScore. The feature boost exists to
114
- * break ties during ranking; letting it feed this decision made a 3,218-person
115
- * PPLA2 town in Romania look strong enough to suppress the very fan-out that
116
- * finds Rome. Boost ranks; population decides whether to keep looking.
117
- */
118
- function looksWeak(hit: GeoHit | undefined): boolean {
119
- return !hit || (hit.population ?? 0) < 50_000;
120
- }
121
-
122
- async function search(name: string, language: string, count: number): Promise<GeoHit[]> {
123
- const url = `${GEOCODE_URL$shared}?name=${encodeURIComponent(name)}&count=${count}&language=${language}&format=json`;
124
- const res = await fetch(url, { headers: { Accept: 'application/json' } });
125
- if (!res.ok) throw new Error(`Geocoding service error (HTTP ${res.status}) resolving "${name}".`);
126
- const data = (await res.json()) as { results?: GeoHit[] };
127
- return (data.results ?? []).filter(
128
- (r) => typeof r?.latitude === 'number' && typeof r?.longitude === 'number',
129
- );
130
- }
131
-
132
- function toResolved(hit: GeoHit, fallbackName: string, lang: string): ResolvedPlace$shared {
133
- return {
134
- name: hit.name ?? fallbackName,
135
- admin1: hit.admin1 ?? null,
136
- country: hit.country ?? null,
137
- country_code: hit.country_code ?? null,
138
- latitude: hit.latitude,
139
- longitude: hit.longitude,
140
- population: typeof hit.population === 'number' ? hit.population : null,
141
- matched_language: lang,
142
- };
143
- }
144
-
145
- /**
146
- * Resolve one place name to the most plausible real city.
147
- *
148
- * `language` is the caller's preferred language; other languages are only tried
149
- * when the best hit so far looks weak.
150
- */
151
- async function resolvePlace(
152
- name: string,
153
- opts: { language?: string; count?: number } = {},
154
- ): Promise<ResolvedPlace$shared | null> {
155
- const lang = opts.language ?? 'en';
156
- // At least 10 candidates: with count=1 there is nothing to rank, which is how
157
- // every instance of this bug shipped.
158
- const count = Math.max(10, opts.count ?? 10);
159
-
160
- const seen = new Map<string, string>(); // hit key -> language it came from
161
- const key = (h: GeoHit) => `${h.latitude},${h.longitude}`;
162
- let pool: GeoHit[] = [];
163
- const absorb = (hits: GeoHit[], viaLang: string) => {
164
- for (const h of hits) if (!seen.has(key(h))) { seen.set(key(h), viaLang); pool.push(h); }
165
- };
166
-
167
- absorb(await search(name, lang, count), lang);
168
- if (!pool.length && lang !== 'en') absorb(await search(name, 'en', count), 'en');
169
-
170
- if (looksWeak(bestOf(pool))) {
171
- for (const alt of EXONYM_LANGS) {
172
- if (alt === lang) continue;
173
- absorb(await search(name, alt, count), alt);
174
- if (!looksWeak(bestOf(pool))) break;
175
- }
176
- }
177
-
178
- let hit = bestOf(pool);
179
- // Open-Meteo matches on the bare place name and returns nothing for
180
- // "City, ST" / "City, State" — but routers routinely produce exactly that
181
- // ("Des Moines, IA"). Retry with the city part before giving up.
182
- if (!hit && name.includes(',')) {
183
- const city = name.split(',')[0]!.trim();
184
- if (city && city.toLowerCase() !== name.toLowerCase()) {
185
- const retry = await search(city, 'en', count);
186
- absorb(retry, 'en');
187
- hit = bestOf(retry);
188
- }
189
- }
190
- return hit ? toResolved(hit, name, seen.get(key(hit)) ?? lang) : null;
191
- }
192
-
193
- /**
194
- * Ranked RAW hits, best first — for packs that must keep the upstream's own
195
- * response shape (open-meteo's `geocode` returns Open-Meteo objects verbatim,
196
- * and normalising them would be a breaking change for its callers).
197
- *
198
- * Same two-part algorithm as resolvePlace: rank by population+boost, and widen
199
- * across endonym languages only when the best hit still looks weak.
200
- */
201
- async function rankedPlaceHits(
202
- name: string,
203
- opts: { language?: string; count?: number } = {},
204
- ): Promise<GeoHit[]> {
205
- const lang = opts.language ?? 'en';
206
- const count = Math.max(10, opts.count ?? 10);
207
- const seen = new Set<string>();
208
- const pool: GeoHit[] = [];
209
- const key = (h: GeoHit) => `${h.latitude},${h.longitude}`;
210
- const absorb = (hits: GeoHit[]) => {
211
- for (const h of hits) if (!seen.has(key(h))) { seen.add(key(h)); pool.push(h); }
212
- };
213
-
214
- absorb(await search(name, lang, count));
215
- if (!pool.length && lang !== 'en') absorb(await search(name, 'en', count));
216
- if (looksWeak(bestOf(pool))) {
217
- for (const alt of EXONYM_LANGS) {
218
- if (alt === lang) continue;
219
- absorb(await search(name, alt, count));
220
- if (!looksWeak(bestOf(pool))) break;
221
- }
222
- }
223
- if (!pool.length && name.includes(',')) {
224
- const city = name.split(',')[0]!.trim();
225
- if (city && city.toLowerCase() !== name.toLowerCase()) absorb(await search(city, 'en', count));
226
- }
227
- return pool.sort((a, b) => geoScore(b) - geoScore(a));
228
- }
229
-
230
- /** Ranked candidates, best first — for packs that offer alternatives. */
231
- async function resolvePlaceCandidates(
232
- name: string,
233
- opts: { language?: string; count?: number; limit?: number } = {},
234
- ): Promise<ResolvedPlace$shared[]> {
235
- const best = await resolvePlace(name, opts);
236
- if (!best) return [];
237
- const lang = best.matched_language;
238
- const hits = await search(name, lang, Math.max(10, opts.count ?? 10));
239
- const ranked = [...hits].sort((a, b) => geoScore(b) - geoScore(a));
240
- const out = ranked.map((h) => toResolved(h, name, lang));
241
- // Guarantee the chosen answer leads, even if it came from a different
242
- // language pass than the one we just re-queried.
243
- if (!out.length || out[0]!.latitude !== best.latitude || out[0]!.longitude !== best.longitude) {
244
- out.unshift(best);
245
- }
246
- return out.slice(0, opts.limit ?? 5);
247
- }
248
-
249
-
250
- /**
251
- * Was this failure OUR OWN web service? — the other half of `internal-db-class.ts`.
252
- *
253
- * fleet #1089 pulled failures from our own Postgres out of `upstream_down` by
254
- * keying on the SQLSTATE inside PostgREST's four-key error envelope. That
255
- * covered the majority and structurally could not cover the rest: the rest
256
- * never reach Postgres, so they carry no SQLSTATE. What was left, measured over
257
- * the 24h to 2026-09-02T15:00Z (fleet #1096):
258
- *
259
- * 5 pipeworx-catalog get_pack_tools Pipeworx catalog error: 522 — error code: 522
260
- * 3 fleet fleet_list_open … upstream_down: Fleet task queue did not respond within 25s
261
- *
262
- * 521/522/523/526 are Cloudflare saying its edge could not reach an ORIGIN, and
263
- * in both of those rows the origin is ours — `gateway.pipeworx.io` for the
264
- * catalog pack (it self-fetches when the gateway hasn't injected a manifest),
265
- * our own Supabase for fleet. There is no third party anywhere in either call.
266
- * Same defect as #1089: our own outage filed under `upstream_down`, the one
267
- * class that means "the source is unreachable and there is nothing for us to
268
- * fix", which is why the problem-tools triage skips it.
269
- *
270
- * WHY NOT A WORDING RULE. The obvious fix is to match `fleet db error:` and
271
- * `Pipeworx catalog error:` in classifyToolError. Each is emitted from exactly
272
- * one site today, so it would work today. It would also rot the first time
273
- * somebody rewords a label — silently, and in the direction of hiding our own
274
- * outage, which is worse than the bug being fixed. Every prose rule in
275
- * error-class.ts has needed widening as packs invented new wording (#409/#450/
276
- * #584); that history is most of that file's comment budget.
277
- *
278
- * WHAT THIS KEYS ON INSTEAD: **the host the call actually reached.** A URL's
279
- * hostname is a fact about the call, not a guess about its prose. Two
280
- * consequences that a pack-level flag could not give us, and the reason the
281
- * flag was rejected:
282
- *
283
- * - It describes the CALL, not the pack. `govcon-intel` fans out to our own
284
- * Supabase AND to genuine third parties; `court-listener` holds our cache
285
- * in Supabase and fetches courtlistener.com. An `internallyHosted: true` on
286
- * either pack would relabel a real third-party outage as ours — inventing
287
- * work, which is the same class of error in the opposite direction.
288
- * - It covers every future internal pack for free, instead of one declared
289
- * slug at a time.
290
- *
291
- * WHY IT SURVIVES A REWORD. The marker below is not matched as a literal by two
292
- * separate files. `markInternalOrigin()` writes it and `internalHostMetricsClass()`
293
- * reads it, both from the single exported `INTERNAL_ORIGIN_MARKER` constant in
294
- * this module — so changing the wording changes both sides in the same edit and
295
- * cannot desynchronise them. The pack's own label (`fleet db error:`,
296
- * `Pipeworx catalog error:`) is not read at all: reword it freely, the class is
297
- * unaffected. That is the property `stripClassPrefix` lacked when it drifted
298
- * from its own classifier three times and needed a CI gate to hold them
299
- * together.
300
- *
301
- * WHERE THE 5xx TEST LIVES. `markInternalOrigin` is called from the places that
302
- * hold the real `Response` — `httpError`/`httpErrorMessage` and the timeout
303
- * branch of `fetchWithTimeout` in `shared/src/http.ts` — so "is this an
304
- * availability failure" is decided from the actual status code, never re-derived
305
- * by scraping a number out of a sentence. A 404 from our own registry for a slug
306
- * that does not exist is a caller's bad argument and is deliberately NOT marked.
307
- */
308
-
309
- /**
310
- * OUR OWN web service was unreachable — not an upstream, and never `upstream_down`.
311
- *
312
- * ONE value, not three, unlike `internal_db_*`. That split existed because a
313
- * slow query, an exhausted pool and an unknown SQLSTATE have different owners
314
- * and different fixes. Here there is only one story to tell — an origin we run
315
- * did not answer the edge — and one owner. A bucket with no distinct owner per
316
- * value is decoration; #724 is what happens when a class holds several
317
- * situations, and inventing sub-values ahead of a reason to act on them
318
- * differently is the same mistake with the sign flipped.
319
- *
320
- * METRICS ONLY, exactly like PLATFORM_KEY_ERROR_CLASS and the internal_db
321
- * values. `classifyToolError` still answers `upstream_down` for the retry and
322
- * hint paths, which only care whether retrying or a sibling tool might work —
323
- * and it might. Nothing a caller sees or is charged changes here.
324
- *
325
- * READ SIDE: this value is in BROKEN_TOOL_CLASSES, FAULT_CLASSES and
326
- * ALL_ERROR_CLASSES in `workers/registry-api/src/index.ts`. All three, or it
327
- * lands on no dashboard — fleet #721 is the warning, where the #719 split
328
- * worked on the write side and was invisible for weeks.
329
- */
330
- const INTERNAL_SERVICE_UNREACHABLE_CLASS = 'internal_service_unreachable';
331
-
332
- /**
333
- * The token that carries "this origin is ours" from the call site to the
334
- * classifier.
335
- *
336
- * Appended to the error message rather than attached to the Error object,
337
- * because the object does not survive the trip: 275 packs return `{ error:
338
- * string }` instead of throwing, the gateway reads `observedError` as a string,
339
- * and the fleet pack rebuilds its error from a captured status + body across a
340
- * retry loop. A property on an Error would be dropped by every one of those
341
- * paths and the class would work in tests and vanish in production.
342
- *
343
- * WORDING IS LOAD-BEARING, same rule as labelAge's note in authority.ts. This
344
- * string is appended to a pack's thrown Error message (shared/src/http.ts),
345
- * and a thrown Error's message is exactly what the gateway hands back to the
346
- * caller as `content[0].text` when nothing rewrites it (workers/gateway/src
347
- * catches the throw and sets `rawResult.message = stripClassPrefix(error)`,
348
- * which does not touch this suffix) — so the original wording,
349
- * " [pipeworx-hosted origin — our own service, not a third party]", was not a
350
- * theoretical leak: it shipped live on pipeworx-catalog's 522s, 7 times in 6
351
- * hours on 2026-09-02 (see tests/golden-internal-service.test.ts), verbatim
352
- * naming Pipeworx as the host. check:hosting-claims never caught it because it
353
- * did not scan shared/ at all (task #2009). Reworded to describe the
354
- * OBSERVATION (the origin did not answer) without a claim about who runs it —
355
- * the identical fix labelAge got: drop the possessive, keep the fact.
356
- */
357
- const INTERNAL_ORIGIN_MARKER = ' [origin did not respond — retry before concluding the named source is down]';
358
-
359
- /**
360
- * Supabase's data plane for a project is `<ref>.supabase.co`, where the ref is
361
- * exactly twenty lowercase letters (ours is `pqauisounztsgdgfkhke`).
362
- *
363
- * Matching the shape rather than listing the ref keeps this correct when we add
364
- * a project — `supabaseEnv` on a pack entry already points some packs at a
365
- * second one — while still excluding `status.supabase.co`, which is Supabase's
366
- * own status page and emphatically not our database. Verified 2026-09-02 by
367
- * `grep -rhoE '[a-z0-9-]+\.supabase\.(co|in)' mcps shared workers scripts`: the
368
- * only real project ref anywhere in the tree is ours, the rest are doc
369
- * placeholders (`abc`, `xyz`, `example`) which this pattern also excludes. Same
370
- * finding internal-db-class.ts relies on for the PostgREST envelope being ours
371
- * by construction.
372
- */
373
- const SUPABASE_PROJECT_HOST = /^[a-z]{20}\.supabase\.(co|in)$/;
374
-
375
- /**
376
- * Is this a host WE run?
377
- *
378
- * Deliberately NOT including `*.workers.dev`: plenty of third-party APIs are
379
- * hosted on workers.dev, so the suffix says where something runs and not who
380
- * owns it. Every internal call we actually make goes to a `pipeworx.io`
381
- * hostname or to our Supabase project, both of which are ownership facts.
382
- *
383
- * `workers/gateway/src/provenance.ts`'s `OUR_HOSTS` answers the same
384
- * question and DOES include `workers.dev` — a documented divergence
385
- * (task #2051), not a bug to converge. That list decides what a response may
386
- * cite as a data SOURCE, where a false negative (citing our own worker as an
387
- * external source) is the hosting-disclosure leak this whole file exists to
388
- * prevent, so it errs broad. This one decides who gets BLAMED for a 5xx in
389
- * outage metrics read by on-call, where a false positive (crediting our own
390
- * infra with a third party's outage) hides the real failure, so it errs
391
- * narrow. Same suffix, opposite direction, because they are never called for
392
- * the same reason.
393
- *
394
- * Returns false on anything unparseable rather than throwing — this runs inside
395
- * an error path, and an error path that can itself throw turns a diagnosable
396
- * failure into a mystery.
397
- */
398
- function isPipeworxOrigin(url: string | URL | undefined | null): boolean {
399
- if (!url) return false;
400
- let host: string;
401
- try {
402
- host = new URL(url instanceof URL ? url.href : url).hostname.toLowerCase();
403
- } catch {
404
- return false;
405
- }
406
- if (host === 'pipeworx.io' || host.endsWith('.pipeworx.io')) return true;
407
- return SUPABASE_PROJECT_HOST.test(host);
408
- }
409
-
410
- /**
411
- * Append the marker when this failure was OUR origin failing to answer.
412
- *
413
- * `status` is the HTTP status when there is one, and omitted for a timeout —
414
- * where there is no response at all, and "the origin did not answer" is the
415
- * whole observation. Statuses below 500 are left alone: a 404 from our own
416
- * registry for a slug that does not exist is the caller's argument, not our
417
- * outage, and marking it would put ordinary 404s on the incident dashboard.
418
- *
419
- * Idempotent, so a message that is wrapped and re-marked on the way up (the
420
- * fleet pack's retry loop re-throws through two layers) carries the marker once.
421
- */
422
- function markInternalOrigin(
423
- message: string,
424
- url: string | URL | undefined | null,
425
- status?: number,
426
- ): string {
427
- if (status !== undefined && status < 500) return message;
428
- if (!isPipeworxOrigin(url)) return message;
429
- if (message.includes(INTERNAL_ORIGIN_MARKER)) return message;
430
- return message + INTERNAL_ORIGIN_MARKER;
431
- }
432
-
433
- /**
434
- * Which blob4 value a failure from our own web services books as, or undefined
435
- * if this is not one.
436
- *
437
- * Ordered AFTER `internalDbMetricsClass` at the call site: a PostgREST envelope
438
- * from our own Supabase is a strictly more specific statement about the same
439
- * row (which of our services, and why), and the two cannot disagree about
440
- * whether the failure is ours.
441
- */
442
- function internalHostMetricsClass(error: string): string | undefined {
443
- return error.includes(INTERNAL_ORIGIN_MARKER) ? INTERNAL_SERVICE_UNREACHABLE_CLASS : undefined;
444
- }
445
-
446
-
447
- /**
448
- * One place to turn a failed `fetch` into an error a caller can act on.
449
- *
450
- * Nearly every pack was written the same way:
451
- *
452
- * if (!res.ok) throw new Error(`Unsplash: ${res.status}`);
453
- *
454
- * which discards the response body — and the body is usually where the upstream
455
- * says what was actually wrong ("**symbol** not found: GBP", "parameter `year`
456
- * out of range", "unknown taxonomy id"). The caller gets a number, cannot
457
- * self-correct, and retries the same broken call. A 2026-07-31 sweep found this
458
- * shape in 481 of 1,400 packs, 47 of them PLATFORM-keyed.
459
- *
460
- * It also hides bugs one level down. Two of the first three packs audited had a
461
- * second defect that only existed because of this line: unsplash's rate-limit
462
- * branch sat BELOW a catch-all and was unreachable, and bea-gov parsed
463
- * `BEAAPI.Error.APIErrorDescription` below a `!res.ok` throw that made the
464
- * parsing dead code for every non-200.
465
- *
466
- * DELIBERATELY NOT A CLASSIFIER. It does not add `user_error:` /
467
- * `upstream_down:` prefixes. Those decide which tier a failure lands in, and the
468
- * `error` tier is what the daily problem-tools list is built from — it means
469
- * "Pipeworx has a defect". A 400 is genuinely ambiguous: often a caller's bad
470
- * argument, but sometimes a query WE built wrong (ted-eu comma-joined its CPV
471
- * values into something TED rejected, and that bug was found only because it sat
472
- * in `error`). Blanket-classifying 400s as caller mistakes would have hidden it.
473
- * A pack that KNOWS which it is should keep saying so explicitly; this helper is
474
- * for the 481 that say nothing at all.
475
- */
476
-
477
- /** Longest upstream explanation we'll pass through. Enough for a real message,
478
- * short enough that an HTML page or a stack trace can't swamp the error. */
479
-
480
- const MAX_DETAIL = 300;
481
-
482
- /**
483
- * Default bound for `fetchWithTimeout` when a pack doesn't state its own.
484
- *
485
- * 25s mirrors the number `epo-ops` landed on after measuring the real failure:
486
- * a degraded upstream that doesn't error, it just never answers, and a Worker
487
- * sits in `await fetch()` until ITS OWN execution budget kills the request —
488
- * which can take minutes, not seconds (epo_ops_search_patents measured 4-8
489
- * MINUTE hangs before this existed). 25s is short enough that a caller gets a
490
- * fast, actionable error instead of holding the connection, and long enough
491
- * that it doesn't false-trip on a merely-slow-but-alive upstream.
492
- */
493
- const DEFAULT_FETCH_TIMEOUT_MS = 25_000;
494
-
495
- /**
496
- * Read the body of a failed response and fold it into a throwable Error.
497
- *
498
- * Usage — note the `await`, which is the one thing that makes this a mechanical
499
- * change rather than a drop-in:
500
- *
501
- * if (!res.ok) throw await httpError(res, 'Unsplash');
502
- *
503
- * Safe to call on any non-ok response: a body that is missing, empty, unreadable
504
- * or HTML degrades to exactly the old `Name: 404` string rather than throwing
505
- * something new from inside the error path.
506
- */
507
- async function httpError(res: Response, name: string): Promise<Error> {
508
- return new Error(await httpErrorMessage(res, name));
509
- }
510
-
511
- /** The message text without constructing an Error — for packs that need to wrap
512
- * it in their own envelope or add an explicit classification prefix. */
513
- async function httpErrorMessage(res: Response, name: string): Promise<string> {
514
- // The one place a 5xx from a host WE run gets stamped as ours. `res.url` is
515
- // the URL the fetch actually resolved to (after redirects), so this is a fact
516
- // about the call rather than a guess from the `name` the pack passed in —
517
- // reword that label freely, the class does not move. See
518
- // internal-host-class.ts; no-op for every third-party upstream, which is why
519
- // this touches 481 packs' error text and changes none of it.
520
- return markInternalOrigin(
521
- `${name}: ${res.status}${detailSuffix(await readDetail(res))}${responseHints(res)}`,
522
- res.url,
523
- res.status,
524
- );
525
- }
526
-
527
- /**
528
- * Facts about a FAILED response that live in its headers, not its body, appended
529
- * so the gateway's classifier (which only ever sees this string) can tell a limit
530
- * from a wall from a refusal (fleet #3477). Both are standard, both are absent on
531
- * almost every response, and neither is invented: `Retry-After` is the server
532
- * saying "come back later" (the one thing that makes a 403 a throttle), and
533
- * Cloudflare marks a bot challenge with `cf-mitigated: challenge`.
534
- */
535
- function responseHints(res: Response): string {
536
- const hints: string[] = [];
537
- const retryAfter = res.headers.get('retry-after');
538
- if (retryAfter) hints.push(`retry-after: ${retryAfter.slice(0, 40)}`);
539
- if ((res.headers.get('cf-mitigated') ?? '').toLowerCase() === 'challenge') hints.push('Cloudflare bot challenge (cf-mitigated: challenge)');
540
- return hints.length ? ` (${hints.join('; ')})` : '';
541
- }
542
-
543
- /**
544
- * Just the upstream's own explanation — no name, no status.
545
- *
546
- * For a pack that has already said both in its own sentence. epo-ops reads
547
- * `EPO rejected this search as too large (HTTP 413) — ${httpErrorMessage(…)}`,
548
- * which rendered as `… (HTTP 413) — EPO: 413.` once the XML detail was being
549
- * dropped: the upstream named twice, the status twice, and the one thing EPO
550
- * actually said ("Not enough characters before truncation character") nowhere
551
- * (fleet #712). Returns '' when the body carries nothing readable, so a caller
552
- * can fall back to its own wording.
553
- */
554
- async function upstreamDetail(res: Response): Promise<string> {
555
- return readDetail(res);
556
- }
557
-
558
- /**
559
- * Read a SUCCESSFUL response as JSON, failing loudly when it isn't JSON.
560
- *
561
- * `httpError` above only ever runs on `!res.ok`, which leaves the nastier half
562
- * of the problem unhandled: an upstream that answers **HTTP 200 with an HTML
563
- * page**. A bot wall, a login redirect, a maintenance interstitial and a CDN
564
- * error page are all 200s, so `res.ok` is true, and `res.json()` then throws
565
- * `Unexpected token '<', "<!DOCTYPE "... is not valid JSON`.
566
- *
567
- * That string is the problem. It names no upstream, carries no status, and
568
- * reads like a parser bug in Pipeworx — so it lands in the `error` tier, which
569
- * means "we have a defect", and the caller is told nothing they can act on.
570
- * data.govt.nz sat dead behind an Imperva challenge this way and every
571
- * status-code health check we own reported it green (7889a845). A zero-length
572
- * body has the same shape: `Unexpected end of JSON input`, seen this week on
573
- * uk-gazette (83% of external calls) and census.
574
- *
575
- * UNLIKE `httpError`, this one DOES classify, and the asymmetry is deliberate.
576
- * A 400 is genuinely ambiguous — often the caller's bad argument, sometimes a
577
- * query we built wrong — so blanket-classifying it would hide our own bugs.
578
- * There is no such ambiguity here: **no argument a caller can pass makes a JSON
579
- * API return an HTML page.** It is always the upstream, so `upstream_down:` is
580
- * a statement of fact rather than a guess, and it keeps these out of the
581
- * problem-tools list where they crowd out real defects.
582
- *
583
- * const data = await parseJson<Feed>(res, 'UK Gazette');
584
- *
585
- * Call it only after the `!res.ok` check — on a failed response you want
586
- * `httpError`, which mines the body for the upstream's own explanation.
587
- */
588
- async function parseJson<T>(res: Response, name: string): Promise<T> {
589
- let raw: string;
590
- try {
591
- raw = await res.text();
592
- } catch {
593
- throw new Error(
594
- `upstream_down: ${name} returned a body that could not be read (HTTP ${res.status}). ` +
595
- 'The connection most likely dropped mid-response; retrying is reasonable.',
596
- );
597
- }
598
-
599
- const type = res.headers.get('content-type') ?? 'no content-type';
600
-
601
- if (!raw.trim()) {
602
- throw new Error(
603
- `upstream_down: ${name} answered HTTP ${res.status} with an EMPTY body where JSON was expected (${type}). ` +
604
- 'Nothing about the request can cause this — it is an upstream fault, and the same call may well work on retry.',
605
- );
606
- }
607
-
608
- // Checked before parsing rather than in the catch, because knowing it is
609
- // markup is what turns "we failed to parse something" into "they served a
610
- // web page" — the second is diagnosable, the first is not.
611
- const head = raw.slice(0, 200).trimStart().toLowerCase();
612
- if (head.startsWith('<!doctype') || head.startsWith('<html') || head.startsWith('<?xml')) {
613
- const kind = head.startsWith('<?xml') ? 'an XML document' : 'an HTML page';
614
- // The summary, not the source. Pasting the first 120 characters of a web
615
- // page handed the agent `<!DOCTYPE html><html lang="en"…` — the same leak
616
- // this branch exists to describe (fleet #712).
617
- throw new Error(
618
- `upstream_down: ${name} answered HTTP ${res.status} with ${kind} instead of JSON (${type}). ` +
619
- 'That is typically a bot wall, a login redirect or a maintenance page — it is returned as a SUCCESS, ' +
620
- `so status-code health checks read it as fine. No argument change will get past it. ` +
621
- `The page says: ${summarizeErrorBody(raw) || 'nothing readable'}`,
622
- );
623
- }
624
-
625
- try {
626
- return JSON.parse(raw) as T;
627
- } catch {
628
- throw new Error(
629
- `upstream_down: ${name} answered HTTP ${res.status} with a body that is not valid JSON (${type}). ` +
630
- `It begins: ${stripMarkup(raw).slice(0, 120) || '(unreadable)'}`,
631
- );
632
- }
633
- }
634
-
635
- /**
636
- * `fetch`, but bounded — the fix for a systemic gap found 2026-08-30: a grep
637
- * audit of every pack's `mcps/*\/src/index.ts` found 1,339 of ~1,500 call
638
- * `fetch()` with NO timeout guard anywhere in the file. Two of those
639
- * (epo-ops, statcan) were confirmed live-hanging for 4-8 minutes before this
640
- * existed — every unguarded call carries the same risk, just unconfirmed.
641
- *
642
- * Mirrors the `epoFetch` wrapper `mcps/epo-ops/src/index.ts` shipped first:
643
- * bound the request with `AbortSignal.timeout`, and on a timeout/abort throw
644
- * an `upstream_down:` error that names the upstream and the bound rather than
645
- * letting the raw `TimeoutError`/`AbortError` (which names neither) propagate.
646
- * `upstream_down:` is deliberate, same reasoning as `parseJson` above — no
647
- * argument a caller passes can make an upstream hang, so it is always the
648
- * upstream's fault, and marking it that way keeps a slow API off the
649
- * problem-tools list where it would crowd out our own defects.
650
- *
651
- * Usage — a mechanical swap for a bare `fetch(url, init)`:
652
- *
653
- * const res = await fetchWithTimeout(url, init, 'Some API');
654
- *
655
- * Pass `timeoutMs` as a fourth argument to override the default for a pack
656
- * with a known-slower upstream; the label should be the same short name you'd
657
- * pass to `httpError`/`httpErrorMessage` for that call.
658
- */
659
- async function fetchWithTimeout(
660
- url: string | URL,
661
- init: RequestInit = {},
662
- name: string,
663
- timeoutMs: number = DEFAULT_FETCH_TIMEOUT_MS,
664
- ): Promise<Response> {
665
- try {
666
- return await fetch(url, { ...init, signal: AbortSignal.timeout(timeoutMs) });
667
- } catch (err) {
668
- if (err instanceof Error && (err.name === 'TimeoutError' || err.name === 'AbortError')) {
669
- // States the OBSERVATION (no response in N seconds), not a diagnosis.
670
- // "appears to be degraded" is an inference about the vendor that we have
671
- // not checked, and it is wrong in a way that misdirects whoever reads it:
672
- // a timeout from a Worker can equally mean OUR egress is blocked.
673
- //
674
- // Measured today (2026-09-01, fleet #1047): every call to
675
- // mainnet.base.org failed from the x402 facilitator while the identical
676
- // request from a laptop returned 200. Base was entirely healthy; the
677
- // public RPC refuses Cloudflare Worker egress. Had this message fired
678
- // there it would have blamed Base by name, and the next person would have
679
- // waited for a vendor outage to clear that did not exist.
680
- // A timeout has no status to test — there is no response at all — so
681
- // `markInternalOrigin` is called without one: an origin we run that never
682
- // answered is an availability failure by definition. This is the half of
683
- // fleet #1096 with neither a SQLSTATE nor a status code to key on.
684
- throw new Error(
685
- markInternalOrigin(
686
- `upstream_down: ${name} did not respond within ${timeoutMs / 1000}s. ` +
687
- `That can be ${name} being slow or down, or this environment being unable to reach it ` +
688
- `(some hosts refuse datacenter/Worker egress) — retry shortly, and check reachability ` +
689
- `from elsewhere before concluding ${name} is down.`,
690
- url,
691
- ),
692
- );
693
- }
694
- // Fleet #2382. Everything that isn't a timeout/abort here is a genuine
695
- // NETWORK-LEVEL failure — DNS resolution, connection refused, TLS handshake,
696
- // Cloudflare's own "Network connection lost." — meaning `fetch()` itself
697
- // threw and no HTTP response of any kind was ever received. Until this fix
698
- // that raw exception was rethrown VERBATIM: a bare `TypeError: fetch failed`
699
- // (or the Workers-runtime equivalent) names no upstream, carries no class
700
- // token, and reads exactly like a defect in OUR code — because it says
701
- // nothing about the call at all. It landed in `error`, the tier that means
702
- // "Pipeworx has a defect", for every one of the (at the time of writing)
703
- // ~470 packs that call this helper directly with no wrapper of their own.
704
- //
705
- // `dexscreener` hit this independently (fleet #1579) and fixed it with a
706
- // bespoke per-pack try/catch around `fetchWithTimeout`. That fix is correct
707
- // but only covers one pack; every other caller of this shared helper still
708
- // leaked the raw exception. Moving the same fix HERE — the one place that
709
- // already carries the timeout case — covers every pack that uses
710
- // `fetchWithTimeout` without a wrapper, for free, and without widening
711
- // `classifyToolError`'s regex list: the fix is giving the message a proper
712
- // `upstream_down:` token at the point the two facts (no response was ever
713
- // received, and which host we were trying to reach) are actually in hand,
714
- // not teaching the classifier to guess from prose after the fact.
715
- //
716
- // Safe on the same grounds as the timeout branch above: no argument a
717
- // caller passes can make `fetch()` itself throw a connection-level error,
718
- // so this is always an availability failure, never a caller mistake. Same
719
- // `markInternalOrigin` treatment — an origin we run that never answered is
720
- // still ours, not a third party's outage.
721
- const raw = err instanceof Error ? err.message : String(err);
722
- throw new Error(
723
- markInternalOrigin(
724
- `upstream_down: could not reach ${name} at all (${raw.slice(0, 160)}). ` +
725
- `No request reached ${name}, so this says NOTHING about whether the arguments you passed ` +
726
- 'are valid — do not re-check them on the strength of this error. Retry shortly.',
727
- url,
728
- ),
729
- );
730
- }
731
- }
732
-
733
- function detailSuffix(detail: string): string {
734
- return detail ? ` — ${detail}` : '';
735
- }
736
-
737
- async function readDetail(res: Response): Promise<string> {
738
- let raw: string;
739
- try {
740
- raw = await res.text();
741
- } catch {
742
- // Body already consumed, or the connection died mid-read. The status alone
743
- // is still worth throwing — never let the error path throw its own error.
744
- return '';
745
- }
746
- return summarizeErrorBody(raw);
747
- }
748
-
749
- /**
750
- * Turn ANY error body — JSON, HTML, XML or plain text — into one short phrase
751
- * that never contains markup.
752
- *
753
- * This used to just drop an HTML or XML body on the floor, on the reasoning
754
- * that markup crowds out the status. That was half right. Dropping it loses the
755
- * one sentence a caller could have acted on: an `Access Denied` title, an SDMX
756
- * `<message:Error>` text, an OPS fault string. A 2026-08-30 support sweep
757
- * measured 13 of 291 caller-facing error rows carrying a raw page or document
758
- * verbatim, across 11 packs, and in every one of them the useful content —
759
- * "Access Denied", "Invalid country code", "SCRAPE_TIMEOUT" — was in there,
760
- * buried in markup the agent had to parse out of a string (fleet #712).
761
- *
762
- * So: extract the meaning, discard the markup. The output is passed through
763
- * `stripMarkup` unconditionally, which is what lets `check:error-body-leak`
764
- * assert mechanically that no caller-facing message can contain `<?xml`,
765
- * `<!DOCTYPE` or `<html`.
766
- */
767
- function summarizeErrorBody(raw: string): string {
768
- if (!raw || !raw.trim()) return '';
769
-
770
- const head = raw.slice(0, 400).trimStart().toLowerCase();
771
-
772
- // An HTML error page (Cloudflare interstitial, nginx default, a login
773
- // redirect) says what it is in its <title>, and almost nowhere else.
774
- if (head.startsWith('<!doctype') || head.startsWith('<html')) {
775
- const title = htmlTitle(raw);
776
- return title
777
- ? `${title} (upstream returned an HTML error page, not an API response)`
778
- : 'upstream returned an HTML error page, not an API response';
779
- }
780
-
781
- // XML fault documents — EPO OPS, SDMX (`<message:Error>`), SOAP faults. The
782
- // human sentence sits in a child element whose tag name says what it is.
783
- if (head.startsWith('<?xml') || head.startsWith('<')) {
784
- const fault = xmlFaultText(raw);
785
- return fault
786
- ? `${stripMarkup(fault).slice(0, MAX_DETAIL)} (from the upstream's XML error document)`
787
- : 'upstream returned an XML error document with no readable message';
788
- }
789
-
790
- // Most JSON error bodies bury one human sentence among ids and echoed request
791
- // params. Prefer that sentence; fall back to the whole body when the shape is
792
- // unfamiliar, since an unfamiliar shape is exactly when we can least afford to
793
- // guess wrong and show nothing.
794
- const fromJson = messageFromJson(raw);
795
- return stripMarkup(fromJson ?? raw).slice(0, MAX_DETAIL);
796
- }
797
-
798
- /** The `<title>` of an HTML error page, or its first `<h1>` — the two places a
799
- * bot wall, a 502 and an "Access Denied" all state what happened. */
800
- function htmlTitle(raw: string): string | null {
801
- const head = raw.slice(0, 4000);
802
- for (const re of [/<title[^>]*>([\s\S]*?)<\/title>/i, /<h1[^>]*>([\s\S]*?)<\/h1>/i]) {
803
- const m = re.exec(head);
804
- const text = m ? stripMarkup(m[1]) : '';
805
- if (text) return text.slice(0, 160);
806
- }
807
- return null;
808
- }
809
-
810
- /** Tag names that carry the explanation in an XML fault document, namespace
811
- * prefix optional (`<message:Error>`, `<com:Text>`, `<faultstring>`). */
812
- const XML_FAULT_TAG_RE =
813
- /<(?:[A-Za-z0-9_.-]+:)?(?:text|message|description|faultstring|reason|detail|title|errormessage|error)\b[^>]*>([^<]{2,400})</i;
814
-
815
- function xmlFaultText(raw: string): string | null {
816
- const head = raw.slice(0, 8000);
817
- const tagged = XML_FAULT_TAG_RE.exec(head);
818
- if (tagged && tagged[1].trim()) return tagged[1];
819
-
820
- // Nothing conventionally named — take the longest text node instead. A fault
821
- // document with one sentence in an oddly named element is still readable;
822
- // returning nothing at all is not.
823
- let best = '';
824
- for (const m of head.matchAll(/>([^<>]{8,400})</g)) {
825
- const text = m[1].trim();
826
- if (text.length > best.length) best = text;
827
- }
828
- return best || null;
829
- }
830
-
831
- /**
832
- * Remove every tag and stray angle bracket, then collapse whitespace.
833
- *
834
- * Applied to everything on the way out, including the JSON and plain-text
835
- * paths, because an upstream is free to embed markup in a JSON string field —
836
- * and a leak is a leak regardless of which branch produced it.
837
- */
838
- function stripMarkup(s: string): string {
839
- return collapse(decodeEntities(s.replace(/<[^>]*>/g, ' ')).replace(/[<>]/g, ' '));
840
- }
841
-
842
- /** The handful of entities that show up in error-page titles. Decoded AFTER
843
- * tags are stripped and BEFORE the angle-bracket sweep, so `&lt;script&gt;`
844
- * in a title cannot decode into markup that survives — EMBL-EBI's ChEMBL 500
845
- * page renders as `500 Internal Server Error &lt; EMBL-EBI` otherwise. */
846
- function decodeEntities(s: string): string {
847
- return s
848
- .replace(/&(?:amp|#0*38);/gi, '&')
849
- .replace(/&(?:lt|#0*60);/gi, '<')
850
- .replace(/&(?:gt|#0*62);/gi, '>')
851
- .replace(/&(?:quot|#0*34);/gi, '"')
852
- .replace(/&(?:#0*39|apos|#x0*27);/gi, "'")
853
- .replace(/&nbsp;/gi, ' ');
854
- }
855
-
856
- /** The conventional "what went wrong" field, under any of the names upstreams
857
- * actually use. Checked in order; first non-empty string wins. */
858
- const MESSAGE_KEYS = [
859
- 'message', 'error_message', 'errorMessage', 'detail', 'details',
860
- 'description', 'error_description', 'reason', 'title', 'fault',
861
- ];
862
-
863
- function messageFromJson(raw: string): string | null {
864
- let parsed: unknown;
865
- try {
866
- parsed = JSON.parse(raw);
867
- } catch {
868
- return null;
869
- }
870
- return pickMessage(parsed, 0);
871
- }
872
-
873
- function pickMessage(node: unknown, depth: number): string | null {
874
- // Two levels covers `{error: {message}}` and `{errors: [{detail}]}`, the two
875
- // shapes that account for nearly all of them, without walking a large payload.
876
- if (depth > 2 || node == null) return null;
877
-
878
- if (typeof node === 'string') return node.trim() || null;
879
-
880
- if (Array.isArray(node)) {
881
- for (const item of node) {
882
- const found = pickMessage(item, depth + 1);
883
- if (found) return found;
884
- }
885
- return null;
886
- }
887
-
888
- if (typeof node !== 'object') return null;
889
- const obj = node as Record<string, unknown>;
890
-
891
- for (const key of MESSAGE_KEYS) {
892
- const v = obj[key];
893
- if (typeof v === 'string' && v.trim()) return v.trim();
894
- }
895
- // `{error: …}` where error is itself an object or a string — the single most
896
- // common wrapper, so it is worth descending into by name rather than scanning
897
- // every key and risking picking up an echoed request parameter.
898
- for (const key of ['error', 'errors', 'fault', 'Error', 'data']) {
899
- if (key in obj) {
900
- const found = pickMessage(obj[key], depth + 1);
901
- if (found) return found;
902
- }
903
- }
904
- return null;
905
- }
906
-
907
- /** Errors are read in a single line of log output; newlines and runs of
908
- * whitespace make a multi-line body unreadable there. */
909
- function collapse(s: string): string {
910
- return s.replace(/\s+/g, ' ').trim();
911
- }
912
-
913
- // ── Bounded readers — peak memory, not just wall time (fleet #3322) ──────
914
- //
915
- // `parseJson` above does `await res.text()` then `JSON.parse(raw)`, so a
916
- // multi-megabyte upstream body is held TWICE at peak: once as the raw string
917
- // and once as the object graph (which for an array of small objects runs
918
- // several times the byte size). Nothing in `workers/gateway` can see that
919
- // peak — it happens inside the pack, before any gateway code holds a payload —
920
- // which is why the gateway-side payload caps of #3293 and #3320 did not move
921
- // the 503/1102 rate at all.
922
- //
923
- // Measured on the live gateway, 2026-10-19, sha fa5de5c34, direct tools/call:
924
- // statcan_list_cubes 8,049,566 B <- one upstream array, no limit argument
925
- // gnomad_gene BRCA1 2,732,706 B
926
- // Five of those in flight at the quality runner's width is ~40 MB of parse
927
- // peak against a 128 MB isolate; Cloudflare kills the isolate and every
928
- // request in flight returns 503/1102, which is why the victims look random.
929
- //
930
- // `readJsonArrayPrefix` is the fix for the dominant shape: an upstream that
931
- // returns one enormous top-level JSON array with no server-side paging. It
932
- // parses elements off the stream and DISCARDS the consumed bytes, so peak
933
- // retained memory is one element plus one chunk rather than the whole body,
934
- // and it cancels the body as soon as it has what it was asked for.
935
-
936
- /** Default ceiling for a single upstream JSON body read into an isolate.
937
- * Chosen so the known tail (statcan 8.0 MB, gnomad 2.7 MB) is bounded well
938
- * below the point where a handful of concurrent reads exhausts 128 MB. */
939
- const DEFAULT_MAX_JSON_BYTES = 1_500_000;
940
-
941
- interface JsonArrayPrefix<T> {
942
- /** The decoded elements, at most `maxItems` of them. */
943
- items: T[];
944
- /** True when the upstream array had more elements than we read. */
945
- truncated: boolean;
946
- /** Bytes actually pulled off the socket — the number that bounds memory. */
947
- bytesRead: number;
948
- /** `Content-Length` if the upstream declared one, else null. */
949
- declaredBytes: number | null;
950
- }
951
-
952
- /**
953
- * Read the FIRST `maxItems` elements of a top-level JSON array off a response
954
- * stream without ever materialising the whole body.
955
- *
956
- * Use this instead of `parseJson` when the upstream returns one big array and
957
- * offers no `limit`/`offset` of its own. Peak retained memory is one element
958
- * plus the current chunk; the body is cancelled as soon as `maxItems` is
959
- * reached, so the remaining megabytes are never transferred at all.
960
- *
961
- * const { items, truncated } = await readJsonArrayPrefix<Cube>(
962
- * res, 'StatCan WDS', { maxItems: limit + offset });
963
- *
964
- * `maxBytes` is a second, independent backstop for a pathological element (a
965
- * single 50 MB object): hitting it stops the read and sets `truncated`, rather
966
- * than throwing, because a partial answer plus an explicit truncated flag is
967
- * more useful to a caller than an error.
968
- */
969
- async function readJsonArrayPrefix<T>(
970
- res: Response,
971
- name: string,
972
- opts: { maxItems: number; maxBytes?: number },
973
- ): Promise<JsonArrayPrefix<T>> {
974
- const maxBytes = opts.maxBytes ?? DEFAULT_MAX_JSON_BYTES;
975
- const declaredRaw = Number(res.headers.get('content-length'));
976
- const declaredBytes = Number.isFinite(declaredRaw) && declaredRaw > 0 ? declaredRaw : null;
977
-
978
- // No streaming body available (a mock, or an already-buffered response):
979
- // fall back to the whole-text path so behaviour stays correct, just unbounded.
980
- if (!res.body) {
981
- const all = await parseJson<T[]>(res, name);
982
- if (!Array.isArray(all)) throw new Error(`upstream_down: ${name} returned ${typeof all} where a JSON array was expected.`);
983
- return {
984
- items: all.slice(0, opts.maxItems),
985
- truncated: all.length > opts.maxItems,
986
- bytesRead: declaredBytes ?? 0,
987
- declaredBytes,
988
- };
989
- }
990
-
991
- const reader = res.body.getReader();
992
- const decoder = new TextDecoder();
993
- const items: T[] = [];
994
-
995
- let buf = '';
996
- let scan = 0;
997
- let bytesRead = 0;
998
- let started = false; // consumed the opening '['
999
- let arrayEnded = false;
1000
- let depth = 0;
1001
- let inStr = false;
1002
- let esc = false;
1003
- let elemStart = -1;
1004
- let truncated = false;
1005
-
1006
- const push = (text: string) => {
1007
- const t = text.trim();
1008
- if (!t) return;
1009
- items.push(JSON.parse(t) as T);
1010
- };
1011
-
1012
- try {
1013
- outer: while (items.length < opts.maxItems && !arrayEnded) {
1014
- const { value, done } = await reader.read();
1015
- if (done) break;
1016
- bytesRead += value.byteLength;
1017
- buf += decoder.decode(value, { stream: true });
1018
- if (bytesRead > maxBytes) { truncated = true; break; }
1019
-
1020
- for (; scan < buf.length; scan++) {
1021
- const ch = buf[scan];
1022
-
1023
- if (inStr) {
1024
- if (esc) { esc = false; continue; }
1025
- if (ch === '\\') { esc = true; continue; }
1026
- if (ch === '"') inStr = false;
1027
- continue;
1028
- }
1029
-
1030
- if (!started) {
1031
- if (ch === '[') { started = true; continue; }
1032
- if (ch.trim() === '') continue;
1033
- throw new Error(
1034
- `upstream_down: ${name} returned a body that does not begin with a JSON array (saw ${JSON.stringify(ch)}). ` +
1035
- 'This reader is only for upstreams that answer with one top-level array.',
1036
- );
1037
- }
1038
-
1039
- if (ch === '"') { if (depth === 0 && elemStart < 0) elemStart = scan; inStr = true; continue; }
1040
-
1041
- if (ch === '{' || ch === '[') {
1042
- if (depth === 0 && elemStart < 0) elemStart = scan;
1043
- depth++;
1044
- continue;
1045
- }
1046
-
1047
- if (ch === '}' || ch === ']') {
1048
- if (depth > 0) {
1049
- depth--;
1050
- if (depth === 0 && elemStart >= 0) {
1051
- push(buf.slice(elemStart, scan + 1));
1052
- elemStart = -1;
1053
- if (items.length >= opts.maxItems) { truncated = true; scan++; break outer; }
1054
- }
1055
- continue;
1056
- }
1057
- // depth 0 and a closing bracket: the end of the top-level array.
1058
- if (elemStart >= 0) { push(buf.slice(elemStart, scan)); elemStart = -1; }
1059
- arrayEnded = true;
1060
- break outer;
1061
- }
1062
-
1063
- if (ch === ',' && depth === 0) {
1064
- if (elemStart >= 0) {
1065
- push(buf.slice(elemStart, scan));
1066
- elemStart = -1;
1067
- if (items.length >= opts.maxItems) { truncated = true; scan++; break outer; }
1068
- }
1069
- continue;
1070
- }
1071
-
1072
- // A bare scalar element (number / true / false / null).
1073
- if (depth === 0 && elemStart < 0 && ch.trim() !== '') elemStart = scan;
1074
- }
1075
-
1076
- // Drop everything already consumed. THIS is what bounds memory: without
1077
- // it `buf` grows to the full body and the reader is streaming in name only.
1078
- if (elemStart >= 0) {
1079
- buf = buf.slice(elemStart);
1080
- scan -= elemStart;
1081
- elemStart = 0;
1082
- } else {
1083
- buf = buf.slice(scan);
1084
- scan = 0;
1085
- }
1086
- }
1087
- } finally {
1088
- // Cancel rather than drain: the point is that the remaining megabytes are
1089
- // never transferred. A cancel on an already-finished body is a no-op.
1090
- await reader.cancel().catch(() => {});
1091
- }
1092
-
1093
- return { items, truncated, bytesRead, declaredBytes };
1094
- }
1095
-
1096
- /**
1097
- * `parseJson`, with a hard ceiling on how many bytes are read.
1098
- *
1099
- * For an upstream whose body is not a single array (so `readJsonArrayPrefix`
1100
- * does not apply) but which can still answer with megabytes. Unlike the array
1101
- * reader this one THROWS when the cap is hit, because a truncated prefix of an
1102
- * arbitrary JSON document is not parseable and there is no partial answer to
1103
- * hand back. A declared `Content-Length` over the cap is refused before a
1104
- * single chunk is read.
1105
- */
1106
- async function parseJsonBounded<T>(
1107
- res: Response,
1108
- name: string,
1109
- maxBytes: number = DEFAULT_MAX_JSON_BYTES,
1110
- ): Promise<T> {
1111
- const declaredRaw = Number(res.headers.get('content-length'));
1112
- const declared = Number.isFinite(declaredRaw) && declaredRaw > 0 ? declaredRaw : null;
1113
-
1114
- const tooBig = (n: number) =>
1115
- new Error(
1116
- `upstream_down: ${name} answered with ${n.toLocaleString()} bytes, over this pack's ${maxBytes.toLocaleString()}-byte ceiling. ` +
1117
- 'The body is not read past the ceiling on purpose — a few concurrent reads of a body that size exhaust the ' +
1118
- 'Worker isolate and take every other in-flight request down with them. Narrow the request (a smaller range, ' +
1119
- 'a specific id, or a lower limit) and retry.',
1120
- );
1121
-
1122
- if (declared !== null && declared > maxBytes) {
1123
- await res.body?.cancel().catch(() => {});
1124
- throw tooBig(declared);
1125
- }
1126
- if (!res.body) return parseJson<T>(res, name);
1127
-
1128
- const reader = res.body.getReader();
1129
- const decoder = new TextDecoder();
1130
- let raw = '';
1131
- let bytesRead = 0;
1132
- try {
1133
- for (;;) {
1134
- const { value, done } = await reader.read();
1135
- if (done) break;
1136
- bytesRead += value.byteLength;
1137
- if (bytesRead > maxBytes) throw tooBig(bytesRead);
1138
- raw += decoder.decode(value, { stream: true });
1139
- }
1140
- raw += decoder.decode();
1141
- } finally {
1142
- await reader.cancel().catch(() => {});
1143
- }
1144
-
1145
- if (!raw.trim()) {
1146
- throw new Error(
1147
- `upstream_down: ${name} answered HTTP ${res.status} with an EMPTY body where JSON was expected. ` +
1148
- 'Nothing about the request can cause this — it is an upstream fault, and the same call may well work on retry.',
1149
- );
1150
- }
1151
- try {
1152
- return JSON.parse(raw) as T;
1153
- } catch {
1154
- throw new Error(
1155
- `upstream_down: ${name} answered HTTP ${res.status} with a body that is not valid JSON. ` +
1156
- `It begins: ${stripMarkup(raw).slice(0, 120) || '(unreadable)'}`,
1157
- );
1158
- }
1159
- }
1160
- /**
1161
- * OpenAQ MCP — global air-quality measurements via the OpenAQ v3 API.
1162
- *
1163
- * OpenAQ v2 was shut down (HTTP 410); this targets v3, which REQUIRES an API key
1164
- * (X-API-Key header). The gateway injects PLATFORM_OPENAQ_TOKEN as `_apiKey`;
1165
- * users may pass their own. Free key at https://explore.openaq.org/.
1166
- *
1167
- * Tools:
1168
- * - air_quality_near: latest pollutant readings at the nearest station to a lat/lon
1169
- * - find_stations: discover monitoring stations by coordinates+radius, country, or bbox
1170
- * - get_latest: latest reading for every pollutant at a station (by location id)
1171
- * - get_measurements: historical hourly/daily series for a sensor
1172
- */
1173
-
1174
-
1175
- // Bound every fetch() in this pack to a fixed timeout — an upstream that
1176
- // degrades without erroring would otherwise hold the Worker in `await fetch()`
1177
- // until its own execution budget kills the request (minutes, not seconds).
1178
- // Mirrors the epoFetch / usaspending retryFetch pattern (fleet #685).
1179
- async function pwFetch(url: string | URL, init?: RequestInit): Promise<Response> {
1180
- return fetchWithTimeout(url, init ?? {}, 'OpenAQ');
1181
- }
1182
-
1183
- const BASE = 'https://api.openaq.org/v3';
1184
-
1185
- // ── API types ─────────────────────────────────────────────────────────
1186
- interface Sensor {
1187
- id: number;
1188
- parameter?: { name?: string; units?: string; displayName?: string };
1189
- }
1190
- interface Location {
1191
- id: number;
1192
- name?: string;
1193
- locality?: string | null;
1194
- country?: { code?: string; name?: string };
1195
- coordinates?: { latitude?: number; longitude?: number };
1196
- sensors?: Sensor[];
1197
- distance?: number;
1198
- }
1199
- interface LatestRow {
1200
- sensorsId: number;
1201
- value: number;
1202
- datetime?: { utc?: string };
1203
- }
1204
-
1205
- // ── Helpers ───────────────────────────────────────────────────────────
1206
- function extractKey(args: Record<string, unknown>): string {
1207
- const k = typeof args._apiKey === 'string' ? args._apiKey.trim() : '';
1208
- delete args._apiKey;
1209
- if (!k) {
1210
- throw new Error(
1211
- 'OpenAQ requires an API key: pass your key as the _apiKey argument (free at https://explore.openaq.org/).',
1212
- );
1213
- }
1214
- return k;
1215
- }
1216
-
1217
- async function oaqGet<T>(path: string, key: string): Promise<T> {
1218
- const res = await pwFetch(`${BASE}${path}`, {
1219
- headers: { 'X-API-Key': key, Accept: 'application/json', 'User-Agent': 'pipeworx-mcp' },
1220
- });
1221
- if (res.status === 401) throw new Error('OpenAQ: invalid or missing API key (401).');
1222
- if (res.status === 429) throw new Error('OpenAQ: rate limit exceeded (429). Retry shortly.');
1223
- if (res.status === 404) return { results: [] } as unknown as T;
1224
- if (!res.ok) throw await httpError(res, 'OpenAQ error');
1225
- return res.json() as Promise<T>;
1226
- }
1227
-
1228
- function formatStation(l: Location) {
1229
- return {
1230
- id: l.id,
1231
- name: l.name ?? null,
1232
- locality: l.locality ?? null,
1233
- country: l.country?.code ?? null,
1234
- country_name: l.country?.name ?? null,
1235
- latitude: l.coordinates?.latitude ?? null,
1236
- longitude: l.coordinates?.longitude ?? null,
1237
- distance_m: l.distance != null ? Math.round(l.distance) : undefined,
1238
- sensors: (l.sensors ?? []).map((s) => ({
1239
- sensor_id: s.id,
1240
- parameter: s.parameter?.name ?? null,
1241
- units: s.parameter?.units ?? null,
1242
- })),
1243
- };
1244
- }
1245
-
1246
- function radiusMeters(radiusKm: number | undefined): number {
1247
- return Math.min(25000, Math.max(100, Math.round((radiusKm ?? 12) * 1000)));
1248
- }
1249
-
1250
- // ── Tool definitions ──────────────────────────────────────────────────
1251
- const tools: McpToolExport['tools'] = [
1252
- {
1253
- name: 'air_quality_near',
1254
- description:
1255
- 'Get the latest air-quality readings (PM2.5, PM10, O3, NO2, SO2, CO, etc.) at the monitoring station NEAREST to a latitude/longitude. PREFER for "air quality near me", "what is the air quality at <coordinates>", "is the air bad in <place> right now". Returns the nearest station with its most recent pollutant values, plus other nearby stations.',
1256
- inputSchema: {
1257
- type: 'object',
1258
- properties: {
1259
- latitude: { type: 'number', description: 'Latitude in decimal degrees (e.g., 40.785).' },
1260
- longitude: { type: 'number', description: 'Longitude in decimal degrees (e.g., -73.968).' },
1261
- radius_km: { type: 'number', description: 'Search radius in km (default 12, max 25 — OpenAQ limit).' },
1262
- _apiKey: { type: 'string', description: 'OpenAQ API key (optional; the gateway supplies one).' },
1263
- },
1264
- required: ['latitude', 'longitude'],
1265
- },
1266
- },
1267
- {
1268
- name: 'find_stations',
1269
- description:
1270
- 'Find OpenAQ air-quality monitoring stations by location: a city or place name, coordinates+radius (nearest first), a country (ISO 3166-1 alpha-2 code), or a bounding box. Returns station id, name, country, coordinates, and the pollutants each measures (with sensor ids), plus resolved_place echoing the coordinates a city name resolved to. This is the first step for a historical series: find the station here, then pass its sensor_id to get_measurements. At least one location argument is required — the tool will not return an unfiltered global list.',
1271
- inputSchema: {
1272
- type: 'object',
1273
- properties: {
1274
- city: { type: 'string', description: 'City or place name, e.g. "Delhi", "Los Angeles", "Kraków". Geocoded to coordinates, then searched by radius. Add the country for ambiguous names ("Cambridge, UK").' },
1275
- latitude: { type: 'number', description: 'Latitude for a radius search (use with longitude).' },
1276
- longitude: { type: 'number', description: 'Longitude for a radius search (use with latitude).' },
1277
- radius_km: { type: 'number', description: 'Radius in km for a coordinate or city search (default 12, max 25).' },
1278
- country: { type: 'string', description: 'ISO 3166-1 alpha-2 country code (e.g. "US", "IN", "GB").' },
1279
- bbox: { type: 'string', description: 'Bounding box "minLon,minLat,maxLon,maxLat" to search within.' },
1280
- limit: { type: 'number', description: 'Max stations to return (1-100, default 20).' },
1281
- _apiKey: { type: 'string', description: 'OpenAQ API key (optional; the gateway supplies one).' },
1282
- },
1283
- required: [],
1284
- },
1285
- },
1286
- {
1287
- name: 'get_latest',
1288
- description:
1289
- 'Get the latest reading for every pollutant at a specific OpenAQ station (by location id from find_stations). Returns each parameter (PM2.5, O3, NO2, etc.) with its value, units, and measurement time.',
1290
- inputSchema: {
1291
- type: 'object',
1292
- properties: {
1293
- location_id: { type: 'number', description: 'OpenAQ location/station id (from find_stations or air_quality_near).' },
1294
- _apiKey: { type: 'string', description: 'OpenAQ API key (optional; the gateway supplies one).' },
1295
- },
1296
- required: ['location_id'],
1297
- },
1298
- },
1299
- {
1300
- name: 'get_measurements',
1301
- description:
1302
- 'Get the most recent time series (hourly or daily aggregates) for a single OpenAQ sensor (sensor id from find_stations), newest first. Use for pollutant trends over time at one station/parameter. Returns the sensor\'s LAST `limit` readings — note that many OpenAQ sensors are archived and stopped reporting years ago, so "most recent" can legitimately be several years old: always read the returned timestamps rather than assuming the data is current. Use find_stations or get_latest to pick a sensor that is still reporting.',
1303
- inputSchema: {
1304
- type: 'object',
1305
- properties: {
1306
- sensor_id: { type: 'number', description: 'OpenAQ sensor id (a station+parameter pair, from find_stations).' },
1307
- period: { type: 'string', description: 'Aggregation: "hours" (default) or "days".' },
1308
- limit: { type: 'number', description: 'How many of the most recent data points to return (1-1000, default 24).' },
1309
- _apiKey: { type: 'string', description: 'OpenAQ API key (optional; the gateway supplies one).' },
1310
- },
1311
- required: ['sensor_id'],
1312
- },
1313
- },
1314
- ];
1315
-
1316
- // ── Dispatch ──────────────────────────────────────────────────────────
1317
- async function callTool(name: string, args: Record<string, unknown>): Promise<unknown> {
1318
- const key = extractKey(args);
1319
- switch (name) {
1320
- case 'air_quality_near':
1321
- return airQualityNear(args.latitude as number, args.longitude as number, (args.radius_km as number) ?? 12, key);
1322
- case 'find_stations':
1323
- return findStations(args, key);
1324
- case 'get_latest':
1325
- return getLatest(args.location_id as number, key);
1326
- case 'get_measurements':
1327
- return getMeasurements(
1328
- args.sensor_id as number,
1329
- typeof args.period === 'string' ? args.period : 'hours',
1330
- (args.limit as number) ?? 24,
1331
- key,
1332
- );
1333
- default:
1334
- throw new Error(`Unknown tool: ${name}`);
1335
- }
1336
- }
1337
-
1338
- // ── Implementations ───────────────────────────────────────────────────
1339
- async function findStations(args: Record<string, unknown>, key: string) {
1340
- const limit = Math.min(100, Math.max(1, (args.limit as number) ?? 20));
1341
- const params = new URLSearchParams({ limit: String(limit) });
1342
-
1343
- let lat = args.latitude as number | undefined;
1344
- let lon = args.longitude as number | undefined;
1345
- let resolvedPlace: ResolvedPlace | null = null;
1346
-
1347
- // A city name is how a person (and therefore a router) actually asks. Geocode it
1348
- // to coordinates and hand back what we resolved to, so a wrong city is visible in
1349
- // the answer rather than silently producing stations on another continent.
1350
- const city = typeof args.city === 'string' ? args.city.trim() : '';
1351
- if (city && !(Number.isFinite(lat) && Number.isFinite(lon))) {
1352
- const hit = await geocodePlace(city);
1353
- if (!hit) {
1354
- throw new Error(
1355
- `Could not resolve "${city}" to a location. Try adding a country ("Cambridge, UK"), ` +
1356
- `or pass latitude/longitude directly.`,
1357
- );
1358
- }
1359
- lat = hit.latitude;
1360
- lon = hit.longitude;
1361
- resolvedPlace = hit;
1362
- }
1363
-
1364
- if (Number.isFinite(lat) && Number.isFinite(lon)) {
1365
- params.set('coordinates', `${lat},${lon}`);
1366
- params.set('radius', String(radiusMeters(args.radius_km as number | undefined)));
1367
- }
1368
- if (typeof args.country === 'string' && args.country.trim()) params.set('iso', args.country.trim().toUpperCase());
1369
- if (typeof args.bbox === 'string' && args.bbox.trim()) params.set('bbox', args.bbox.trim());
1370
-
1371
- // Without a locating argument OpenAQ happily returns the first N stations on earth.
1372
- // That reads like an answer to "stations in Delhi" while being nothing of the kind,
1373
- // so refuse rather than emit a global slice.
1374
- if (!params.has('coordinates') && !params.has('iso') && !params.has('bbox')) {
1375
- throw new Error(
1376
- 'A location is required: pass city, or latitude+longitude, or country (ISO alpha-2), or bbox. ' +
1377
- 'Without one this would return an arbitrary global list rather than stations near you.',
1378
- );
1379
- }
1380
-
1381
- const data = await oaqGet<{ meta?: { found?: number | string }; results?: Location[] }>(
1382
- `/locations?${params}`,
1383
- key,
1384
- );
1385
- const stations = (data.results ?? []).map(formatStation);
1386
- return {
1387
- ...(resolvedPlace
1388
- ? {
1389
- resolved_place: {
1390
- query: city,
1391
- name: resolvedPlace.name,
1392
- admin1: resolvedPlace.admin1,
1393
- country_code: resolvedPlace.country_code,
1394
- latitude: resolvedPlace.latitude,
1395
- longitude: resolvedPlace.longitude,
1396
- },
1397
- }
1398
- : {}),
1399
- found: data.meta?.found ?? stations.length,
1400
- count: stations.length,
1401
- stations,
1402
- ...(stations.length === 0
1403
- ? {
1404
- message: resolvedPlace
1405
- ? `No OpenAQ stations within the search radius of ${resolvedPlace.name}. Try a larger radius_km (max 25).`
1406
- : 'No OpenAQ stations matched. Try a larger radius_km (max 25) or a wider area.',
1407
- }
1408
- : {}),
1409
- };
1410
- }
1411
-
1412
- const GEOCODE_URL = 'https://geocoding-api.open-meteo.com/v1/search';
1413
-
1414
- interface ResolvedPlace {
1415
- name: string;
1416
- admin1: string | null;
1417
- country_code: string | null;
1418
- latitude: number;
1419
- longitude: number;
1420
- }
1421
-
1422
- // Place name → coordinates. Ranking + endonym fallback live in
1423
- // in three days: each called the geocoder with count=1 under a comment claiming
1424
- // Open-Meteo ranks by relevance and population. It does not — "Roma" returned a
1425
- // Romanian village of 3,218 people, and Rome is absent from the ENGLISH result
1426
- // set entirely, so sorting alone would only have moved the answer to Lesotho.
1427
- async function geocodePlace(place: string): Promise<ResolvedPlace | null> {
1428
- const hit = await resolvePlace(place);
1429
- if (!hit) return null;
1430
- return {
1431
- name: hit.name,
1432
- admin1: hit.admin1,
1433
- country_code: hit.country_code,
1434
- latitude: hit.latitude,
1435
- longitude: hit.longitude,
1436
- };
1437
- }
1438
-
1439
- async function getLatest(locationId: number, key: string) {
1440
- if (!Number.isFinite(locationId)) throw new Error('location_id is required (numeric).');
1441
- // Fetch the station for the sensorId → parameter map, then its latest values.
1442
- const [loc, latest] = await Promise.all([
1443
- oaqGet<{ results?: Location[] }>(`/locations/${locationId}`, key),
1444
- oaqGet<{ results?: LatestRow[] }>(`/locations/${locationId}/latest`, key),
1445
- ]);
1446
- const station = loc.results?.[0];
1447
- const paramBySensor = new Map<number, { name?: string; units?: string }>();
1448
- for (const s of station?.sensors ?? []) {
1449
- paramBySensor.set(s.id, { name: s.parameter?.name, units: s.parameter?.units });
1450
- }
1451
- const readings = (latest.results ?? []).map((r) => {
1452
- const p = paramBySensor.get(r.sensorsId);
1453
- return {
1454
- parameter: p?.name ?? null,
1455
- value: r.value,
1456
- units: p?.units ?? null,
1457
- sensor_id: r.sensorsId,
1458
- datetime: r.datetime?.utc ?? null,
1459
- };
1460
- });
1461
- return {
1462
- station: station ? formatStation(station) : { id: locationId },
1463
- count: readings.length,
1464
- readings,
1465
- };
1466
- }
1467
-
1468
- async function airQualityNear(latitude: number, longitude: number, radiusKm: number, key: string) {
1469
- if (!Number.isFinite(latitude) || !Number.isFinite(longitude)) {
1470
- throw new Error('latitude and longitude are required numbers (decimal degrees).');
1471
- }
1472
- const params = new URLSearchParams({
1473
- coordinates: `${latitude},${longitude}`,
1474
- radius: String(radiusMeters(radiusKm)),
1475
- limit: '10',
1476
- });
1477
- const data = await oaqGet<{ results?: Location[] }>(`/locations?${params}`, key);
1478
- const stations = data.results ?? [];
1479
- if (stations.length === 0) {
1480
- return {
1481
- center: { latitude, longitude },
1482
- radius_km: Math.min(25, radiusKm),
1483
- message: 'No OpenAQ monitoring stations within the radius. Try a larger radius_km (max 25).',
1484
- nearest: null,
1485
- readings: [],
1486
- };
1487
- }
1488
- // results are ordered nearest-first for a coordinate query.
1489
- const nearest = stations[0];
1490
- const latest = await getLatest(nearest.id, key);
1491
- // getLatest refetches /locations/{id}, which lacks the distance from the
1492
- // coordinate query — carry it over from the nearby-search result.
1493
- const nearestStation = { ...latest.station, distance_m: nearest.distance != null ? Math.round(nearest.distance) : undefined };
1494
- return {
1495
- center: { latitude, longitude },
1496
- radius_km: Math.min(25, radiusKm),
1497
- nearest: nearestStation,
1498
- readings: latest.readings,
1499
- other_nearby: stations.slice(1, 6).map((s) => ({
1500
- id: s.id,
1501
- name: s.name ?? null,
1502
- distance_m: s.distance != null ? Math.round(s.distance) : undefined,
1503
- })),
1504
- };
1505
- }
1506
-
1507
- type AggRow = {
1508
- value?: number;
1509
- parameter?: { name?: string; units?: string };
1510
- period?: { datetimeFrom?: { utc?: string }; datetimeTo?: { utc?: string } };
1511
- };
1512
- type AggPage = { results?: AggRow[] };
1513
- type SensorDetail = {
1514
- results?: Array<{
1515
- parameter?: { name?: string; units?: string };
1516
- datetimeFirst?: { utc?: string };
1517
- datetimeLast?: { utc?: string };
1518
- }>;
1519
- };
1520
-
1521
- const PAGE = 1000;
1522
-
1523
- // Reaching the NEWEST readings on /v3/sensors/{id}/{hours,days} is entirely
1524
- // upstream-shaped, and every obvious approach is wrong — measured, not assumed:
1525
- // - sort_order / order_by are accepted and silently ignored, so page 1 is the
1526
- // OLDEST data. LA sensor 25193 answered "recent ozone trend" with 2020
1527
- // readings while the tool description promised "newest first".
1528
- // - datetime_from is ignored outright by /days (a 10-day window still
1529
- // returned 2017), and on /hours it makes the query expensive enough that
1530
- // every active sensor 408s — even a 2-day window.
1531
- // - meta.found is the string ">N", not a count, so it can't locate the end.
1532
- // What IS reliable: the order is stable and oldest-first, and /sensors/{id}
1533
- // reports datetimeFirst/datetimeLast. That span bounds how many buckets can
1534
- // exist, which bounds the page count from ABOVE (gaps only ever reduce it), so
1535
- // we can jump to the last page and walk back if we overshot.
1536
- async function findLastPage(sensorId: number, agg: 'hours' | 'days', hiPage: number, key: string) {
1537
- const nonEmpty = async (p: number) => {
1538
- const r = await oaqGet<AggPage>(`/sensors/${sensorId}/${agg}?limit=1&page=${(p - 1) * PAGE + 1}`, key);
1539
- return (r.results ?? []).length > 0;
1540
- };
1541
- if (await nonEmpty(hiPage)) return hiPage;
1542
- // Binary search for the last page that has rows. Bounded by log2(hiPage) —
1543
- // ~7 tiny requests even for a decade of hourly data — and each probe asks for
1544
- // a single row, so this is cheap regardless of how it converges.
1545
- let lo = 1;
1546
- let hi = hiPage;
1547
- let best = 0;
1548
- while (lo <= hi) {
1549
- const mid = Math.floor((lo + hi) / 2);
1550
- if (await nonEmpty(mid)) { best = mid; lo = mid + 1; } else { hi = mid - 1; }
1551
- }
1552
- return best;
1553
- }
1554
-
1555
- // Paging to the end costs OpenAQ a deep offset scan, and past roughly 40k rows
1556
- // it gives up with a 408 — an active station with a decade of HOURLY data is
1557
- // exactly that case (sensor 25193, ~80k rows, 408s every time; the same sensor's
1558
- // daily aggregates are 24x shallower and answer fine). Daily is a real answer to
1559
- // "what's the trend here", so degrade to it rather than failing — but say so in
1560
- // `aggregation`, because silently changing the grain is its own class of bug
1561
- // (see the resolver-grain traps we've hit before).
1562
- function isTimeout(e: unknown) {
1563
- return /\b(408|504|timeout)\b/i.test((e as Error)?.message ?? '');
1564
- }
1565
-
1566
- async function tailRows(sensorId: number, agg: 'hours' | 'days', want: number, first: string, last: string, key: string) {
1567
- const bucketMs = agg === 'days' ? 86_400_000 : 3_600_000;
1568
- const span = Date.parse(last) - Date.parse(first);
1569
- const maxBuckets = Number.isFinite(span) ? Math.floor(Math.max(0, span) / bucketMs) + 1 : 0;
1570
- const lastPage = await findLastPage(sensorId, agg, Math.max(1, Math.ceil(maxBuckets / PAGE)), key);
1571
- if (lastPage <= 0) return [];
1572
- const page = await oaqGet<AggPage>(`/sensors/${sensorId}/${agg}?limit=${PAGE}&page=${lastPage}`, key);
1573
- let rows = page.results ?? [];
1574
- // A short final page still has to yield `want` rows, so top it up from the
1575
- // one before rather than returning whatever the remainder happened to be.
1576
- if (rows.length < want && lastPage > 1) {
1577
- const prev = await oaqGet<AggPage>(`/sensors/${sensorId}/${agg}?limit=${PAGE}&page=${lastPage - 1}`, key);
1578
- rows = [...(prev.results ?? []), ...rows];
1579
- }
1580
- return rows;
1581
- }
1582
-
1583
- async function getMeasurements(sensorId: number, period: string, limit: number, key: string) {
1584
- if (!Number.isFinite(sensorId)) throw new Error('sensor_id is required (numeric).');
1585
- const requestedAgg: 'hours' | 'days' = period === 'days' ? 'days' : 'hours';
1586
- const want = Math.min(1000, Math.max(1, limit));
1587
-
1588
- const detail = await oaqGet<SensorDetail>(`/sensors/${sensorId}`, key);
1589
- const sensor = detail.results?.[0];
1590
- const first = sensor?.datetimeFirst?.utc;
1591
- const last = sensor?.datetimeLast?.utc;
1592
-
1593
- let agg = requestedAgg;
1594
- let downgraded = false;
1595
- let rows: AggRow[] = [];
1596
- if (first && last) {
1597
- try {
1598
- rows = await tailRows(sensorId, agg, want, first, last, key);
1599
- } catch (e) {
1600
- if (!isTimeout(e) || agg === 'days') throw e;
1601
- agg = 'days';
1602
- downgraded = true;
1603
- rows = await tailRows(sensorId, agg, want, first, last, key);
1604
- }
1605
- } else {
1606
- // No span to bound the search. Page 1 is the OLDEST data, which is the bug
1607
- // this function exists to fix — return it explicitly labelled rather than
1608
- // passing it off as current.
1609
- rows = (await oaqGet<AggPage>(`/sensors/${sensorId}/${agg}?limit=${PAGE}`, key)).results ?? [];
1610
- }
1611
-
1612
- const sorted = rows.slice().sort((a, b) => {
1613
- const at = a.period?.datetimeFrom?.utc ?? '';
1614
- const bt = b.period?.datetimeFrom?.utc ?? '';
1615
- return bt.localeCompare(at);
1616
- }).slice(0, want);
1617
-
1618
- const withParam = sorted.find((r) => r.parameter?.name);
1619
- return {
1620
- sensor_id: sensorId,
1621
- aggregation: agg,
1622
- parameter: withParam?.parameter?.name ?? sensor?.parameter?.name ?? null,
1623
- units: withParam?.parameter?.units ?? sensor?.parameter?.units ?? null,
1624
- count: sorted.length,
1625
- // The sensor's own last-reported timestamp, straight from OpenAQ. If this
1626
- // is years old the sensor is archived — that's the single most useful field
1627
- // here, because "most recent" and "current" are not the same thing.
1628
- sensor_last_reported: last ?? null,
1629
- latest: sorted[0]?.period?.datetimeFrom?.utc ?? null,
1630
- ...(downgraded ? { requested_period: requestedAgg } : {}),
1631
- ...(downgraded
1632
- ? { note: `OpenAQ timed out serving hourly data for sensor ${sensorId} — it has too many hourly rows to page through. These are DAILY averages instead; read the aggregation field. For hourly resolution, use a sensor with a shorter history.` }
1633
- : !first || !last
1634
- ? { note: 'OpenAQ reports no first/last timestamp for this sensor, so these are its OLDEST readings, not its newest. Trust the timestamps, not the ordering.' }
1635
- : sorted.length === 0
1636
- ? { note: `Sensor ${sensorId} has no ${agg === 'days' ? 'daily' : 'hourly'} aggregates. Try period:"${agg === 'days' ? 'hours' : 'days'}", or another sensor from find_stations.` }
1637
- : {}),
1638
- measurements: sorted.map((r) => ({
1639
- value: r.value ?? null,
1640
- from: r.period?.datetimeFrom?.utc ?? null,
1641
- to: r.period?.datetimeTo?.utc ?? null,
1642
- })),
1643
- };
1644
- }
1645
-
1646
- export default { tools, callTool, meter: { credits: 5 } } satisfies McpToolExport;