@chrischall/mcp-utils 0.10.5 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +123 -17
- package/dist/auth/cached-token.d.ts +45 -0
- package/dist/auth/cached-token.d.ts.map +1 -0
- package/dist/auth/cached-token.js +53 -0
- package/dist/auth/cached-token.js.map +1 -0
- package/dist/auth/es256.d.ts +23 -0
- package/dist/auth/es256.d.ts.map +1 -0
- package/dist/auth/es256.js +27 -0
- package/dist/auth/es256.js.map +1 -0
- package/dist/auth/index.d.ts +2 -0
- package/dist/auth/index.d.ts.map +1 -1
- package/dist/auth/index.js +2 -0
- package/dist/auth/index.js.map +1 -1
- package/dist/concurrency/index.d.ts +1 -0
- package/dist/concurrency/index.d.ts.map +1 -1
- package/dist/concurrency/index.js +1 -0
- package/dist/concurrency/index.js.map +1 -1
- package/dist/concurrency/single-flight.d.ts +48 -0
- package/dist/concurrency/single-flight.d.ts.map +1 -0
- package/dist/concurrency/single-flight.js +76 -0
- package/dist/concurrency/single-flight.js.map +1 -0
- package/dist/config/index.d.ts +36 -0
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +41 -0
- package/dist/config/index.js.map +1 -1
- package/dist/dates/index.d.ts +28 -0
- package/dist/dates/index.d.ts.map +1 -1
- package/dist/dates/index.js +48 -0
- package/dist/dates/index.js.map +1 -1
- package/dist/errors/index.d.ts +28 -3
- package/dist/errors/index.d.ts.map +1 -1
- package/dist/errors/index.js +52 -4
- package/dist/errors/index.js.map +1 -1
- package/dist/fetchproxy/index.d.ts +26 -1
- package/dist/fetchproxy/index.d.ts.map +1 -1
- package/dist/fetchproxy/index.js +38 -13
- package/dist/fetchproxy/index.js.map +1 -1
- package/dist/fs/index.d.ts +1 -0
- package/dist/fs/index.d.ts.map +1 -1
- package/dist/fs/index.js +1 -0
- package/dist/fs/index.js.map +1 -1
- package/dist/fs/output.d.ts +57 -0
- package/dist/fs/output.d.ts.map +1 -0
- package/dist/fs/output.js +106 -0
- package/dist/fs/output.js.map +1 -0
- package/dist/html/index.d.ts.map +1 -1
- package/dist/html/index.js +58 -6
- package/dist/html/index.js.map +1 -1
- package/dist/http/index.d.ts +35 -0
- package/dist/http/index.d.ts.map +1 -1
- package/dist/http/index.js +30 -2
- package/dist/http/index.js.map +1 -1
- package/dist/http/net-atoms.d.ts +54 -0
- package/dist/http/net-atoms.d.ts.map +1 -0
- package/dist/http/net-atoms.js +76 -0
- package/dist/http/net-atoms.js.map +1 -0
- package/dist/http/response-cache.d.ts +55 -0
- package/dist/http/response-cache.d.ts.map +1 -0
- package/dist/http/response-cache.js +84 -0
- package/dist/http/response-cache.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/response/index.d.ts +17 -0
- package/dist/response/index.d.ts.map +1 -1
- package/dist/response/index.js +28 -0
- package/dist/response/index.js.map +1 -1
- package/dist/scrape/index.d.ts +144 -0
- package/dist/scrape/index.d.ts.map +1 -0
- package/dist/scrape/index.js +460 -0
- package/dist/scrape/index.js.map +1 -0
- package/dist/session/index.d.ts +43 -0
- package/dist/session/index.d.ts.map +1 -1
- package/dist/session/index.js +44 -6
- package/dist/session/index.js.map +1 -1
- package/dist/zod/index.d.ts +1 -0
- package/dist/zod/index.d.ts.map +1 -1
- package/dist/zod/index.js +1 -0
- package/dist/zod/index.js.map +1 -1
- package/dist/zod/parse-lenient.d.ts +36 -0
- package/dist/zod/parse-lenient.d.ts.map +1 -0
- package/dist/zod/parse-lenient.js +37 -0
- package/dist/zod/parse-lenient.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Small HTTP/network atoms shared across the fleet: `Retry-After` parsing
|
|
3
|
+
* (getyourguide / musicbrainz / viator / tripadvisor), host splitting
|
|
4
|
+
* (workday's multi-datacenter subdomains), descriptive User-Agent lines
|
|
5
|
+
* (musicbrainz / getyourguide — rate-limited public APIs block generic UAs),
|
|
6
|
+
* and RFC 6266 `Content-Disposition` filename extraction (ofw downloads).
|
|
7
|
+
*/
|
|
8
|
+
/** Options for {@link parseRetryAfterMs}. */
|
|
9
|
+
export interface ParseRetryAfterOptions {
|
|
10
|
+
/** Delay when the header is missing or unparseable. Defaults to 2000 ms. */
|
|
11
|
+
defaultMs?: number;
|
|
12
|
+
/** Upper bound on the honored delay. Defaults to 30 000 ms. */
|
|
13
|
+
capMs?: number;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Parse an RFC 9110 `Retry-After` header (the delta-seconds form) into a
|
|
17
|
+
* bounded delay in milliseconds. Missing / non-numeric / negative values fall
|
|
18
|
+
* back to `defaultMs` **verbatim** — `capMs` bounds only header-derived delays
|
|
19
|
+
* (an upstream asking for a 10-minute wait shouldn't pin a tool call open, but
|
|
20
|
+
* the caller's own configured fallback is trusted as-is). The HTTP-date form
|
|
21
|
+
* is not parsed (no donor upstream uses it) and falls back to the default.
|
|
22
|
+
*/
|
|
23
|
+
export declare function parseRetryAfterMs(header: string | null | undefined, opts?: ParseRetryAfterOptions): number;
|
|
24
|
+
/** Result of {@link splitHost}. */
|
|
25
|
+
export interface SplitHost {
|
|
26
|
+
/** The registrable-ish domain: the last two labels (or the whole host when ≤ 2 labels). */
|
|
27
|
+
domain: string;
|
|
28
|
+
/** Everything left of the domain, when present (`'wd5'`, `'a.b'`). */
|
|
29
|
+
subdomain?: string;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Split a hostname into `{ domain, subdomain? }` where `domain` is the last
|
|
33
|
+
* two labels: `wd5.myworkday.com` → `{ domain: 'myworkday.com', subdomain:
|
|
34
|
+
* 'wd5' }`. Hosts with ≤ 2 labels return just `{ domain }`. Consolidates
|
|
35
|
+
* workday's `splitHost`, which computes a fetchproxy transport's
|
|
36
|
+
* `domains`/`defaultSubdomain` pair from the configured host. (Naive
|
|
37
|
+
* two-label split — public-suffix domains like `.co.uk` aren't special-cased;
|
|
38
|
+
* no fleet upstream lives on one.)
|
|
39
|
+
*/
|
|
40
|
+
export declare function splitHost(host: string): SplitHost;
|
|
41
|
+
/**
|
|
42
|
+
* Build the descriptive `User-Agent` a rate-limited public API expects:
|
|
43
|
+
* `name/version (+contactUrl)`. MusicBrainz-style APIs block empty/generic
|
|
44
|
+
* UAs, so every fleet client should send one of these.
|
|
45
|
+
*/
|
|
46
|
+
export declare function buildUserAgent(name: string, version: string, contactUrl?: string): string;
|
|
47
|
+
/**
|
|
48
|
+
* Extract the filename from a `Content-Disposition` header, preferring the
|
|
49
|
+
* RFC 6266 `filename*=UTF-8''percent-encoded` form over the quoted
|
|
50
|
+
* `filename="…"` fallback (matching ofw's download path). Returns `undefined`
|
|
51
|
+
* when the header is missing or carries no filename.
|
|
52
|
+
*/
|
|
53
|
+
export declare function parseContentDispositionFilename(header: string | null | undefined): string | undefined;
|
|
54
|
+
//# sourceMappingURL=net-atoms.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"net-atoms.d.ts","sourceRoot":"","sources":["../../src/http/net-atoms.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,6CAA6C;AAC7C,MAAM,WAAW,sBAAsB;IACrC,4EAA4E;IAC5E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EACjC,IAAI,GAAE,sBAA2B,GAChC,MAAM,CAOR;AAED,mCAAmC;AACnC,MAAM,WAAW,SAAS;IACxB,2FAA2F;IAC3F,MAAM,EAAE,MAAM,CAAC;IACf,sEAAsE;IACtE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,CAOjD;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAEzF;AAED;;;;;GAKG;AACH,wBAAgB,+BAA+B,CAC7C,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAChC,MAAM,GAAG,SAAS,CAcpB"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Small HTTP/network atoms shared across the fleet: `Retry-After` parsing
|
|
3
|
+
* (getyourguide / musicbrainz / viator / tripadvisor), host splitting
|
|
4
|
+
* (workday's multi-datacenter subdomains), descriptive User-Agent lines
|
|
5
|
+
* (musicbrainz / getyourguide — rate-limited public APIs block generic UAs),
|
|
6
|
+
* and RFC 6266 `Content-Disposition` filename extraction (ofw downloads).
|
|
7
|
+
*/
|
|
8
|
+
/**
|
|
9
|
+
* Parse an RFC 9110 `Retry-After` header (the delta-seconds form) into a
|
|
10
|
+
* bounded delay in milliseconds. Missing / non-numeric / negative values fall
|
|
11
|
+
* back to `defaultMs` **verbatim** — `capMs` bounds only header-derived delays
|
|
12
|
+
* (an upstream asking for a 10-minute wait shouldn't pin a tool call open, but
|
|
13
|
+
* the caller's own configured fallback is trusted as-is). The HTTP-date form
|
|
14
|
+
* is not parsed (no donor upstream uses it) and falls back to the default.
|
|
15
|
+
*/
|
|
16
|
+
export function parseRetryAfterMs(header, opts = {}) {
|
|
17
|
+
const defaultMs = opts.defaultMs ?? 2000;
|
|
18
|
+
const capMs = opts.capMs ?? 30_000;
|
|
19
|
+
if (header == null)
|
|
20
|
+
return defaultMs;
|
|
21
|
+
const trimmed = header.trim();
|
|
22
|
+
if (!/^\d+$/.test(trimmed))
|
|
23
|
+
return defaultMs;
|
|
24
|
+
return Math.min(Number(trimmed) * 1000, capMs);
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Split a hostname into `{ domain, subdomain? }` where `domain` is the last
|
|
28
|
+
* two labels: `wd5.myworkday.com` → `{ domain: 'myworkday.com', subdomain:
|
|
29
|
+
* 'wd5' }`. Hosts with ≤ 2 labels return just `{ domain }`. Consolidates
|
|
30
|
+
* workday's `splitHost`, which computes a fetchproxy transport's
|
|
31
|
+
* `domains`/`defaultSubdomain` pair from the configured host. (Naive
|
|
32
|
+
* two-label split — public-suffix domains like `.co.uk` aren't special-cased;
|
|
33
|
+
* no fleet upstream lives on one.)
|
|
34
|
+
*/
|
|
35
|
+
export function splitHost(host) {
|
|
36
|
+
const labels = host.split('.');
|
|
37
|
+
if (labels.length <= 2)
|
|
38
|
+
return { domain: host };
|
|
39
|
+
return {
|
|
40
|
+
domain: labels.slice(-2).join('.'),
|
|
41
|
+
subdomain: labels.slice(0, -2).join('.'),
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Build the descriptive `User-Agent` a rate-limited public API expects:
|
|
46
|
+
* `name/version (+contactUrl)`. MusicBrainz-style APIs block empty/generic
|
|
47
|
+
* UAs, so every fleet client should send one of these.
|
|
48
|
+
*/
|
|
49
|
+
export function buildUserAgent(name, version, contactUrl) {
|
|
50
|
+
return contactUrl ? `${name}/${version} (+${contactUrl})` : `${name}/${version}`;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Extract the filename from a `Content-Disposition` header, preferring the
|
|
54
|
+
* RFC 6266 `filename*=UTF-8''percent-encoded` form over the quoted
|
|
55
|
+
* `filename="…"` fallback (matching ofw's download path). Returns `undefined`
|
|
56
|
+
* when the header is missing or carries no filename.
|
|
57
|
+
*/
|
|
58
|
+
export function parseContentDispositionFilename(header) {
|
|
59
|
+
if (!header)
|
|
60
|
+
return undefined;
|
|
61
|
+
const star = /filename\*\s*=\s*(?:UTF-8|utf-8)''([^;]+)/.exec(header);
|
|
62
|
+
if (star?.[1]) {
|
|
63
|
+
try {
|
|
64
|
+
return decodeURIComponent(star[1].trim());
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
// Malformed percent-encoding — fall through to the quoted form.
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
const quoted = /filename\s*=\s*"([^"]+)"/.exec(header);
|
|
71
|
+
if (quoted?.[1])
|
|
72
|
+
return quoted[1];
|
|
73
|
+
const bare = /filename\s*=\s*([^;\s]+)/.exec(header);
|
|
74
|
+
return bare?.[1];
|
|
75
|
+
}
|
|
76
|
+
//# sourceMappingURL=net-atoms.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"net-atoms.js","sourceRoot":"","sources":["../../src/http/net-atoms.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAUH;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAC/B,MAAiC,EACjC,OAA+B,EAAE;IAEjC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,MAAM,CAAC;IACnC,IAAI,MAAM,IAAI,IAAI;QAAE,OAAO,SAAS,CAAC;IACrC,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,SAAS,CAAC;IAC7C,OAAO,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,CAAC;AACjD,CAAC;AAUD;;;;;;;;GAQG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC/B,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAChD,OAAO;QACL,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QAClC,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;KACzC,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,OAAe,EAAE,UAAmB;IAC/E,OAAO,UAAU,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,OAAO,MAAM,UAAU,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,OAAO,EAAE,CAAC;AACnF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,+BAA+B,CAC7C,MAAiC;IAEjC,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAC9B,MAAM,IAAI,GAAG,2CAA2C,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACtE,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACd,IAAI,CAAC;YACH,OAAO,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,gEAAgE;QAClE,CAAC;IACH,CAAC;IACD,MAAM,MAAM,GAAG,0BAA0B,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvD,IAAI,MAAM,EAAE,CAAC,CAAC,CAAC;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,0BAA0B,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrD,OAAO,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC"}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `createResponseCache` — bounded, tiered-TTL, in-memory response cache for
|
|
3
|
+
* billed / rate-limited APIs.
|
|
4
|
+
*
|
|
5
|
+
* Hoists the cache triplicated across flightaware / viator / tripadvisor:
|
|
6
|
+
* per-key `Map` of `{ expiresAt, value }`, a short `dynamic` tier for live data
|
|
7
|
+
* and a long `static` tier for reference data (airports, canonical lookups),
|
|
8
|
+
* a hard entry bound with expired-first-then-oldest eviction, and an
|
|
9
|
+
* injectable clock for deterministic tests. Pair the TTLs with
|
|
10
|
+
* `readTtlMsEnv('<SVC>_CACHE_TTL', …)` / `readTtlMsEnv('<SVC>_STATIC_CACHE_TTL', …)`.
|
|
11
|
+
*
|
|
12
|
+
* Writes must never be cached — only route reads through this.
|
|
13
|
+
*/
|
|
14
|
+
/** Default maximum number of live entries (the donor fleets' shared bound). */
|
|
15
|
+
export declare const RESPONSE_CACHE_MAX_ENTRIES = 256;
|
|
16
|
+
/** Options for {@link createResponseCache}. */
|
|
17
|
+
export interface ResponseCacheOptions {
|
|
18
|
+
/**
|
|
19
|
+
* TTL per tier, in milliseconds. `dynamic` is the default tier; add any
|
|
20
|
+
* named tiers you need (the fleet convention is `static` for reference
|
|
21
|
+
* data). A tier with TTL `0` never stores — caching disabled.
|
|
22
|
+
*/
|
|
23
|
+
ttlMs: {
|
|
24
|
+
dynamic: number;
|
|
25
|
+
} & Record<string, number>;
|
|
26
|
+
/** Max live entries before eviction. Defaults to {@link RESPONSE_CACHE_MAX_ENTRIES}. */
|
|
27
|
+
maxEntries?: number;
|
|
28
|
+
/** Injectable clock (defaults to `Date.now`) — for tests. */
|
|
29
|
+
now?: () => number;
|
|
30
|
+
}
|
|
31
|
+
/** The cache returned by {@link createResponseCache}. */
|
|
32
|
+
export interface ResponseCache<V = unknown> {
|
|
33
|
+
/**
|
|
34
|
+
* The cached value for `key`, or `undefined` on miss/expiry. Tiers matter at
|
|
35
|
+
* WRITE time only — the TTL is baked into the entry by {@link set} /
|
|
36
|
+
* {@link fetchThrough} — so lookups take no tier.
|
|
37
|
+
*/
|
|
38
|
+
get(key: string): V | undefined;
|
|
39
|
+
/** Store `value` for `key` under `tier`'s TTL (no-op when that TTL is 0). */
|
|
40
|
+
set(key: string, value: V, tier?: string): void;
|
|
41
|
+
/** Cache-through read: return the cached value or run `load` and cache it. */
|
|
42
|
+
fetchThrough(key: string, load: () => Promise<V>, tier?: string): Promise<V>;
|
|
43
|
+
/** Drop everything. */
|
|
44
|
+
clear(): void;
|
|
45
|
+
/** Number of entries currently held (including not-yet-swept expired ones). */
|
|
46
|
+
readonly size: number;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Build a bounded in-memory TTL cache with named tiers. Keys are caller-chosen
|
|
50
|
+
* — the donors key on the full request path (and body, for POST-read APIs like
|
|
51
|
+
* viator). Eviction on insert when full: expired entries first, then the
|
|
52
|
+
* oldest by insertion order.
|
|
53
|
+
*/
|
|
54
|
+
export declare function createResponseCache<V = unknown>(opts: ResponseCacheOptions): ResponseCache<V>;
|
|
55
|
+
//# sourceMappingURL=response-cache.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"response-cache.d.ts","sourceRoot":"","sources":["../../src/http/response-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,+EAA+E;AAC/E,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAE9C,+CAA+C;AAC/C,MAAM,WAAW,oBAAoB;IACnC;;;;OAIG;IACH,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpD,wFAAwF;IACxF,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,6DAA6D;IAC7D,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;CACpB;AAED,yDAAyD;AACzD,MAAM,WAAW,aAAa,CAAC,CAAC,GAAG,OAAO;IACxC;;;;OAIG;IACH,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,GAAG,SAAS,CAAC;IAChC,6EAA6E;IAC7E,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChD,8EAA8E;IAC9E,YAAY,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAC7E,uBAAuB;IACvB,KAAK,IAAI,IAAI,CAAC;IACd,+EAA+E;IAC/E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,CAAC,GAAG,OAAO,EAAE,IAAI,EAAE,oBAAoB,GAAG,aAAa,CAAC,CAAC,CAAC,CA0D7F"}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `createResponseCache` — bounded, tiered-TTL, in-memory response cache for
|
|
3
|
+
* billed / rate-limited APIs.
|
|
4
|
+
*
|
|
5
|
+
* Hoists the cache triplicated across flightaware / viator / tripadvisor:
|
|
6
|
+
* per-key `Map` of `{ expiresAt, value }`, a short `dynamic` tier for live data
|
|
7
|
+
* and a long `static` tier for reference data (airports, canonical lookups),
|
|
8
|
+
* a hard entry bound with expired-first-then-oldest eviction, and an
|
|
9
|
+
* injectable clock for deterministic tests. Pair the TTLs with
|
|
10
|
+
* `readTtlMsEnv('<SVC>_CACHE_TTL', …)` / `readTtlMsEnv('<SVC>_STATIC_CACHE_TTL', …)`.
|
|
11
|
+
*
|
|
12
|
+
* Writes must never be cached — only route reads through this.
|
|
13
|
+
*/
|
|
14
|
+
/** Default maximum number of live entries (the donor fleets' shared bound). */
|
|
15
|
+
export const RESPONSE_CACHE_MAX_ENTRIES = 256;
|
|
16
|
+
/**
|
|
17
|
+
* Build a bounded in-memory TTL cache with named tiers. Keys are caller-chosen
|
|
18
|
+
* — the donors key on the full request path (and body, for POST-read APIs like
|
|
19
|
+
* viator). Eviction on insert when full: expired entries first, then the
|
|
20
|
+
* oldest by insertion order.
|
|
21
|
+
*/
|
|
22
|
+
export function createResponseCache(opts) {
|
|
23
|
+
const now = opts.now ?? Date.now;
|
|
24
|
+
const maxEntries = opts.maxEntries ?? RESPONSE_CACHE_MAX_ENTRIES;
|
|
25
|
+
const store = new Map();
|
|
26
|
+
const ttlFor = (tier) => opts.ttlMs[tier] ?? 0;
|
|
27
|
+
function evictForInsert() {
|
|
28
|
+
if (store.size < maxEntries)
|
|
29
|
+
return;
|
|
30
|
+
// Pass 1: drop everything already expired.
|
|
31
|
+
const t = now();
|
|
32
|
+
for (const [key, entry] of store) {
|
|
33
|
+
if (entry.expiresAt <= t)
|
|
34
|
+
store.delete(key);
|
|
35
|
+
}
|
|
36
|
+
// Pass 2: still full → drop oldest insertions until under the bound.
|
|
37
|
+
while (store.size >= maxEntries) {
|
|
38
|
+
const oldest = store.keys().next();
|
|
39
|
+
if (oldest.done)
|
|
40
|
+
break;
|
|
41
|
+
store.delete(oldest.value);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
function get(key) {
|
|
45
|
+
const entry = store.get(key);
|
|
46
|
+
if (!entry)
|
|
47
|
+
return undefined;
|
|
48
|
+
if (entry.expiresAt <= now()) {
|
|
49
|
+
store.delete(key);
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
return entry.value;
|
|
53
|
+
}
|
|
54
|
+
function set(key, value, tier = 'dynamic') {
|
|
55
|
+
const ttl = ttlFor(tier);
|
|
56
|
+
if (ttl <= 0)
|
|
57
|
+
return; // tier disabled
|
|
58
|
+
// Re-setting an existing key must not trigger eviction of a peer.
|
|
59
|
+
if (!store.has(key))
|
|
60
|
+
evictForInsert();
|
|
61
|
+
else
|
|
62
|
+
store.delete(key); // refresh insertion order
|
|
63
|
+
store.set(key, { expiresAt: now() + ttl, value });
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
get,
|
|
67
|
+
set,
|
|
68
|
+
async fetchThrough(key, load, tier = 'dynamic') {
|
|
69
|
+
const hit = get(key);
|
|
70
|
+
if (hit !== undefined)
|
|
71
|
+
return hit;
|
|
72
|
+
const value = await load();
|
|
73
|
+
set(key, value, tier);
|
|
74
|
+
return value;
|
|
75
|
+
},
|
|
76
|
+
clear() {
|
|
77
|
+
store.clear();
|
|
78
|
+
},
|
|
79
|
+
get size() {
|
|
80
|
+
return store.size;
|
|
81
|
+
},
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
//# sourceMappingURL=response-cache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"response-cache.js","sourceRoot":"","sources":["../../src/http/response-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,+EAA+E;AAC/E,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAC;AAkC9C;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAc,IAA0B;IACzE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC;IACjC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,0BAA0B,CAAC;IACjE,MAAM,KAAK,GAAG,IAAI,GAAG,EAA2C,CAAC;IAEjE,MAAM,MAAM,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE/D,SAAS,cAAc;QACrB,IAAI,KAAK,CAAC,IAAI,GAAG,UAAU;YAAE,OAAO;QACpC,2CAA2C;QAC3C,MAAM,CAAC,GAAG,GAAG,EAAE,CAAC;QAChB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,KAAK,EAAE,CAAC;YACjC,IAAI,KAAK,CAAC,SAAS,IAAI,CAAC;gBAAE,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAC9C,CAAC;QACD,qEAAqE;QACrE,OAAO,KAAK,CAAC,IAAI,IAAI,UAAU,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,MAAM,CAAC,IAAI;gBAAE,MAAM;YACvB,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,SAAS,GAAG,CAAC,GAAW;QACtB,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK;YAAE,OAAO,SAAS,CAAC;QAC7B,IAAI,KAAK,CAAC,SAAS,IAAI,GAAG,EAAE,EAAE,CAAC;YAC7B,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAClB,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,KAAK,CAAC,KAAK,CAAC;IACrB,CAAC;IAED,SAAS,GAAG,CAAC,GAAW,EAAE,KAAQ,EAAE,IAAI,GAAG,SAAS;QAClD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QACzB,IAAI,GAAG,IAAI,CAAC;YAAE,OAAO,CAAC,gBAAgB;QACtC,kEAAkE;QAClE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,cAAc,EAAE,CAAC;;YACjC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,0BAA0B;QAClD,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,OAAO;QACL,GAAG;QACH,GAAG;QACH,KAAK,CAAC,YAAY,CAAC,GAAW,EAAE,IAAsB,EAAE,IAAI,GAAG,SAAS;YACtE,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YACrB,IAAI,GAAG,KAAK,SAAS;gBAAE,OAAO,GAAG,CAAC;YAClC,MAAM,KAAK,GAAG,MAAM,IAAI,EAAE,CAAC;YAC3B,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;YACtB,OAAO,KAAK,CAAC;QACf,CAAC;QACD,KAAK;YACH,KAAK,CAAC,KAAK,EAAE,CAAC;QAChB,CAAC;QACD,IAAI,IAAI;YACN,OAAO,KAAK,CAAC,IAAI,CAAC;QACpB,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/dist/index.d.ts
CHANGED
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC;AACvC,cAAc,kBAAkB,CAAC;AACjC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC;AACvC,cAAc,kBAAkB,CAAC;AACjC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC"}
|
package/dist/index.js
CHANGED
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC;AACvC,cAAc,kBAAkB,CAAC;AACjC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,eAAe,CAAC;AAC9B,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC;AACvC,cAAc,kBAAkB,CAAC;AACjC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,mBAAmB,CAAC"}
|
package/dist/response/index.d.ts
CHANGED
|
@@ -31,4 +31,21 @@ export declare function flattenJsonApi(payload: unknown): unknown;
|
|
|
31
31
|
* `deepMapStringField(data, 'eventDate', dmyToIso)`.
|
|
32
32
|
*/
|
|
33
33
|
export declare function deepMapStringField<T>(value: T, field: string, map: (v: string) => string): T;
|
|
34
|
+
/**
|
|
35
|
+
* Shallow-copy an object, dropping every `undefined`-valued key. `null` and
|
|
36
|
+
* falsy values survive — only `undefined` means "not provided".
|
|
37
|
+
*
|
|
38
|
+
* Consolidates the byte-identical `compact()` (skylight) / `prune()` (viator /
|
|
39
|
+
* alltrails) helpers the compact-projection tools call dozens of times per repo.
|
|
40
|
+
*/
|
|
41
|
+
export declare function pruneUndefined<T extends Record<string, unknown>>(obj: T): Partial<T>;
|
|
42
|
+
/**
|
|
43
|
+
* Coerce a maybe-array to an array: an array passes through (same reference), a
|
|
44
|
+
* bare value is wrapped, and `null`/`undefined` become `[]`.
|
|
45
|
+
*
|
|
46
|
+
* Consolidates the `toArray` twins in canvas-parent / infinitecampus — both
|
|
47
|
+
* guard XML→JSON serializers where a single item arrives as a bare object
|
|
48
|
+
* rather than a 1-element array.
|
|
49
|
+
*/
|
|
50
|
+
export declare function toArray<T>(value: T | T[] | null | undefined): T[];
|
|
34
51
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/response/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AAIzE;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,cAAc,CAIxD;AAED,sEAAsE;AACtE,eAAO,MAAM,UAAU,mBAAa,CAAC;AAErC,qEAAqE;AACrE,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,CAE1D;AAED,yDAAyD;AACzD,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,cAAc,CAI5E;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,CAK3D;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAoBxD;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,CAAC,CAY5F"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/response/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oCAAoC,CAAC;AAIzE;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,GAAG,cAAc,CAIxD;AAED,sEAAsE;AACtE,eAAO,MAAM,UAAU,mBAAa,CAAC;AAErC,qEAAqE;AACrE,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,CAE1D;AAED,yDAAyD;AACzD,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,cAAc,CAI5E;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,MAAM,GAAG,cAAc,CAK3D;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAoBxD;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,GAAG,CAAC,CAY5F;AAED;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAMpF;AAED;;;;;;;GAOG;AACH,wBAAgB,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,GAAG,SAAS,GAAG,CAAC,EAAE,CAGjE"}
|
package/dist/response/index.js
CHANGED
|
@@ -87,4 +87,32 @@ export function deepMapStringField(value, field, map) {
|
|
|
87
87
|
}
|
|
88
88
|
return value;
|
|
89
89
|
}
|
|
90
|
+
/**
|
|
91
|
+
* Shallow-copy an object, dropping every `undefined`-valued key. `null` and
|
|
92
|
+
* falsy values survive — only `undefined` means "not provided".
|
|
93
|
+
*
|
|
94
|
+
* Consolidates the byte-identical `compact()` (skylight) / `prune()` (viator /
|
|
95
|
+
* alltrails) helpers the compact-projection tools call dozens of times per repo.
|
|
96
|
+
*/
|
|
97
|
+
export function pruneUndefined(obj) {
|
|
98
|
+
const out = {};
|
|
99
|
+
for (const key of Object.keys(obj)) {
|
|
100
|
+
if (obj[key] !== undefined)
|
|
101
|
+
out[key] = obj[key];
|
|
102
|
+
}
|
|
103
|
+
return out;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Coerce a maybe-array to an array: an array passes through (same reference), a
|
|
107
|
+
* bare value is wrapped, and `null`/`undefined` become `[]`.
|
|
108
|
+
*
|
|
109
|
+
* Consolidates the `toArray` twins in canvas-parent / infinitecampus — both
|
|
110
|
+
* guard XML→JSON serializers where a single item arrives as a bare object
|
|
111
|
+
* rather than a 1-element array.
|
|
112
|
+
*/
|
|
113
|
+
export function toArray(value) {
|
|
114
|
+
if (value === null || value === undefined)
|
|
115
|
+
return [];
|
|
116
|
+
return Array.isArray(value) ? value : [value];
|
|
117
|
+
}
|
|
90
118
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/response/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD;;;;GAIG;AACH,MAAM,UAAU,UAAU,CAAC,IAAa;IACtC,OAAO;QACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;KACjE,CAAC;AACJ,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,MAAM,UAAU,GAAG,UAAU,CAAC;AAErC,qEAAqE;AACrE,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;AAC/C,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,WAAW,CAAC,MAAc,EAAE,QAAgB;IAC1D,OAAO;QACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;KACrD,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,OAAe;IACzC,OAAO;QACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;QACzD,OAAO,EAAE,IAAI;KACd,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,OAAgB;IAC7C,MAAM,UAAU,GAAG,CAAC,IAAa,EAAW,EAAE;QAC5C,IAAI,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QAC3D,MAAM,GAAG,GAAG,IAA+B,CAAC;QAC5C,MAAM,KAAK,GAAG,GAAG,CAAC,UAAU,CAAC;QAC7B,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAChD,MAAM,MAAM,GAA4B,EAAE,GAAI,KAAiC,EAAE,CAAC;YAClF,IAAI,IAAI,IAAI,GAAG;gBAAE,MAAM,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,MAAM,IAAI,GAAG;gBAAE,MAAM,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;YAC1C,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC;IACpE,MAAM,IAAI,GAAG,OAAkC,CAAC;IAChD,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC;QAAE,OAAO,OAAO,CAAC;IACtC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IACvB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACrD,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAI,KAAQ,EAAE,KAAa,EAAE,GAA0B;IACvF,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,KAAK,MAAM,IAAI,IAAI,KAAK;YAAE,kBAAkB,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IACjE,CAAC;SAAM,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvD,MAAM,GAAG,GAAG,KAAgC,CAAC;QAC7C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YACnB,IAAI,GAAG,KAAK,KAAK,IAAI,OAAO,CAAC,KAAK,QAAQ;gBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;;gBACzD,kBAAkB,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/response/index.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAEnD;;;;GAIG;AACH,MAAM,UAAU,UAAU,CAAC,IAAa;IACtC,OAAO;QACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;KACjE,CAAC;AACJ,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,MAAM,UAAU,GAAG,UAAU,CAAC;AAErC,qEAAqE;AACrE,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;AAC/C,CAAC;AAED,yDAAyD;AACzD,MAAM,UAAU,WAAW,CAAC,MAAc,EAAE,QAAgB;IAC1D,OAAO;QACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC;KACrD,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,OAAe;IACzC,OAAO;QACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC;QACzD,OAAO,EAAE,IAAI;KACd,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,OAAgB;IAC7C,MAAM,UAAU,GAAG,CAAC,IAAa,EAAW,EAAE;QAC5C,IAAI,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC;QAC3D,MAAM,GAAG,GAAG,IAA+B,CAAC;QAC5C,MAAM,KAAK,GAAG,GAAG,CAAC,UAAU,CAAC;QAC7B,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAChD,MAAM,MAAM,GAA4B,EAAE,GAAI,KAAiC,EAAE,CAAC;YAClF,IAAI,IAAI,IAAI,GAAG;gBAAE,MAAM,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,MAAM,IAAI,GAAG;gBAAE,MAAM,CAAC,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC;YAC1C,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,IAAI,OAAO,KAAK,IAAI,IAAI,OAAO,OAAO,KAAK,QAAQ;QAAE,OAAO,OAAO,CAAC;IACpE,MAAM,IAAI,GAAG,OAAkC,CAAC;IAChD,IAAI,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC;QAAE,OAAO,OAAO,CAAC;IACtC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IACvB,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACrD,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAI,KAAQ,EAAE,KAAa,EAAE,GAA0B;IACvF,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,KAAK,MAAM,IAAI,IAAI,KAAK;YAAE,kBAAkB,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IACjE,CAAC;SAAM,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvD,MAAM,GAAG,GAAG,KAAgC,CAAC;QAC7C,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YACnB,IAAI,GAAG,KAAK,KAAK,IAAI,OAAO,CAAC,KAAK,QAAQ;gBAAE,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;;gBACzD,kBAAkB,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,cAAc,CAAoC,GAAM;IACtE,MAAM,GAAG,GAAe,EAAE,CAAC;IAC3B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAmB,EAAE,CAAC;QACrD,IAAI,GAAG,CAAC,GAAG,CAAC,KAAK,SAAS;YAAE,GAAG,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,OAAO,CAAI,KAAiC;IAC1D,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IACrD,OAAO,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;AAChD,CAAC"}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `scrape` — zero-dependency string/JSON extraction primitives for
|
|
3
|
+
* server-rendered pages.
|
|
4
|
+
*
|
|
5
|
+
* Consolidates the SSR JSON-store extraction stack the fleet re-implemented at
|
|
6
|
+
* least five times: musescore's `store.ts` (entity recovery + string-aware
|
|
7
|
+
* bracket matching), tock's `redux-state.ts` (top-level slice extraction +
|
|
8
|
+
* JS-literal repair) and `parse.ts` (cycle-guarded deep walkers), zillow's
|
|
9
|
+
* `page-props.ts` (`findArrayByShape`), etix's `parse.ts` (JSON-LD / OpenGraph
|
|
10
|
+
* / dataLayer readers), plus the entity-decode/strip-tags helpers duplicated in
|
|
11
|
+
* alltrails / musescore / signupgenius and the anti-XSSI guard strippers in
|
|
12
|
+
* workday / canvas / redfin.
|
|
13
|
+
*
|
|
14
|
+
* Everything here is pure string/JSON work — NO `node-html-parser`. DOM-level
|
|
15
|
+
* scraping (tables, link lists) lives in `@chrischall/mcp-utils/html`, which
|
|
16
|
+
* carries that optional peer dep; this module is safe for the core barrel.
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* Decode HTML character references: decimal (`A`) and hex (`A`)
|
|
20
|
+
* numerics plus the common named entities. `&` is decoded **last** so a
|
|
21
|
+
* double-escaped entity survives exactly one level (`&lt;` → `<`, not
|
|
22
|
+
* `<`) — the ordering both alltrails' `stripHtml` and musescore's `decodeText`
|
|
23
|
+
* depend on when recovering attribute-escaped JSON. Unknown entities pass
|
|
24
|
+
* through untouched.
|
|
25
|
+
*/
|
|
26
|
+
export declare function decodeHtmlEntities(text: string): string;
|
|
27
|
+
/**
|
|
28
|
+
* Strip an HTML **fragment** to readable text: tags → spaces, entities decoded
|
|
29
|
+
* (via {@link decodeHtmlEntities}), whitespace collapsed and trimmed.
|
|
30
|
+
*
|
|
31
|
+
* For a full document where inline `<script>`/`<style>` *content* must not leak
|
|
32
|
+
* into the output, use `extractPlainTextFromHtml` from
|
|
33
|
+
* `@chrischall/mcp-utils/html` — this fragment-level helper does not remove
|
|
34
|
+
* element content, only markup.
|
|
35
|
+
*/
|
|
36
|
+
export declare function stripHtml(html: string): string;
|
|
37
|
+
/**
|
|
38
|
+
* Replace bare JS `undefined` literals with `null` so a scraped JS object
|
|
39
|
+
* literal becomes parseable JSON. String-aware: occurrences inside single- or
|
|
40
|
+
* double-quoted strings are left alone, as are identifiers that merely contain
|
|
41
|
+
* the word. Consolidates tock's `sanitizeJsLiterals`.
|
|
42
|
+
*/
|
|
43
|
+
export declare function sanitizeJsLiterals(text: string): string;
|
|
44
|
+
/**
|
|
45
|
+
* Given `text[start]` on a `{` or `[`, walk the balanced structure — tracking
|
|
46
|
+
* double- AND single-quoted strings and backslash escapes so brackets inside
|
|
47
|
+
* string literals don't terminate it early — and return the **exclusive** end
|
|
48
|
+
* index. Returns `-1` when `start` isn't on an opener or the structure never
|
|
49
|
+
* closes. Consolidates musescore's `matchBracket` and the brace-walk inside
|
|
50
|
+
* tock's `extractReduxSlice` (regex can't do this: the stores nest and escape).
|
|
51
|
+
*/
|
|
52
|
+
export declare function matchBalanced(text: string, start: number): number;
|
|
53
|
+
/** Options for {@link extractJsonAfterMarker}. */
|
|
54
|
+
export interface ExtractJsonOptions {
|
|
55
|
+
/** Run {@link sanitizeJsLiterals} before parsing (JS object literals). */
|
|
56
|
+
sanitize?: boolean;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Find the first of `markers` in `text`, skip to the next `{` or `[`, walk the
|
|
60
|
+
* balanced structure ({@link matchBalanced}), and `JSON.parse` it. Returns
|
|
61
|
+
* `null` on any failure — marker absent, no opener, unbalanced, or invalid
|
|
62
|
+
* JSON — so callers can branch without a try/catch.
|
|
63
|
+
*
|
|
64
|
+
* The array-capable, multi-marker, JS-literal-tolerant generalization of the
|
|
65
|
+
* `/html` subpath's `extractJsonFromHtml` (which is object-only and marker-form
|
|
66
|
+
* fixed). Use this for `window.$REDUX_STATE` slices (tock), `__NEXT_DATA__`
|
|
67
|
+
* analogues, and attribute-escaped stores (run the text through
|
|
68
|
+
* {@link decodeHtmlEntities} first when the blob is entity-escaped).
|
|
69
|
+
*/
|
|
70
|
+
export declare function extractJsonAfterMarker(text: string, markers: string | string[], opts?: ExtractJsonOptions): unknown;
|
|
71
|
+
/**
|
|
72
|
+
* Parse every `<script type="application/ld+json">` block in the page,
|
|
73
|
+
* skipping malformed ones. Returns the parsed values in document order.
|
|
74
|
+
* Consolidates etix's / tripadvisor's ld+json block iteration.
|
|
75
|
+
*
|
|
76
|
+
* Linear-time by construction: `indexOf`-driven tag scanning (find `<script`,
|
|
77
|
+
* find its `>`, find `</script>`), never a `[^>]*>` regex — the previous
|
|
78
|
+
* single mega-regex was O(n³) and its `[^>]*>`-per-opener replacement is still
|
|
79
|
+
* O(n²); string `indexOf` doesn't backtrack, so an unterminated
|
|
80
|
+
* `<script type="application/ld+json"` flood can't pin the event loop.
|
|
81
|
+
*/
|
|
82
|
+
export declare function extractJsonLdBlocks(html: string): unknown[];
|
|
83
|
+
/**
|
|
84
|
+
* Find the first schema.org entity of `@type === type` across the page's
|
|
85
|
+
* JSON-LD blocks — checking each block itself, its array items, its `@graph`
|
|
86
|
+
* items, and each candidate's `mainEntity` (the etix pattern: the Event often
|
|
87
|
+
* hides inside a `WebPage.mainEntity`). Returns the matching node, or `null`.
|
|
88
|
+
*/
|
|
89
|
+
export declare function findJsonLdEntity(html: string, type: string): Record<string, unknown> | null;
|
|
90
|
+
/**
|
|
91
|
+
* Read a `<meta property="og:…" content="…">` value, tolerant of either
|
|
92
|
+
* attribute order (both occur in the wild — etix's `ogContent`) and of
|
|
93
|
+
* `name=`-keyed OpenGraph. The content is entity-decoded. Returns `undefined`
|
|
94
|
+
* when the property is absent.
|
|
95
|
+
*
|
|
96
|
+
* Linear-time by construction: `indexOf`-driven `<meta …>` scanning, reading
|
|
97
|
+
* the `property`/`content` attributes WITHIN each single (bounded) tag — never
|
|
98
|
+
* a `<meta\s[^>]*PROP[^>]*content…` regex, which backtracked quadratically on a
|
|
99
|
+
* flood of unterminated `<meta property=…` (a ReDoS on hostile scraped HTML).
|
|
100
|
+
*/
|
|
101
|
+
export declare function ogContent(html: string, property: string): string | undefined;
|
|
102
|
+
/**
|
|
103
|
+
* Locate an array inside a drifting JSON container (a `pageProps` /
|
|
104
|
+
* `__NEXT_DATA__` blob): try `directKeys` first, then scan the container's
|
|
105
|
+
* top-level array values and return the first whose FIRST element passes
|
|
106
|
+
* `looksRight`. An empty array at a direct key is trusted (there is no first
|
|
107
|
+
* element to test). Returns `[]` on a miss — never `undefined` — so projections
|
|
108
|
+
* degrade to "no rows" instead of crashing when the upstream renames a key.
|
|
109
|
+
* Consolidates zillow's `page-props.ts#findArrayByShape`.
|
|
110
|
+
*/
|
|
111
|
+
export declare function findArrayByShape<T = unknown>(container: Record<string, unknown>, directKeys: string[], looksRight: (first: unknown) => boolean): T[];
|
|
112
|
+
/**
|
|
113
|
+
* Depth-first, cycle-guarded walk collecting every array whose FIRST element
|
|
114
|
+
* passes `pred`. Recurses into matched arrays too (nested hits are collected).
|
|
115
|
+
* Consolidates tock's `collectArrays` — the "walk for the shape, don't
|
|
116
|
+
* hard-code the path" discipline for SSR stores.
|
|
117
|
+
*/
|
|
118
|
+
export declare function deepCollectArrays(root: unknown, pred: (first: unknown) => boolean): unknown[][];
|
|
119
|
+
/**
|
|
120
|
+
* Depth-first, cycle-guarded search for the first non-array object passing
|
|
121
|
+
* `pred`. Returns `null` when nothing matches. Consolidates tock's
|
|
122
|
+
* `findObject`.
|
|
123
|
+
*/
|
|
124
|
+
export declare function deepFindObject(root: unknown, pred: (obj: Record<string, unknown>) => boolean): Record<string, unknown> | null;
|
|
125
|
+
/**
|
|
126
|
+
* Detect a Cloudflare challenge interstitial by its DEFINITIVE markers only:
|
|
127
|
+
* the `_cf_chl_opt` bootstrap object or a `<title>Just a moment` page title.
|
|
128
|
+
* Deliberately does NOT match `cdn-cgi/challenge-platform` or
|
|
129
|
+
* `challenges.cloudflare.com` references — Cloudflare inlines those on CLEARED
|
|
130
|
+
* pages too, and matching them false-positives on legit pages (the musescore
|
|
131
|
+
* "search works but detail pages fail" bug). Consolidates the byte-identical
|
|
132
|
+
* predicates in musescore and tock.
|
|
133
|
+
*/
|
|
134
|
+
export declare function isCloudflareChallenge(html: string): boolean;
|
|
135
|
+
/**
|
|
136
|
+
* Strip anti-XSSI / anti-JSON-hijacking guard prefixes from a response body so
|
|
137
|
+
* it parses as JSON: `)]}'`, `while(1);`, `for(;;);`, and `{}&&` (redfin's
|
|
138
|
+
* stingray envelope), plus surrounding whitespace, applied repeatedly until no
|
|
139
|
+
* guard remains. Unguarded input passes through unchanged. Consolidates
|
|
140
|
+
* workday's `stripJsonGuard`, canvas-parent's `while(1);` strip, and redfin's
|
|
141
|
+
* `stripStingrayPrefix`.
|
|
142
|
+
*/
|
|
143
|
+
export declare function stripJsonGuard(text: string): string;
|
|
144
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/scrape/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAeH;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQvD;AAiBD;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAK9C;AAQD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CA+BvD;AAED;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAyBjE;AAED,kDAAkD;AAClD,MAAM,WAAW,kBAAkB;IACjC,0EAA0E;IAC1E,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,EAC1B,IAAI,GAAE,kBAAuB,GAC5B,OAAO,CA4BT;AA4BD;;;;;;;;;;GAUG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,EAAE,CAsB3D;AASD;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAe3F;AAOD;;;;;;;;;;GAUG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAgB5E;AAMD;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,CAAC,GAAG,OAAO,EAC1C,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAClC,UAAU,EAAE,MAAM,EAAE,EACpB,UAAU,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,GACtC,CAAC,EAAE,CASL;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,GAAG,OAAO,EAAE,EAAE,CAgB/F;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,IAAI,EAAE,OAAO,EACb,IAAI,EAAE,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,GAC9C,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAsBhC;AAMD;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAE3D;AAOD;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CASnD"}
|