@steve31415/baselib 3.55.0 → 3.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/auth.d.ts +2 -1
- package/dist/auth.js +8 -1
- package/dist/fleet-dns.d.ts +52 -0
- package/dist/fleet-dns.js +164 -0
- package/dist/front-end-retry.d.ts +28 -0
- package/dist/front-end-retry.js +50 -0
- package/dist/s2s.d.ts +3 -1
- package/dist/s2s.js +6 -2
- package/dist/twitter-mirror.d.ts +9 -2
- package/dist/twitter-mirror.js +14 -6
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -49,7 +49,8 @@ job does not fast-forward the checkout under the build.
|
|
|
49
49
|
`twitter` provides Mirror-first tweet lookup, TwitterAPI.io batch/search/
|
|
50
50
|
article calls, and thread assembly through direct reply edges. It includes
|
|
51
51
|
one level of quotes, article bodies and URL expansions. Use one shared
|
|
52
|
-
`MirrorTweetClient(mirrorUrl, serviceFetch)`
|
|
52
|
+
`MirrorTweetClient(mirrorUrl, serviceFetch, { logger })` (the logger gets
|
|
53
|
+
the WARN when a refused query is retried) and `TwitterApiIoClient(key)`
|
|
53
54
|
per run; `assembleThread(id, { api, mirror, ancestors: 'all' })` returns
|
|
54
55
|
role-tagged tweets (with `links`: expanded URLs and link-card text), source
|
|
55
56
|
counts and warnings; add `replies: { rule: 'digest', followed }` for Digest's
|
package/dist/auth.d.ts
CHANGED
|
@@ -28,7 +28,8 @@ declare module 'hono' {
|
|
|
28
28
|
export interface RequireIdentityOptions extends S2sOptions {
|
|
29
29
|
/** Base URL of the auth service. */
|
|
30
30
|
authUrl?: string;
|
|
31
|
-
/** Injectable for tests (
|
|
31
|
+
/** Injectable for tests (default: fetch with the auth host's address
|
|
32
|
+
* cached in-process, fleet-dns.ts). */
|
|
32
33
|
whoamiFetch?: typeof fetch;
|
|
33
34
|
logger?: Logger;
|
|
34
35
|
/** Distinguishes API requests (401/403 JSON) from page requests (redirect
|
package/dist/auth.js
CHANGED
|
@@ -31,6 +31,7 @@ import { createMiddleware } from 'hono/factory';
|
|
|
31
31
|
import { authMode } from './config.js';
|
|
32
32
|
import { truncate } from './log-core.js';
|
|
33
33
|
import { RateLimiter, resolveS2sCaller } from './s2s.js';
|
|
34
|
+
import { fleetAddressState, fleetFetch } from './fleet-dns.js';
|
|
34
35
|
export const DEFAULT_AUTH_URL = 'https://auth.apps.snewman.net';
|
|
35
36
|
export const SESSION_COOKIE = 'pw_session';
|
|
36
37
|
/** The `code` in a 403 JSON body when a signed-in user is refused by the
|
|
@@ -45,7 +46,8 @@ function parseRole(value) {
|
|
|
45
46
|
}
|
|
46
47
|
export function requireIdentity(opts) {
|
|
47
48
|
const authUrl = (opts.authUrl ?? DEFAULT_AUTH_URL).replace(/\/$/, '');
|
|
48
|
-
const whoamiFetch = opts.whoamiFetch ??
|
|
49
|
+
const whoamiFetch = opts.whoamiFetch ?? fleetFetch;
|
|
50
|
+
const authHost = new URL(authUrl).hostname;
|
|
49
51
|
const hitMs = opts.cacheTtl?.hitMs ?? 600_000;
|
|
50
52
|
const missMs = opts.cacheTtl?.missMs ?? 30_000;
|
|
51
53
|
const policy = opts.userAccess ?? 'owner';
|
|
@@ -155,6 +157,10 @@ export function requireIdentity(opts) {
|
|
|
155
157
|
async function callWhoami(c, key, cached, cookieHeader, joiners) {
|
|
156
158
|
const started = Date.now();
|
|
157
159
|
const cacheState = !cached ? 'absent' : cached.user ? 'expired-user' : 'expired-unauthenticated';
|
|
160
|
+
// Whether this call must look up the auth host's address (absent) or has
|
|
161
|
+
// it cached in-process (fresh, or stale and refreshing in the
|
|
162
|
+
// background); undefined for a non-fleet authUrl.
|
|
163
|
+
const authAddress = fleetAddressState(authHost);
|
|
158
164
|
const callFields = {
|
|
159
165
|
cacheState,
|
|
160
166
|
...(cached ? { entryAgeMs: started - cached.filled } : {}),
|
|
@@ -166,6 +172,7 @@ export function requireIdentity(opts) {
|
|
|
166
172
|
path: truncate(c.req.path, 200),
|
|
167
173
|
upgrade: (c.req.header('upgrade') ?? '').toLowerCase() === 'websocket',
|
|
168
174
|
cookieCount: cookieHeader.split(';').filter((part) => part.includes('=')).length,
|
|
175
|
+
...(authAddress ? { authAddress } : {}),
|
|
169
176
|
};
|
|
170
177
|
hitsSinceLastCall = 0;
|
|
171
178
|
callCount++;
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { type LookupAddress } from 'node:dns';
|
|
2
|
+
import type { LookupFunction } from 'node:net';
|
|
3
|
+
import { Agent } from 'undici';
|
|
4
|
+
export declare const FLEET_DOMAIN_SUFFIX = ".apps.snewman.net";
|
|
5
|
+
/** How long a resolved fleet address is used before a background refresh.
|
|
6
|
+
* Long, because the address is a reserved static IP and a change is
|
|
7
|
+
* healed by the connect-failure path anyway. */
|
|
8
|
+
export declare const FLEET_ADDRESS_TTL_MS: number;
|
|
9
|
+
export declare function isFleetHost(hostname: string): boolean;
|
|
10
|
+
export type FleetAddressState = 'fresh' | 'stale' | 'absent';
|
|
11
|
+
export interface FleetAddressCacheOptions {
|
|
12
|
+
/** All addresses for a fleet host (default: dns.promises.lookup, all). */
|
|
13
|
+
resolve?: (hostname: string) => Promise<LookupAddress[]>;
|
|
14
|
+
/** Non-fleet hosts, and families the cached answer lacks (default dns.lookup). */
|
|
15
|
+
systemLookup?: LookupFunction;
|
|
16
|
+
ttlMs?: number;
|
|
17
|
+
now?: () => number;
|
|
18
|
+
/** Called when a connect failure drops an entry. */
|
|
19
|
+
onInvalidate?: (e: {
|
|
20
|
+
hostname: string;
|
|
21
|
+
addresses: string[];
|
|
22
|
+
reason: string;
|
|
23
|
+
}) => void;
|
|
24
|
+
}
|
|
25
|
+
export declare class FleetAddressCache {
|
|
26
|
+
private readonly entries;
|
|
27
|
+
private readonly inflight;
|
|
28
|
+
private readonly resolve;
|
|
29
|
+
private readonly systemLookup;
|
|
30
|
+
private readonly ttlMs;
|
|
31
|
+
private readonly now;
|
|
32
|
+
private readonly onInvalidate?;
|
|
33
|
+
constructor(opts?: FleetAddressCacheOptions);
|
|
34
|
+
/** net.connect's `lookup` option. */
|
|
35
|
+
readonly lookup: LookupFunction;
|
|
36
|
+
/** undefined for a non-fleet host. */
|
|
37
|
+
state(hostname: string): FleetAddressState | undefined;
|
|
38
|
+
/** Forget a host's address, so its next connection resolves afresh. */
|
|
39
|
+
invalidate(hostname: string, reason: string): void;
|
|
40
|
+
private refresh;
|
|
41
|
+
private answer;
|
|
42
|
+
}
|
|
43
|
+
/** An undici Agent whose connections resolve through `cache` and drop the
|
|
44
|
+
* cached address when a connection fails for an address reason. */
|
|
45
|
+
export declare function createFleetAgent(cache: FleetAddressCache): Agent;
|
|
46
|
+
/** The dispatcher baselib's server-to-server calls use. */
|
|
47
|
+
export declare function fleetDispatcher(): Agent;
|
|
48
|
+
/** The process-wide cache's state for `hostname` (undefined: not a fleet host). */
|
|
49
|
+
export declare function fleetAddressState(hostname: string): FleetAddressState | undefined;
|
|
50
|
+
/** fetch() through fleetDispatcher(). `dispatcher` is Node's (undici's)
|
|
51
|
+
* extension of RequestInit, absent from the DOM type. */
|
|
52
|
+
export declare const fleetFetch: typeof fetch;
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
// An in-process address cache for fleet hosts (*.apps.snewman.net), used by
|
|
2
|
+
// the server-to-server calls baselib makes: the session check against auth
|
|
3
|
+
// (whoami) and serviceFetch.
|
|
4
|
+
//
|
|
5
|
+
// Why: on Cloud Run, Node's dns.lookup goes through getaddrinfo to the
|
|
6
|
+
// metadata-server resolver, which keeps an answer for at most ~300 s
|
|
7
|
+
// whatever the record's TTL (the fleet's A records carry 3600 s since
|
|
8
|
+
// 2026-10-30; it made no difference). An infrequent caller therefore paid a
|
|
9
|
+
// ~60 ms lookup on almost every call: whoami took 113 ms p50 when the app's
|
|
10
|
+
// previous call was 300-3600 s earlier against 52 ms under 300 s, 79 % of
|
|
11
|
+
// calls in the slow bucket (2026-10-30 → 11-05,
|
|
12
|
+
// ~/migration/research/whoami-latency-2026-10-30.md, "Follow-up 2026-11-05").
|
|
13
|
+
//
|
|
14
|
+
// What: every fleet host resolves to the one static load-balancer address,
|
|
15
|
+
// so a resolved answer is kept for FLEET_ADDRESS_TTL_MS and, once older,
|
|
16
|
+
// still served at once while a background lookup refreshes it. A connection
|
|
17
|
+
// that fails in a way an address change would cause (refused, timed out,
|
|
18
|
+
// unreachable) drops the entry, so the next call resolves afresh: an
|
|
19
|
+
// address change heals after one failed call. Only the lookup changes: the
|
|
20
|
+
// URL, Host header and TLS server name (SNI) are still the hostname. Other
|
|
21
|
+
// hosts go straight to dns.lookup, uncached.
|
|
22
|
+
import dns from 'node:dns';
|
|
23
|
+
import { Agent, buildConnector } from 'undici';
|
|
24
|
+
import { processMeta } from './process-meta.js';
|
|
25
|
+
export const FLEET_DOMAIN_SUFFIX = '.apps.snewman.net';
|
|
26
|
+
/** How long a resolved fleet address is used before a background refresh.
|
|
27
|
+
* Long, because the address is a reserved static IP and a change is
|
|
28
|
+
* healed by the connect-failure path anyway. */
|
|
29
|
+
export const FLEET_ADDRESS_TTL_MS = 60 * 60_000;
|
|
30
|
+
/** Connect errors that a stale address would cause. Anything else (a TLS
|
|
31
|
+
* or HTTP failure) says nothing about the address and keeps the entry. */
|
|
32
|
+
const ADDRESS_FAILURE_CODES = new Set(['ECONNREFUSED', 'ETIMEDOUT', 'EHOSTUNREACH', 'ENETUNREACH', 'UND_ERR_CONNECT_TIMEOUT']);
|
|
33
|
+
export function isFleetHost(hostname) {
|
|
34
|
+
return normalize(hostname).endsWith(FLEET_DOMAIN_SUFFIX);
|
|
35
|
+
}
|
|
36
|
+
function normalize(hostname) {
|
|
37
|
+
return hostname.toLowerCase().replace(/\.$/, '');
|
|
38
|
+
}
|
|
39
|
+
export class FleetAddressCache {
|
|
40
|
+
entries = new Map();
|
|
41
|
+
inflight = new Map();
|
|
42
|
+
resolve;
|
|
43
|
+
systemLookup;
|
|
44
|
+
ttlMs;
|
|
45
|
+
now;
|
|
46
|
+
onInvalidate;
|
|
47
|
+
constructor(opts = {}) {
|
|
48
|
+
this.resolve = opts.resolve ?? ((hostname) => dns.promises.lookup(hostname, { all: true }));
|
|
49
|
+
this.systemLookup = opts.systemLookup ?? dns.lookup;
|
|
50
|
+
this.ttlMs = opts.ttlMs ?? FLEET_ADDRESS_TTL_MS;
|
|
51
|
+
this.now = opts.now ?? Date.now;
|
|
52
|
+
this.onInvalidate = opts.onInvalidate;
|
|
53
|
+
}
|
|
54
|
+
/** net.connect's `lookup` option. */
|
|
55
|
+
lookup = (hostname, options, callback) => {
|
|
56
|
+
if (!isFleetHost(hostname))
|
|
57
|
+
return this.systemLookup(hostname, options, callback);
|
|
58
|
+
const key = normalize(hostname);
|
|
59
|
+
const entry = this.entries.get(key);
|
|
60
|
+
if (entry) {
|
|
61
|
+
if (this.now() - entry.resolvedAt >= this.ttlMs)
|
|
62
|
+
this.refresh(key).catch(() => { }); // keep the old answer
|
|
63
|
+
return this.answer(entry, hostname, options, callback);
|
|
64
|
+
}
|
|
65
|
+
this.refresh(key).then((fresh) => this.answer(fresh, hostname, options, callback), (err) => callback(err, options.all ? [] : ''));
|
|
66
|
+
};
|
|
67
|
+
/** undefined for a non-fleet host. */
|
|
68
|
+
state(hostname) {
|
|
69
|
+
if (!isFleetHost(hostname))
|
|
70
|
+
return undefined;
|
|
71
|
+
const entry = this.entries.get(normalize(hostname));
|
|
72
|
+
if (!entry)
|
|
73
|
+
return 'absent';
|
|
74
|
+
return this.now() - entry.resolvedAt < this.ttlMs ? 'fresh' : 'stale';
|
|
75
|
+
}
|
|
76
|
+
/** Forget a host's address, so its next connection resolves afresh. */
|
|
77
|
+
invalidate(hostname, reason) {
|
|
78
|
+
const key = normalize(hostname);
|
|
79
|
+
const entry = this.entries.get(key);
|
|
80
|
+
if (!entry)
|
|
81
|
+
return;
|
|
82
|
+
this.entries.delete(key);
|
|
83
|
+
this.onInvalidate?.({ hostname: key, addresses: entry.addresses.map((a) => a.address), reason });
|
|
84
|
+
}
|
|
85
|
+
// One resolution per host at a time; a failure is never cached.
|
|
86
|
+
refresh(key) {
|
|
87
|
+
let pending = this.inflight.get(key);
|
|
88
|
+
if (!pending) {
|
|
89
|
+
pending = this.resolve(key)
|
|
90
|
+
.then((addresses) => {
|
|
91
|
+
if (addresses.length === 0)
|
|
92
|
+
throw Object.assign(new Error(`no address for ${key}`), { code: 'ENOTFOUND' });
|
|
93
|
+
const entry = { addresses, resolvedAt: this.now() };
|
|
94
|
+
this.entries.set(key, entry);
|
|
95
|
+
return entry;
|
|
96
|
+
})
|
|
97
|
+
.finally(() => this.inflight.delete(key));
|
|
98
|
+
this.inflight.set(key, pending);
|
|
99
|
+
}
|
|
100
|
+
return pending;
|
|
101
|
+
}
|
|
102
|
+
answer(entry, hostname, options, callback) {
|
|
103
|
+
const family = familyNumber(options.family);
|
|
104
|
+
const addresses = family ? entry.addresses.filter((a) => a.family === family) : entry.addresses;
|
|
105
|
+
if (addresses.length === 0)
|
|
106
|
+
return this.systemLookup(hostname, options, callback);
|
|
107
|
+
if (options.all)
|
|
108
|
+
callback(null, addresses);
|
|
109
|
+
else
|
|
110
|
+
callback(null, addresses[0].address, addresses[0].family);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
function familyNumber(family) {
|
|
114
|
+
if (family === 'IPv4')
|
|
115
|
+
return 4;
|
|
116
|
+
if (family === 'IPv6')
|
|
117
|
+
return 6;
|
|
118
|
+
return family ?? 0;
|
|
119
|
+
}
|
|
120
|
+
/** The code of a connect error, looking inside the AggregateError that
|
|
121
|
+
* Node's happy-eyeballs connect raises when every address failed. */
|
|
122
|
+
function addressFailureCode(err) {
|
|
123
|
+
const codes = [err, ...(err.errors ?? [])].map((e) => e?.code);
|
|
124
|
+
return codes.find((c) => typeof c === 'string' && ADDRESS_FAILURE_CODES.has(c));
|
|
125
|
+
}
|
|
126
|
+
/** An undici Agent whose connections resolve through `cache` and drop the
|
|
127
|
+
* cached address when a connection fails for an address reason. */
|
|
128
|
+
export function createFleetAgent(cache) {
|
|
129
|
+
const connect = buildConnector({ lookup: cache.lookup });
|
|
130
|
+
return new Agent({
|
|
131
|
+
connect: (opts, callback) => connect(opts, (...args) => {
|
|
132
|
+
const err = args[0];
|
|
133
|
+
if (err) {
|
|
134
|
+
const code = addressFailureCode(err);
|
|
135
|
+
if (code)
|
|
136
|
+
cache.invalidate(opts.hostname, code);
|
|
137
|
+
}
|
|
138
|
+
callback(...args);
|
|
139
|
+
}),
|
|
140
|
+
});
|
|
141
|
+
}
|
|
142
|
+
let shared;
|
|
143
|
+
function sharedFleet() {
|
|
144
|
+
if (!shared) {
|
|
145
|
+
const cache = new FleetAddressCache({
|
|
146
|
+
// Rare (the load balancer's address would have to change), so worth
|
|
147
|
+
// a line. No logger reaches this module: stdout, like db.ts's backstop.
|
|
148
|
+
onInvalidate: (e) => console.warn('fleet address dropped after connect failure', { ...processMeta(), ...e }),
|
|
149
|
+
});
|
|
150
|
+
shared = { cache, agent: createFleetAgent(cache) };
|
|
151
|
+
}
|
|
152
|
+
return shared;
|
|
153
|
+
}
|
|
154
|
+
/** The dispatcher baselib's server-to-server calls use. */
|
|
155
|
+
export function fleetDispatcher() {
|
|
156
|
+
return sharedFleet().agent;
|
|
157
|
+
}
|
|
158
|
+
/** The process-wide cache's state for `hostname` (undefined: not a fleet host). */
|
|
159
|
+
export function fleetAddressState(hostname) {
|
|
160
|
+
return sharedFleet().cache.state(hostname);
|
|
161
|
+
}
|
|
162
|
+
/** fetch() through fleetDispatcher(). `dispatcher` is Node's (undici's)
|
|
163
|
+
* extension of RequestInit, absent from the DOM type. */
|
|
164
|
+
export const fleetFetch = (input, init) => fetch(input, { ...init, dispatcher: fleetDispatcher() });
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Logger } from './log-core.js';
|
|
2
|
+
/** The statuses retried. A 500 page is left out: Cloud Run also sends one
|
|
3
|
+
* when the app crashes mid-request, and that is not worth a second try. */
|
|
4
|
+
export declare const FRONT_END_REFUSAL_STATUSES: ReadonlySet<number>;
|
|
5
|
+
/** Pause before the one retry. Each attempt keeps whatever timeout `send`
|
|
6
|
+
* gives it, so a caller's worst case is two of its timeouts plus this. */
|
|
7
|
+
export declare const FRONT_END_RETRY_DELAY_MS = 1000;
|
|
8
|
+
/** True when `status` and `body` are Google's front end refusing the request
|
|
9
|
+
* (a 502/503/504 HTML page), not the app answering. */
|
|
10
|
+
export declare function isFrontEndRefusal(status: number, body: string): boolean;
|
|
11
|
+
export interface FrontEndRetryOptions {
|
|
12
|
+
logger: Pick<Logger, 'warn'>;
|
|
13
|
+
/** The WARN logged before the retry, e.g. 'mirror_query_retry'. */
|
|
14
|
+
event: string;
|
|
15
|
+
/** Extra fields for that WARN (the caller's label, path, page). It adds
|
|
16
|
+
* status, body (first 200 chars) and elapsedMs of the refused attempt. */
|
|
17
|
+
meta?: Record<string, unknown>;
|
|
18
|
+
/** Default FRONT_END_RETRY_DELAY_MS. */
|
|
19
|
+
delayMs?: number;
|
|
20
|
+
}
|
|
21
|
+
/** Send one idempotent request; when Google's front end refuses it
|
|
22
|
+
* (isFrontEndRefusal), log a WARN and send it once more after the delay.
|
|
23
|
+
* `send` builds the request afresh each time, timeout signal included.
|
|
24
|
+
* Returns the last Response as `send` produced it: any failure other than a
|
|
25
|
+
* first refusal, and a second refusal, come back with the body unread for
|
|
26
|
+
* the caller's own ERROR path. A thrown error (network, timeout) is not
|
|
27
|
+
* retried here. */
|
|
28
|
+
export declare function withFrontEndRetry(send: () => Promise<Response>, opts: FrontEndRetryOptions): Promise<Response>;
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// One retry for a request Google's front end refused without passing it to
|
|
2
|
+
// the app (PW KNOWN_LOG_PATTERNS.md, "Cloud Run's front end answers a 5xx
|
|
3
|
+
// while the instance stays up"). The load balancer or Cloud Run's serving
|
|
4
|
+
// front end answers with its own HTML error page and the target's Cloud Run
|
|
5
|
+
// request log has no entry: 2026-11-06 12:00:01Z, Digest's Mirror tweet
|
|
6
|
+
// query got a 502 page after 8.93 s and its whole sync failed at ERROR. The
|
|
7
|
+
// LB re-sends a bodiless request on a 502 itself, so it is mostly POSTs with
|
|
8
|
+
// a body (Mirror's record queries) that surface the refusal.
|
|
9
|
+
//
|
|
10
|
+
// Only for reads and other idempotent requests. That a refused request never
|
|
11
|
+
// reached the app is inferred from the missing Cloud Run entry, not
|
|
12
|
+
// guaranteed, so a write that must not run twice is never sent through here.
|
|
13
|
+
/** The statuses retried. A 500 page is left out: Cloud Run also sends one
|
|
14
|
+
* when the app crashes mid-request, and that is not worth a second try. */
|
|
15
|
+
export const FRONT_END_REFUSAL_STATUSES = new Set([502, 503, 504]);
|
|
16
|
+
/** Google's error page; an app's own error is JSON or plain text. */
|
|
17
|
+
const GOOGLE_ERROR_PAGE = /^\s*(<!doctype html|<html)/i;
|
|
18
|
+
/** Pause before the one retry. Each attempt keeps whatever timeout `send`
|
|
19
|
+
* gives it, so a caller's worst case is two of its timeouts plus this. */
|
|
20
|
+
export const FRONT_END_RETRY_DELAY_MS = 1_000;
|
|
21
|
+
/** True when `status` and `body` are Google's front end refusing the request
|
|
22
|
+
* (a 502/503/504 HTML page), not the app answering. */
|
|
23
|
+
export function isFrontEndRefusal(status, body) {
|
|
24
|
+
return FRONT_END_REFUSAL_STATUSES.has(status) && GOOGLE_ERROR_PAGE.test(body);
|
|
25
|
+
}
|
|
26
|
+
/** Send one idempotent request; when Google's front end refuses it
|
|
27
|
+
* (isFrontEndRefusal), log a WARN and send it once more after the delay.
|
|
28
|
+
* `send` builds the request afresh each time, timeout signal included.
|
|
29
|
+
* Returns the last Response as `send` produced it: any failure other than a
|
|
30
|
+
* first refusal, and a second refusal, come back with the body unread for
|
|
31
|
+
* the caller's own ERROR path. A thrown error (network, timeout) is not
|
|
32
|
+
* retried here. */
|
|
33
|
+
export async function withFrontEndRetry(send, opts) {
|
|
34
|
+
const startedAt = Date.now();
|
|
35
|
+
const first = await send();
|
|
36
|
+
if (!FRONT_END_REFUSAL_STATUSES.has(first.status))
|
|
37
|
+
return first;
|
|
38
|
+
const body = await first.clone().text().catch(() => '');
|
|
39
|
+
if (!isFrontEndRefusal(first.status, body))
|
|
40
|
+
return first;
|
|
41
|
+
await first.body?.cancel().catch(() => { });
|
|
42
|
+
opts.logger.warn(opts.event, {
|
|
43
|
+
...opts.meta,
|
|
44
|
+
status: first.status,
|
|
45
|
+
body: body.slice(0, 200),
|
|
46
|
+
elapsedMs: Date.now() - startedAt,
|
|
47
|
+
});
|
|
48
|
+
await new Promise((resolve) => setTimeout(resolve, opts.delayMs ?? FRONT_END_RETRY_DELAY_MS));
|
|
49
|
+
return send();
|
|
50
|
+
}
|
package/dist/s2s.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { OAuth2Client } from 'google-auth-library';
|
|
2
2
|
import { type Logger } from './log-core.js';
|
|
3
|
+
export * from './front-end-retry.js';
|
|
3
4
|
/** The fleet coding agent's service-account email, read from AGENT_SA — set at
|
|
4
5
|
* deploy time by each service (SECURITY.md: agents have full access, so
|
|
5
6
|
* grantAllows always admits this identity). Env-driven rather than a baked-in
|
|
@@ -80,7 +81,8 @@ export declare const ID_TOKEN_MINT_TIMEOUT_MS = 5000;
|
|
|
80
81
|
* (identity from TEST_S2S_IDENTITY, default 'test-service@test'). The token
|
|
81
82
|
* is cached per audience until ID_TOKEN_REFRESH_MARGIN_MS before it
|
|
82
83
|
* expires; concurrent calls share one mint, which is bounded by
|
|
83
|
-
* ID_TOKEN_MINT_TIMEOUT_MS, and `init.signal` also ends the wait for it.
|
|
84
|
+
* ID_TOKEN_MINT_TIMEOUT_MS, and `init.signal` also ends the wait for it.
|
|
85
|
+
* Fleet hosts' addresses are cached in-process (fleet-dns.ts). */
|
|
84
86
|
export declare function serviceFetch(url: string, opts: {
|
|
85
87
|
audience: string;
|
|
86
88
|
} & RequestInit): Promise<Response>;
|
package/dist/s2s.js
CHANGED
|
@@ -17,6 +17,9 @@ import { OAuth2Client } from 'google-auth-library';
|
|
|
17
17
|
import { createHash } from 'node:crypto';
|
|
18
18
|
import { authMode } from './config.js';
|
|
19
19
|
import { truncate } from './log-core.js';
|
|
20
|
+
import { fleetFetch } from './fleet-dns.js';
|
|
21
|
+
// The one-retry wrapper for a caller's idempotent serviceFetch requests.
|
|
22
|
+
export * from './front-end-retry.js';
|
|
20
23
|
/** The fleet coding agent's service-account email, read from AGENT_SA — set at
|
|
21
24
|
* deploy time by each service (SECURITY.md: agents have full access, so
|
|
22
25
|
* grantAllows always admits this identity). Env-driven rather than a baked-in
|
|
@@ -201,7 +204,8 @@ const idTokenMints = new Map();
|
|
|
201
204
|
* (identity from TEST_S2S_IDENTITY, default 'test-service@test'). The token
|
|
202
205
|
* is cached per audience until ID_TOKEN_REFRESH_MARGIN_MS before it
|
|
203
206
|
* expires; concurrent calls share one mint, which is bounded by
|
|
204
|
-
* ID_TOKEN_MINT_TIMEOUT_MS, and `init.signal` also ends the wait for it.
|
|
207
|
+
* ID_TOKEN_MINT_TIMEOUT_MS, and `init.signal` also ends the wait for it.
|
|
208
|
+
* Fleet hosts' addresses are cached in-process (fleet-dns.ts). */
|
|
205
209
|
export async function serviceFetch(url, opts) {
|
|
206
210
|
const { audience, ...init } = opts;
|
|
207
211
|
const headers = new Headers(init.headers);
|
|
@@ -212,7 +216,7 @@ export async function serviceFetch(url, opts) {
|
|
|
212
216
|
const token = await abortable(idTokenFor(audience), init.signal);
|
|
213
217
|
headers.set('authorization', `Bearer ${token}`);
|
|
214
218
|
}
|
|
215
|
-
return
|
|
219
|
+
return fleetFetch(url, { ...init, headers });
|
|
216
220
|
}
|
|
217
221
|
function idTokenFor(audience) {
|
|
218
222
|
const cached = idTokens.get(audience);
|
package/dist/twitter-mirror.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { serviceFetch } from './s2s.js';
|
|
2
|
+
import type { Logger } from './log-core.js';
|
|
2
3
|
import { type TwitterApiIoTweet, type TweetEntityUrl } from './twitter-client.js';
|
|
3
4
|
export interface MirrorTweetRecord {
|
|
4
5
|
tweet_id: string;
|
|
@@ -36,13 +37,19 @@ export declare const MIRROR_CONVERSATION_MAX_PAGES = 10;
|
|
|
36
37
|
/** Shared per run: batch prefetch seeds the id cache, conversation requests
|
|
37
38
|
* are coalesced, and failed requests are evicted so a later link can retry.
|
|
38
39
|
* Conversations follow Mirror's keyset cursor (record_timestamp DESC, id
|
|
39
|
-
* DESC) while `has_more`, bounded by MIRROR_CONVERSATION_MAX_PAGES.
|
|
40
|
+
* DESC) while `has_more`, bounded by MIRROR_CONVERSATION_MAX_PAGES.
|
|
41
|
+
* A query Google's front end refuses is sent once more (withFrontEndRetry:
|
|
42
|
+
* every query is a read); pass `logger` so the retry logs its WARN
|
|
43
|
+
* (`mirror_tweets_query_retry`). */
|
|
40
44
|
export declare class MirrorTweetClient implements MirrorTwitterClient {
|
|
41
45
|
private readonly mirrorUrl;
|
|
42
46
|
private readonly fetchFn;
|
|
47
|
+
private readonly opts;
|
|
43
48
|
private readonly byId;
|
|
44
49
|
private readonly conversations;
|
|
45
|
-
constructor(mirrorUrl: string, fetchFn: typeof serviceFetch
|
|
50
|
+
constructor(mirrorUrl: string, fetchFn: typeof serviceFetch, opts?: {
|
|
51
|
+
logger?: Pick<Logger, 'warn'>;
|
|
52
|
+
});
|
|
46
53
|
private query;
|
|
47
54
|
getTweetsByIds(ids: string[]): Promise<TwitterApiIoTweet[]>;
|
|
48
55
|
getConversation(id: string): Promise<MirrorConversation>;
|
package/dist/twitter-mirror.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { withFrontEndRetry } from './front-end-retry.js';
|
|
1
2
|
import { assertTweetId } from './twitter-client.js';
|
|
2
3
|
/** Mirror snapshots store URL expansions under `url`, without the original
|
|
3
4
|
* t.co token. Keep both shapes; rendering handles the positional fallback. */
|
|
@@ -37,23 +38,30 @@ export const MIRROR_CONVERSATION_MAX_PAGES = 10;
|
|
|
37
38
|
/** Shared per run: batch prefetch seeds the id cache, conversation requests
|
|
38
39
|
* are coalesced, and failed requests are evicted so a later link can retry.
|
|
39
40
|
* Conversations follow Mirror's keyset cursor (record_timestamp DESC, id
|
|
40
|
-
* DESC) while `has_more`, bounded by MIRROR_CONVERSATION_MAX_PAGES.
|
|
41
|
+
* DESC) while `has_more`, bounded by MIRROR_CONVERSATION_MAX_PAGES.
|
|
42
|
+
* A query Google's front end refuses is sent once more (withFrontEndRetry:
|
|
43
|
+
* every query is a read); pass `logger` so the retry logs its WARN
|
|
44
|
+
* (`mirror_tweets_query_retry`). */
|
|
41
45
|
export class MirrorTweetClient {
|
|
42
46
|
mirrorUrl;
|
|
43
47
|
fetchFn;
|
|
48
|
+
opts;
|
|
44
49
|
byId = new Map();
|
|
45
50
|
conversations = new Map();
|
|
46
|
-
constructor(mirrorUrl, fetchFn) {
|
|
51
|
+
constructor(mirrorUrl, fetchFn, opts = {}) {
|
|
47
52
|
this.mirrorUrl = mirrorUrl;
|
|
48
53
|
this.fetchFn = fetchFn;
|
|
54
|
+
this.opts = opts;
|
|
49
55
|
}
|
|
50
56
|
async query(filter, cursor) {
|
|
51
|
-
const
|
|
52
|
-
|
|
57
|
+
const base = this.mirrorUrl.replace(/\/$/, '');
|
|
58
|
+
const body = JSON.stringify({ filter_sql: filter, columns: '*', limit: MIRROR_PAGE_LIMIT, ...(cursor ? { cursor } : {}) });
|
|
59
|
+
const response = await withFrontEndRetry(() => this.fetchFn(`${base}/api/records/tweets/query`, {
|
|
60
|
+
audience: base, method: 'POST',
|
|
53
61
|
headers: { 'content-type': 'application/json' },
|
|
54
|
-
body
|
|
62
|
+
body,
|
|
55
63
|
signal: AbortSignal.timeout(30_000),
|
|
56
|
-
});
|
|
64
|
+
}), { logger: this.opts.logger ?? { warn: () => { } }, event: 'mirror_tweets_query_retry', meta: { filter: filter.slice(0, 200), cursor: cursor ?? null } });
|
|
57
65
|
if (!response.ok)
|
|
58
66
|
throw new Error(`Mirror tweets query HTTP ${response.status}`);
|
|
59
67
|
const data = await response.json();
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@steve31415/baselib",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.57.0",
|
|
4
4
|
"description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -133,6 +133,7 @@
|
|
|
133
133
|
"google-auth-library": "^10.5.0",
|
|
134
134
|
"hono": "^4.7.0",
|
|
135
135
|
"pg": "^8.14.0",
|
|
136
|
+
"undici": "^6.29.1",
|
|
136
137
|
"web-vitals": "^6.1.1",
|
|
137
138
|
"zod": "^4.6.5"
|
|
138
139
|
},
|