@steve31415/baselib 3.56.0 → 3.57.0

package/README.md

@@ -49,7 +49,8 @@ job does not fast-forward the checkout under the build.
 `twitter` provides Mirror-first tweet lookup, TwitterAPI.io batch/search/
 article calls, and thread assembly through direct reply edges. It includes
 one level of quotes, article bodies and URL expansions. Use one shared
-`MirrorTweetClient(mirrorUrl, serviceFetch)` and `TwitterApiIoClient(key)`
+`MirrorTweetClient(mirrorUrl, serviceFetch, { logger })` (the logger gets
+the WARN when a refused query is retried) and `TwitterApiIoClient(key)`
 per run; `assembleThread(id, { api, mirror, ancestors: 'all' })` returns
 role-tagged tweets (with `links`: expanded URLs and link-card text), source
 counts and warnings; add `replies: { rule: 'digest', followed }` for Digest's

package/dist/front-end-retry.d.ts

@@ -0,0 +1,28 @@
+import type { Logger } from './log-core.js';
+/** The statuses retried. A 500 page is left out: Cloud Run also sends one
+ *  when the app crashes mid-request, and that is not worth a second try. */
+export declare const FRONT_END_REFUSAL_STATUSES: ReadonlySet<number>;
+/** Pause before the one retry. Each attempt keeps whatever timeout `send`
+ *  gives it, so a caller's worst case is two of its timeouts plus this. */
+export declare const FRONT_END_RETRY_DELAY_MS = 1000;
+/** True when `status` and `body` are Google's front end refusing the request
+ *  (a 502/503/504 HTML page), not the app answering. */
+export declare function isFrontEndRefusal(status: number, body: string): boolean;
+export interface FrontEndRetryOptions {
+    logger: Pick<Logger, 'warn'>;
+    /** The WARN logged before the retry, e.g. 'mirror_query_retry'. */
+    event: string;
+    /** Extra fields for that WARN (the caller's label, path, page). It adds
+     *  status, body (first 200 chars) and elapsedMs of the refused attempt. */
+    meta?: Record<string, unknown>;
+    /** Default FRONT_END_RETRY_DELAY_MS. */
+    delayMs?: number;
+}
+/** Send one idempotent request; when Google's front end refuses it
+ *  (isFrontEndRefusal), log a WARN and send it once more after the delay.
+ *  `send` builds the request afresh each time, timeout signal included.
+ *  Returns the last Response as `send` produced it: any failure other than a
+ *  first refusal, and a second refusal, come back with the body unread for
+ *  the caller's own ERROR path. A thrown error (network, timeout) is not
+ *  retried here. */
+export declare function withFrontEndRetry(send: () => Promise<Response>, opts: FrontEndRetryOptions): Promise<Response>;

package/dist/front-end-retry.js

@@ -0,0 +1,50 @@
+// One retry for a request Google's front end refused without passing it to
+// the app (PW KNOWN_LOG_PATTERNS.md, "Cloud Run's front end answers a 5xx
+// while the instance stays up"). The load balancer or Cloud Run's serving
+// front end answers with its own HTML error page and the target's Cloud Run
+// request log has no entry: 2026-11-06 12:00:01Z, Digest's Mirror tweet
+// query got a 502 page after 8.93 s and its whole sync failed at ERROR. The
+// LB re-sends a bodiless request on a 502 itself, so it is mostly POSTs with
+// a body (Mirror's record queries) that surface the refusal.
+//
+// Only for reads and other idempotent requests. That a refused request never
+// reached the app is inferred from the missing Cloud Run entry, not
+// guaranteed, so a write that must not run twice is never sent through here.
+/** The statuses retried. A 500 page is left out: Cloud Run also sends one
+ *  when the app crashes mid-request, and that is not worth a second try. */
+export const FRONT_END_REFUSAL_STATUSES = new Set([502, 503, 504]);
+/** Google's error page; an app's own error is JSON or plain text. */
+const GOOGLE_ERROR_PAGE = /^\s*(<!doctype html|<html)/i;
+/** Pause before the one retry. Each attempt keeps whatever timeout `send`
+ *  gives it, so a caller's worst case is two of its timeouts plus this. */
+export const FRONT_END_RETRY_DELAY_MS = 1_000;
+/** True when `status` and `body` are Google's front end refusing the request
+ *  (a 502/503/504 HTML page), not the app answering. */
+export function isFrontEndRefusal(status, body) {
+    return FRONT_END_REFUSAL_STATUSES.has(status) && GOOGLE_ERROR_PAGE.test(body);
+}
+/** Send one idempotent request; when Google's front end refuses it
+ *  (isFrontEndRefusal), log a WARN and send it once more after the delay.
+ *  `send` builds the request afresh each time, timeout signal included.
+ *  Returns the last Response as `send` produced it: any failure other than a
+ *  first refusal, and a second refusal, come back with the body unread for
+ *  the caller's own ERROR path. A thrown error (network, timeout) is not
+ *  retried here. */
+export async function withFrontEndRetry(send, opts) {
+    const startedAt = Date.now();
+    const first = await send();
+    if (!FRONT_END_REFUSAL_STATUSES.has(first.status))
+        return first;
+    const body = await first.clone().text().catch(() => '');
+    if (!isFrontEndRefusal(first.status, body))
+        return first;
+    await first.body?.cancel().catch(() => { });
+    opts.logger.warn(opts.event, {
+        ...opts.meta,
+        status: first.status,
+        body: body.slice(0, 200),
+        elapsedMs: Date.now() - startedAt,
+    });
+    await new Promise((resolve) => setTimeout(resolve, opts.delayMs ?? FRONT_END_RETRY_DELAY_MS));
+    return send();
+}

package/dist/s2s.d.ts

@@ -1,5 +1,6 @@
 import { OAuth2Client } from 'google-auth-library';
 import { type Logger } from './log-core.js';
+export * from './front-end-retry.js';
 /** The fleet coding agent's service-account email, read from AGENT_SA — set at
  * deploy time by each service (SECURITY.md: agents have full access, so
  * grantAllows always admits this identity). Env-driven rather than a baked-in

package/dist/s2s.js

@@ -18,6 +18,8 @@ import { createHash } from 'node:crypto';
 import { authMode } from './config.js';
 import { truncate } from './log-core.js';
 import { fleetFetch } from './fleet-dns.js';
+// The one-retry wrapper for a caller's idempotent serviceFetch requests.
+export * from './front-end-retry.js';
 /** The fleet coding agent's service-account email, read from AGENT_SA — set at
  * deploy time by each service (SECURITY.md: agents have full access, so
  * grantAllows always admits this identity). Env-driven rather than a baked-in

package/dist/twitter-mirror.d.ts

@@ -1,4 +1,5 @@
 import type { serviceFetch } from './s2s.js';
+import type { Logger } from './log-core.js';
 import { type TwitterApiIoTweet, type TweetEntityUrl } from './twitter-client.js';
 export interface MirrorTweetRecord {
     tweet_id: string;
@@ -36,13 +37,19 @@ export declare const MIRROR_CONVERSATION_MAX_PAGES = 10;
 /** Shared per run: batch prefetch seeds the id cache, conversation requests
  * are coalesced, and failed requests are evicted so a later link can retry.
  * Conversations follow Mirror's keyset cursor (record_timestamp DESC, id
- * DESC) while `has_more`, bounded by MIRROR_CONVERSATION_MAX_PAGES. */
+ * DESC) while `has_more`, bounded by MIRROR_CONVERSATION_MAX_PAGES.
+ * A query Google's front end refuses is sent once more (withFrontEndRetry:
+ * every query is a read); pass `logger` so the retry logs its WARN
+ * (`mirror_tweets_query_retry`). */
 export declare class MirrorTweetClient implements MirrorTwitterClient {
     private readonly mirrorUrl;
     private readonly fetchFn;
+    private readonly opts;
     private readonly byId;
     private readonly conversations;
-    constructor(mirrorUrl: string, fetchFn: typeof serviceFetch);
+    constructor(mirrorUrl: string, fetchFn: typeof serviceFetch, opts?: {
+        logger?: Pick<Logger, 'warn'>;
+    });
     private query;
     getTweetsByIds(ids: string[]): Promise<TwitterApiIoTweet[]>;
     getConversation(id: string): Promise<MirrorConversation>;

package/dist/twitter-mirror.js

@@ -1,3 +1,4 @@
+import { withFrontEndRetry } from './front-end-retry.js';
 import { assertTweetId } from './twitter-client.js';
 /** Mirror snapshots store URL expansions under `url`, without the original
  * t.co token. Keep both shapes; rendering handles the positional fallback. */
@@ -37,23 +38,30 @@ export const MIRROR_CONVERSATION_MAX_PAGES = 10;
 /** Shared per run: batch prefetch seeds the id cache, conversation requests
  * are coalesced, and failed requests are evicted so a later link can retry.
  * Conversations follow Mirror's keyset cursor (record_timestamp DESC, id
- * DESC) while `has_more`, bounded by MIRROR_CONVERSATION_MAX_PAGES. */
+ * DESC) while `has_more`, bounded by MIRROR_CONVERSATION_MAX_PAGES.
+ * A query Google's front end refuses is sent once more (withFrontEndRetry:
+ * every query is a read); pass `logger` so the retry logs its WARN
+ * (`mirror_tweets_query_retry`). */
 export class MirrorTweetClient {
     mirrorUrl;
     fetchFn;
+    opts;
     byId = new Map();
     conversations = new Map();
-    constructor(mirrorUrl, fetchFn) {
+    constructor(mirrorUrl, fetchFn, opts = {}) {
         this.mirrorUrl = mirrorUrl;
         this.fetchFn = fetchFn;
+        this.opts = opts;
     }
     async query(filter, cursor) {
-        const response = await this.fetchFn(`${this.mirrorUrl.replace(/\/$/, '')}/api/records/tweets/query`, {
-            audience: this.mirrorUrl.replace(/\/$/, ''), method: 'POST',
+        const base = this.mirrorUrl.replace(/\/$/, '');
+        const body = JSON.stringify({ filter_sql: filter, columns: '*', limit: MIRROR_PAGE_LIMIT, ...(cursor ? { cursor } : {}) });
+        const response = await withFrontEndRetry(() => this.fetchFn(`${base}/api/records/tweets/query`, {
+            audience: base, method: 'POST',
             headers: { 'content-type': 'application/json' },
-            body: JSON.stringify({ filter_sql: filter, columns: '*', limit: MIRROR_PAGE_LIMIT, ...(cursor ? { cursor } : {}) }),
+            body,
             signal: AbortSignal.timeout(30_000),
-        });
+        }), { logger: this.opts.logger ?? { warn: () => { } }, event: 'mirror_tweets_query_retry', meta: { filter: filter.slice(0, 200), cursor: cursor ?? null } });
         if (!response.ok)
             throw new Error(`Mirror tweets query HTTP ${response.status}`);
         const data = await response.json();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steve31415/baselib",
-  "version": "3.56.0",
+  "version": "3.57.0",
   "description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
   "type": "module",
   "license": "MIT",