@steve31415/baselib 3.36.2 → 3.38.0

package/README.md

@@ -41,7 +41,9 @@ article calls, and thread assembly through direct reply edges. It includes
 one level of quotes, article bodies and URL expansions. Use one shared
 `MirrorTweetClient(mirrorUrl, serviceFetch)` and `TwitterApiIoClient(key)`
 per run; `assembleThread(id, { api, mirror, ancestors: 'all' })` returns
-role-tagged tweets, source counts and warnings. Batch prefetching Mirror
+role-tagged tweets (with `links`: expanded URLs and link-card text), source
+counts and warnings; add `replies: { rule: 'digest', followed }` for Digest's
+reply rule (root-author and followed replies with their context). Batch prefetching Mirror
 ids seeds the shared cache. Reader and Snip retain their copies for now.
 
 Releasing: bump `version`, commit, and run `npm run deploy` (Coder runs it

package/dist/log-browser.js

@@ -88,7 +88,7 @@ export function createBrowserLogger(opts = {}) {
             // keeping are often the ones an offline stretch causes: while the
             // browser reports itself offline, failed attempts don't count, so a
             // batch waits out the outage (until the page dies or the buffer cap
-            // evicts it) instead of being dropped after ~60 s.
+            // evicts it) instead of being dropped after its 6 attempts (~5 min).
             isOffline: () => navigator.onLine === false,
             onGiveUp: (batch, error, reason) => console.warn(`log ship dropped a batch (${reason}): ${String(error)}`, {
                 batch_id: batch.id,

package/dist/log-core.d.ts

@@ -54,13 +54,16 @@ export interface ShipperOptions {
     token: string;
     batchSize?: number;
     intervalMs?: number;
-    /** Total delivery attempts per batch (default 4). */
+    /** Total delivery attempts per batch (default 6; see DEFAULT_RETRY_DELAYS_MS). */
     maxAttempts?: number;
-    /** Backoff before attempt 2, 3, … (default [5s, 15s, 40s]). On an idle
-     *  Cloud Run instance the CPU is throttled between requests, so these
+    /** Backoff before attempt 2, 3, … (default DEFAULT_RETRY_DELAYS_MS). On an
+     *  idle Cloud Run instance the CPU is throttled between requests, so these
      *  fire at the next CPU window (next request or shutdown), not on the
-     *  wall clock. */
+     *  wall clock — which only stretches the budget. */
     retryDelaysMs?: number[];
+    /** While isOffline() is true, how long a batch waits before its next
+     *  (uncounted) attempt (default 40 s). */
+    offlineRetryMs?: number;
     /** Per-attempt fetch timeout (default 10s; 3s aborted most of the fleet's
      *  attempts in the two weeks to 2026-08-28). */
     attemptTimeoutMs?: number;
@@ -74,8 +77,8 @@ export interface ShipperOptions {
     /** True while the platform knows it has no network (the browser passes
      *  `navigator.onLine === false`). A failed attempt made while offline says
      *  nothing about the batch or the ingest service, so it is not counted
-     *  toward maxAttempts and not reported: the batch waits and retries at the
-     *  longest backoff delay, or at once on resume(). The signal only decides
+     *  toward maxAttempts and not reported: the batch waits and retries after
+     *  offlineRetryMs, or at once on resume(). The signal only decides
      *  what a failure means — attempts still run while "offline". */
     isOffline?: () => boolean;
     /** A delivery attempt failed; the batch will be retried. */
@@ -86,6 +89,21 @@ export interface ShipperOptions {
      *  keep, listed in `failures`. Those events are lost; the rest landed. */
     onPartialFailure?: (batch: ShippedBatch, failed: number, failures: unknown[]) => void;
 }
+/** The default retry budget: 6 attempts with 5 + 15 + 40 + 90 + 150 s =
+ *  5 minutes of backoff between the first and the last.
+ *
+ *  Deliberate choice, 2026-10-15 (watchdog task dc78e9b2): the earlier
+ *  4 attempts over 5 + 15 + 40 s (~60 s) dropped a mirror batch on
+ *  2026-10-14, when Axiom ingest answered 503 on and off for ~150 s — 2.5×
+ *  the budget — the only ingest outage in the 4 weeks of markers on record.
+ *  5 minutes covers that episode twice over. Headroom: at the fleet's peak
+ *  five-minute volume (mirror, 1 538 events, ~0.3 MB, 2026-10-08..15) the
+ *  events held through a 5-minute outage fill ~31% of the 5 000-event /
+ *  5 MB buffer cap, so the cap, not the attempt count, bounds a longer
+ *  outage. Longer delays cost nothing on a throttled instance (they ride the
+ *  next CPU window either way) and only matter while ingest is failing. */
+export declare const DEFAULT_MAX_ATTEMPTS = 6;
+export declare const DEFAULT_RETRY_DELAYS_MS: readonly number[];
 export declare class LogShipper {
     private readonly opts;
     private queue;
@@ -102,6 +120,7 @@ export declare class LogShipper {
     private readonly intervalMs;
     private readonly maxAttempts;
     private readonly retryDelaysMs;
+    private readonly offlineRetryMs;
     private readonly attemptTimeoutMs;
     private readonly maxBufferedEvents;
     private readonly maxBufferedBytes;

package/dist/log-core.js

@@ -45,6 +45,21 @@ export function serializeError(err) {
 export function truncate(s, max) {
     return s.length <= max ? s : s.slice(0, max) + `…[+${s.length - max}]`;
 }
+/** The default retry budget: 6 attempts with 5 + 15 + 40 + 90 + 150 s =
+ *  5 minutes of backoff between the first and the last.
+ *
+ *  Deliberate choice, 2026-10-15 (watchdog task dc78e9b2): the earlier
+ *  4 attempts over 5 + 15 + 40 s (~60 s) dropped a mirror batch on
+ *  2026-10-14, when Axiom ingest answered 503 on and off for ~150 s — 2.5×
+ *  the budget — the only ingest outage in the 4 weeks of markers on record.
+ *  5 minutes covers that episode twice over. Headroom: at the fleet's peak
+ *  five-minute volume (mirror, 1 538 events, ~0.3 MB, 2026-10-08..15) the
+ *  events held through a 5-minute outage fill ~31% of the 5 000-event /
+ *  5 MB buffer cap, so the cap, not the attempt count, bounds a longer
+ *  outage. Longer delays cost nothing on a throttled instance (they ride the
+ *  next CPU window either way) and only matter while ingest is failing. */
+export const DEFAULT_MAX_ATTEMPTS = 6;
+export const DEFAULT_RETRY_DELAYS_MS = [5_000, 15_000, 40_000, 90_000, 150_000];
 const RESPONSE_BODY_MAX = 300;
 /** A 4xx is a verdict on the bytes, not the moment: retrying the identical
  *  body cannot succeed. 408 (timeout) and 429 (rate limit) are the exceptions. */
@@ -88,6 +103,7 @@ export class LogShipper {
     intervalMs;
     maxAttempts;
     retryDelaysMs;
+    offlineRetryMs;
     attemptTimeoutMs;
     maxBufferedEvents;
     maxBufferedBytes;
@@ -95,8 +111,9 @@ export class LogShipper {
         this.opts = opts;
         this.batchSize = opts.batchSize ?? 50;
         this.intervalMs = opts.intervalMs ?? 2000;
-        this.maxAttempts = opts.maxAttempts ?? 4;
-        this.retryDelaysMs = opts.retryDelaysMs ?? [5_000, 15_000, 40_000];
+        this.maxAttempts = opts.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
+        this.retryDelaysMs = opts.retryDelaysMs ?? DEFAULT_RETRY_DELAYS_MS;
+        this.offlineRetryMs = opts.offlineRetryMs ?? 40_000;
         this.attemptTimeoutMs = opts.attemptTimeoutMs ?? 10_000;
         this.maxBufferedEvents = opts.maxBufferedEvents ?? 5_000;
         this.maxBufferedBytes = opts.maxBufferedBytes ?? 5 * 1024 * 1024;
@@ -259,7 +276,7 @@ export class LogShipper {
             if (!outcome.ok && !outcome.terminal && this.opts.isOffline?.()) {
                 // Uncounted: the budget is for failures that could be the batch's
                 // or the service's, and an offline failure is neither.
-                batch.nextAttemptAt = Date.now() + this.retryDelaysMs[this.retryDelaysMs.length - 1];
+                batch.nextAttemptAt = Date.now() + this.offlineRetryMs;
                 this.schedulePump();
                 return;
             }

package/dist/twitter-client.d.ts

@@ -4,11 +4,31 @@ export interface TweetAuthor {
     userName: string;
     name: string;
 }
+/** TwitterAPI.io sends `url` (t.co) + `expanded_url`; Mirror stores the
+ * expansion in `url` and may add link-card `title`/`description`. */
 export interface TweetEntityUrl {
     url: string;
     expanded_url?: string;
     display_url?: string;
+    title?: string;
+    description?: string;
 }
+/** A link card as TwitterAPI.io returns it: advanced_search gives the body
+ * directly, the by-id /tweets endpoint wraps it in `legacy` (probed
+ * 2026-10-15, test/fixtures/twitter-digest-live.json). String values such as
+ * `title`, `description` and `card_url` (the t.co token) are binding values. */
+export interface TwitterApiIoCardBody {
+    name?: string;
+    binding_values?: {
+        key: string;
+        value?: {
+            string_value?: string;
+        };
+    }[];
+}
+export type TwitterApiIoCard = TwitterApiIoCardBody & {
+    legacy?: TwitterApiIoCardBody;
+};
 export interface TwitterApiIoTweet {
     id: string;
     text: string;
@@ -20,6 +40,7 @@ export interface TwitterApiIoTweet {
     entities?: {
         urls?: TweetEntityUrl[];
     };
+    card?: TwitterApiIoCard | null;
     article?: unknown;
     articleBody?: {
         title: string;

package/dist/twitter-mirror.d.ts

@@ -17,9 +17,11 @@ export interface MirrorTweetRecord {
     is_deleted?: boolean | number;
     deleted_at?: string | null;
 }
+/** `complete` is false when paging stopped early; `warnings` says why. */
 export interface MirrorConversation {
     tweets: TwitterApiIoTweet[];
     complete: boolean;
+    warnings?: string[];
 }
 export interface MirrorTwitterClient {
     getTweetsByIds(ids: string[]): Promise<TwitterApiIoTweet[]>;
@@ -28,8 +30,13 @@ export interface MirrorTwitterClient {
 /** Mirror snapshots store URL expansions under `url`, without the original
  * t.co token. Keep both shapes; rendering handles the positional fallback. */
 export declare function tweetFromMirror(r: MirrorTweetRecord): TwitterApiIoTweet;
+/** Rows per Mirror page (Mirror's cap) and pages per conversation read. */
+export declare const MIRROR_PAGE_LIMIT = 1000;
+export declare const MIRROR_CONVERSATION_MAX_PAGES = 10;
 /** Shared per run: batch prefetch seeds the id cache, conversation requests
- * are coalesced, and failed requests are evicted so a later link can retry. */
+ * are coalesced, and failed requests are evicted so a later link can retry.
+ * Conversations follow Mirror's keyset cursor (record_timestamp DESC, id
+ * DESC) while `has_more`, bounded by MIRROR_CONVERSATION_MAX_PAGES. */
 export declare class MirrorTweetClient implements MirrorTwitterClient {
     private readonly mirrorUrl;
     private readonly fetchFn;
@@ -39,4 +46,5 @@ export declare class MirrorTweetClient implements MirrorTwitterClient {
     private query;
     getTweetsByIds(ids: string[]): Promise<TwitterApiIoTweet[]>;
     getConversation(id: string): Promise<MirrorConversation>;
+    private readConversation;
 }

package/dist/twitter-mirror.js

@@ -31,8 +31,13 @@ export function tweetFromMirror(r) {
         ...(r.article_content ? { articleBody: { title: r.article_title ?? '', text: r.article_content } } : {}),
     };
 }
+/** Rows per Mirror page (Mirror's cap) and pages per conversation read. */
+export const MIRROR_PAGE_LIMIT = 1000;
+export const MIRROR_CONVERSATION_MAX_PAGES = 10;
 /** Shared per run: batch prefetch seeds the id cache, conversation requests
- * are coalesced, and failed requests are evicted so a later link can retry. */
+ * are coalesced, and failed requests are evicted so a later link can retry.
+ * Conversations follow Mirror's keyset cursor (record_timestamp DESC, id
+ * DESC) while `has_more`, bounded by MIRROR_CONVERSATION_MAX_PAGES. */
 export class MirrorTweetClient {
     mirrorUrl;
     fetchFn;
@@ -42,11 +47,11 @@ export class MirrorTweetClient {
         this.mirrorUrl = mirrorUrl;
         this.fetchFn = fetchFn;
     }
-    async query(filter) {
+    async query(filter, cursor) {
         const response = await this.fetchFn(`${this.mirrorUrl.replace(/\/$/, '')}/api/records/tweets/query`, {
             audience: this.mirrorUrl.replace(/\/$/, ''), method: 'POST',
             headers: { 'content-type': 'application/json' },
-            body: JSON.stringify({ filter_sql: filter, columns: '*', limit: 1000 }),
+            body: JSON.stringify({ filter_sql: filter, columns: '*', limit: MIRROR_PAGE_LIMIT, ...(cursor ? { cursor } : {}) }),
             signal: AbortSignal.timeout(30_000),
         });
         if (!response.ok)
@@ -64,7 +69,9 @@ export class MirrorTweetClient {
         }
         for (const [id, tweet] of tweets)
             this.byId.set(id, tweet);
-        return { tweets: [...tweets.values()], complete: !data.has_more && data.records.length < 1000 };
+        // A full page without `has_more` is treated as more to come (Mirror always sends it today).
+        const hasMore = data.has_more ?? data.records.length >= MIRROR_PAGE_LIMIT;
+        return { tweets: [...tweets.values()], hasMore, cursor: data.cursor ?? null };
     }
     async getTweetsByIds(ids) {
         ids.forEach(assertTweetId);
@@ -83,7 +90,7 @@ export class MirrorTweetClient {
         assertTweetId(id);
         let pending = this.conversations.get(id);
         if (!pending) {
-            pending = this.query(`conversation_id = '${id}'`).catch((error) => {
+            pending = this.readConversation(id).catch((error) => {
                 this.conversations.delete(id);
                 throw error;
             });
@@ -91,4 +98,29 @@ export class MirrorTweetClient {
         }
         return pending;
     }
+    async readConversation(id) {
+        const tweets = new Map();
+        const cursors = new Set();
+        const warnings = [];
+        let cursor;
+        for (let page = 1;; page++) {
+            const result = await this.query(`conversation_id = '${id}'`, cursor);
+            for (const tweet of result.tweets)
+                if (!tweets.has(tweet.id))
+                    tweets.set(tweet.id, tweet);
+            if (!result.hasMore)
+                return { tweets: [...tweets.values()], complete: true, warnings };
+            if (!result.cursor || cursors.has(result.cursor)) {
+                warnings.push('Mirror conversation pagination cursor missing or repeated');
+                break;
+            }
+            if (page >= MIRROR_CONVERSATION_MAX_PAGES) {
+                warnings.push(`Mirror conversation reached the ${MIRROR_CONVERSATION_MAX_PAGES}-page limit`);
+                break;
+            }
+            cursors.add(result.cursor);
+            cursor = result.cursor;
+        }
+        return { tweets: [...tweets.values()], complete: false, warnings };
+    }
 }

package/dist/twitter.d.ts

@@ -4,7 +4,18 @@ export * from './twitter-client.js';
 export * from './twitter-mirror.js';
 export declare function parseTweetId(value: string): string | null;
 export declare function isTwitterUrl(value: string): boolean;
-export type TweetRole = 'ancestor' | 'linked' | 'self-reply' | 'quoted';
+/** `reply`: a tweet the digest rule includes that is neither the root, an
+ * ancestor of the linked tweet, nor one of the linked author's self-replies
+ * (a followed or root-author reply, or a stranger shown as its context). The
+ * root is always `ancestor` unless it is the linked tweet. */
+export type TweetRole = 'ancestor' | 'linked' | 'self-reply' | 'reply' | 'quoted';
+/** An expanded link with its link-card text, when Mirror or the API has it. */
+export interface TweetLink {
+    url: string;
+    displayUrl?: string;
+    title?: string;
+    description?: string;
+}
 export interface ThreadTweet extends TwitterApiIoTweet {
     url: string;
     role: TweetRole;
@@ -13,6 +24,7 @@ export interface ThreadTweet extends TwitterApiIoTweet {
         title: string;
         text: string;
     };
+    links: TweetLink[];
 }
 export interface TwitterThread {
     tweets: ThreadTweet[];
@@ -23,13 +35,38 @@ export interface TwitterThread {
     };
     warnings: string[];
 }
+/** Digest's display rule (digest/src/shared/thread-builder.ts `buildThread`):
+ * the root, every tweet by the root author or a followed handle, and each of
+ * those tweets' ancestor chains (strangers included as context). `followed`
+ * holds handles, compared case-insensitively; a leading `@` is ignored. */
+export interface DigestReplyRule {
+    rule: 'digest';
+    followed: Set<string>;
+}
 export interface AssembleThreadOptions {
     api: TwitterClient;
     mirror?: MirrorTwitterClient;
+    /** Default rule only: the digest rule always includes every ancestor. */
     ancestors?: 'same-author' | 'all';
+    /** Omitted: ancestors plus the linked author's contiguous self-replies. */
+    replies?: DigestReplyRule;
 }
+/** Bounds: ancestor chain length, API search pages, and for the digest rule
+ * the parent ids looked up and the tweets kept (each with a warning). */
+export declare const THREAD_CHAIN_LIMIT = 50;
+export declare const THREAD_SEARCH_PAGE_LIMIT = 5;
+export declare const DIGEST_PARENT_LOOKUP_LIMIT = 200;
+export declare const DIGEST_THREAD_TWEET_LIMIT = 300;
+/** Snowflake order: a longer id is newer; equal lengths compare as strings
+ * (ids exceed Number.MAX_SAFE_INTEGER). Same as Digest's compareTweetIds. */
+export declare function compareTweetIds(a: string, b: string): number;
 export declare function expandTweetUrls(tweet: TwitterApiIoTweet): string;
+/** A tweet's URL entities, expanded, with link-card title and description:
+ * from the entity itself (Mirror's stored shape) or else from the tweet's
+ * TwitterAPI.io card, matched to the entity by its t.co `card_url`. */
+export declare function tweetLinks(tweet: TwitterApiIoTweet): TweetLink[];
 /** Conversation search discovers candidates; only direct parent edges decide
- * membership. The linked author's replies to strangers can never become
- * descendants. Missing ancestors degrade visibly and preserve known text. */
+ * membership. By default the linked author's replies to strangers can never
+ * become descendants; `replies: { rule: 'digest' }` applies Digest's rule
+ * instead. Missing ancestors degrade visibly and preserve known text. */
 export declare function assembleThread(id: string, options: AssembleThreadOptions): Promise<TwitterThread>;

package/dist/twitter.js

@@ -13,6 +13,19 @@ export function parseTweetId(value) {
     }
 }
 export function isTwitterUrl(value) { return parseTweetId(value) !== null; }
+/** Bounds: ancestor chain length, API search pages, and for the digest rule
+ * the parent ids looked up and the tweets kept (each with a warning). */
+export const THREAD_CHAIN_LIMIT = 50;
+export const THREAD_SEARCH_PAGE_LIMIT = 5;
+export const DIGEST_PARENT_LOOKUP_LIMIT = 200;
+export const DIGEST_THREAD_TWEET_LIMIT = 300;
+/** Snowflake order: a longer id is newer; equal lengths compare as strings
+ * (ids exceed Number.MAX_SAFE_INTEGER). Same as Digest's compareTweetIds. */
+export function compareTweetIds(a, b) {
+    if (a.length !== b.length)
+        return a.length - b.length;
+    return a.localeCompare(b);
+}
 const sameAuthor = (a, b) => !!(a.author.id && b.author.id && a.author.id === b.author.id) ||
     !!(a.author.userName && b.author.userName && a.author.userName.toLowerCase() === b.author.userName.toLowerCase());
 export function expandTweetUrls(tweet) {
@@ -33,11 +46,38 @@ export function expandTweetUrls(tweet) {
         text += `\nLinks: ${unmatched.join(' ')}`;
     return text.replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>');
 }
+/** A tweet's URL entities, expanded, with link-card title and description:
+ * from the entity itself (Mirror's stored shape) or else from the tweet's
+ * TwitterAPI.io card, matched to the entity by its t.co `card_url`. */
+export function tweetLinks(tweet) {
+    const card = tweet.card?.legacy ?? tweet.card;
+    const values = new Map();
+    for (const binding of Array.isArray(card?.binding_values) ? card.binding_values : []) {
+        if (typeof binding?.key === 'string' && typeof binding.value?.string_value === 'string')
+            values.set(binding.key, binding.value.string_value);
+    }
+    const cardUrl = values.get('card_url');
+    const text = (value) => typeof value === 'string' && value.trim() ? value : undefined;
+    return (tweet.entities?.urls ?? []).filter((u) => typeof u?.url === 'string').map((u) => {
+        const fromCard = !!cardUrl && u.url === cardUrl;
+        const title = text(u.title) ?? (fromCard ? text(values.get('title')) : undefined);
+        const description = text(u.description) ?? (fromCard ? text(values.get('description')) : undefined);
+        return {
+            url: u.expanded_url || u.url,
+            ...(text(u.display_url) ? { displayUrl: u.display_url } : {}),
+            ...(title ? { title } : {}),
+            ...(description ? { description } : {}),
+        };
+    });
+}
 /** Conversation search discovers candidates; only direct parent edges decide
- * membership. The linked author's replies to strangers can never become
- * descendants. Missing ancestors degrade visibly and preserve known text. */
+ * membership. By default the linked author's replies to strangers can never
+ * become descendants; `replies: { rule: 'digest' }` applies Digest's rule
+ * instead. Missing ancestors degrade visibly and preserve known text. */
 export async function assembleThread(id, options) {
     assertTweetId(id);
+    if (options.replies && options.replies.rule !== 'digest')
+        throw new Error('Unknown thread reply rule');
     const { api, mirror } = options;
     const result = { tweets: [], stats: { mirror2: 0, twitterApi: 0, apiCalls: 0 }, warnings: [] };
     const records = new Map();
@@ -46,7 +86,11 @@ export async function assembleThread(id, options) {
             if (!records.has(tweet.id) || source === 'mirror2')
                 records.set(tweet.id, { tweet, source });
     };
-    const warning = (message, error) => result.warnings.push(`${message}${error instanceof Error ? `: ${error.message}` : ''}`);
+    const warning = (message, error) => {
+        const text = `${message}${error instanceof Error ? `: ${error.message}` : ''}`;
+        if (!result.warnings.includes(text))
+            result.warnings.push(text); // one warning per cause
+    };
     const callApi = async (run) => {
         try {
             const response = await run();
@@ -73,37 +117,25 @@ export async function assembleThread(id, options) {
         if (absent.length)
             add((await callApi(() => api.getTweetsByIds(absent))).tweets, 'twitterapi');
     };
-    await fetchIds([id]);
-    const linked = records.get(id)?.tweet;
-    if (!linked)
-        throw new TweetNotFoundError(id);
-    const cid = linked.conversationId ?? linked.id;
-    let archivedAuthor = false;
-    if (mirror)
-        try {
-            const conversation = await mirror.getConversation(cid);
-            add(conversation.tweets, 'mirror2');
-            archivedAuthor = conversation.complete && conversation.tweets.some((tweet) => sameAuthor(tweet, linked));
-            if (!conversation.complete)
-                warning('Mirror conversation reached its row limit; searching the author via API');
-        }
-        catch (error) {
-            warning('Mirror conversation failed', error);
+    /** `conversation_id:<cid> from:<author>` — the author's replies Mirror may lack. */
+    const searchAuthor = async (author, cid) => {
+        if (!/^\w+$/.test(author.author.userName)) {
+            warning('Twitter conversation author handle missing or invalid');
+            return;
         }
-    if (!archivedAuthor && /^\w+$/.test(linked.author.userName)) {
         let cursor;
         const cursors = new Set();
-        for (let page = 0; page < 5; page++) {
+        for (let page = 0; page < THREAD_SEARCH_PAGE_LIMIT; page++) {
             try {
-                const search = await callApi(() => api.advancedSearch(`conversation_id:${cid} from:${linked.author.userName}`, cursor));
-                add(search.tweets.filter((tweet) => sameAuthor(tweet, linked) && (!tweet.conversationId || tweet.conversationId === cid)), 'twitterapi');
+                const search = await callApi(() => api.advancedSearch(`conversation_id:${cid} from:${author.author.userName}`, cursor));
+                add(search.tweets.filter((tweet) => sameAuthor(tweet, author) && (!tweet.conversationId || tweet.conversationId === cid)), 'twitterapi');
                 if (!search.tweets.length || !search.has_next_page)
                     break;
                 if (!search.next_cursor || cursors.has(search.next_cursor)) {
                     warning('Twitter conversation pagination cursor missing or repeated');
                     break;
                 }
-                if (page === 4)
+                if (page === THREAD_SEARCH_PAGE_LIMIT - 1)
                     warning('Twitter conversation reached the five-page limit');
                 cursors.add(search.next_cursor);
                 cursor = search.next_cursor;
@@ -113,70 +145,256 @@ export async function assembleThread(id, options) {
                 break;
             }
         }
-    }
-    else if (!archivedAuthor)
-        warning('Twitter conversation author handle missing or invalid');
-    const seen = new Set([id]);
-    const ancestors = [];
-    let current = linked;
-    while (current.inReplyToId && ancestors.length < 50) {
-        if (seen.has(current.inReplyToId)) {
-            warning('Tweet ancestor cycle detected');
-            break;
+    };
+    await fetchIds([id]);
+    const linked = records.get(id)?.tweet;
+    if (!linked)
+        throw new TweetNotFoundError(id);
+    /** Digest rule: the linked tweet's ancestors are fetched first, one parent
+     * at a time and outside the shared lookup budget, so a busy thread cannot
+     * crowd them out. Returns the top of the chain; gaps go in `linkedGaps`. */
+    const linkedGaps = new Set();
+    const fillLinkedChain = async () => {
+        let current = linked;
+        const seen = new Set([linked.id]);
+        for (let depth = 0; current.inReplyToId && depth < THREAD_CHAIN_LIMIT; depth++) {
+            const parentId = current.inReplyToId;
+            if (seen.has(parentId) || linkedGaps.has(parentId))
+                break;
+            try {
+                await fetchIds([parentId]);
+            }
+            catch (error) {
+                warning(`Parent ${parentId} unavailable`, error);
+                linkedGaps.add(parentId);
+                break;
+            }
+            const parent = records.get(parentId)?.tweet;
+            if (!parent) {
+                warning(`Parent ${parentId} not found`);
+                linkedGaps.add(parentId);
+                break;
+            }
+            seen.add(parentId);
+            current = parent;
         }
+        return current;
+    };
+    let cid = linked.conversationId || linked.id;
+    if (options.replies && !linked.conversationId && linked.inReplyToId) {
+        // A reply without a conversation id: its root is the top of its chain.
+        const top = await fillLinkedChain();
+        cid = top.id;
+        if (top.inReplyToId)
+            warning(`Tweet ${id} has no conversation id and its root was not found; using ${cid}`);
+    }
+    let conversation;
+    if (mirror)
         try {
-            await fetchIds([current.inReplyToId]);
+            conversation = await mirror.getConversation(cid);
+            add(conversation.tweets, 'mirror2');
+            for (const message of conversation.warnings ?? [])
+                warning(message);
         }
         catch (error) {
-            warning(`Parent ${current.inReplyToId} unavailable`, error);
-            break;
-        }
-        const parent = records.get(current.inReplyToId)?.tweet;
-        if (!parent) {
-            warning(`Parent ${current.inReplyToId} not found`);
-            break;
-        }
-        if ((options.ancestors ?? 'same-author') === 'same-author' && !sameAuthor(parent, linked))
-            break;
-        seen.add(parent.id);
-        ancestors.unshift(parent);
-        current = parent;
-    }
-    if (ancestors.length === 50 && current.inReplyToId)
-        warning('Tweet ancestors reached the 50-tweet limit');
-    const descendants = [];
-    current = linked;
-    while (descendants.length < 50) {
-        const candidates = [...records.values()].map((r) => r.tweet)
-            .filter((tweet) => tweet.inReplyToId === current.id && sameAuthor(tweet, linked) && !seen.has(tweet.id))
-            .sort((a, b) => a.id.length - b.id.length || a.id.localeCompare(b.id));
-        if (!candidates.length)
-            break;
-        if (candidates.length > 1)
-            warning(`Multiple self-replies to ${current.id}; selected earliest branch`);
-        current = candidates[0];
-        seen.add(current.id);
-        descendants.push(current);
-    }
-    if (descendants.length === 50)
-        warning('Tweet self-replies reached the 50-tweet limit');
-    const members = [...ancestors, linked, ...descendants];
+            warning('Mirror conversation failed', error);
+        }
+    const incompleteWarning = (who) => {
+        if (conversation && !conversation.complete && !conversation.warnings?.length)
+            warning(`Mirror conversation incomplete; searching ${who} via API`);
+    };
+    /** The default rule: ancestors (same author unless `ancestors: 'all'`) and
+     * the linked author's earliest contiguous chain of direct self-replies. */
+    const authorThread = async () => {
+        const archivedAuthor = !!conversation?.complete && conversation.tweets.some((tweet) => sameAuthor(tweet, linked));
+        incompleteWarning('the author');
+        if (!archivedAuthor)
+            await searchAuthor(linked, cid);
+        const seen = new Set([id]);
+        const ancestors = [];
+        let current = linked;
+        while (current.inReplyToId && ancestors.length < THREAD_CHAIN_LIMIT) {
+            if (seen.has(current.inReplyToId)) {
+                warning('Tweet ancestor cycle detected');
+                break;
+            }
+            try {
+                await fetchIds([current.inReplyToId]);
+            }
+            catch (error) {
+                warning(`Parent ${current.inReplyToId} unavailable`, error);
+                break;
+            }
+            const parent = records.get(current.inReplyToId)?.tweet;
+            if (!parent) {
+                warning(`Parent ${current.inReplyToId} not found`);
+                break;
+            }
+            if ((options.ancestors ?? 'same-author') === 'same-author' && !sameAuthor(parent, linked))
+                break;
+            seen.add(parent.id);
+            ancestors.unshift(parent);
+            current = parent;
+        }
+        if (ancestors.length === THREAD_CHAIN_LIMIT && current.inReplyToId)
+            warning(`Tweet ancestors reached the ${THREAD_CHAIN_LIMIT}-tweet limit`);
+        const descendants = [];
+        current = linked;
+        while (descendants.length < THREAD_CHAIN_LIMIT) {
+            const candidates = [...records.values()].map((r) => r.tweet)
+                .filter((tweet) => tweet.inReplyToId === current.id && sameAuthor(tweet, linked) && !seen.has(tweet.id))
+                .sort((a, b) => compareTweetIds(a.id, b.id));
+            if (!candidates.length)
+                break;
+            if (candidates.length > 1)
+                warning(`Multiple self-replies to ${current.id}; selected earliest branch`);
+            current = candidates[0];
+            seen.add(current.id);
+            descendants.push(current);
+        }
+        if (descendants.length === THREAD_CHAIN_LIMIT)
+            warning(`Tweet self-replies reached the ${THREAD_CHAIN_LIMIT}-tweet limit`);
+        return {
+            members: [...ancestors, linked, ...descendants],
+            roleOf: (tweet) => tweet.id === id ? 'linked' : ancestors.includes(tweet) ? 'ancestor' : 'self-reply',
+        };
+    };
+    /** Digest's rule over Mirror's conversation plus the root author's replies
+     * from the API. The search is skipped only when Mirror's read is complete
+     * and the root author is followed (Mirror polls followed accounts; for
+     * others it holds only what a one-off conversation fetch found — the live
+     * probe's unfollowed root author had a reply Mirror lacked). */
+    const digestThread = async (rule) => {
+        const followed = new Set([...rule.followed].map((handle) => handle.replace(/^@/, '').toLowerCase()));
+        await fillLinkedChain();
+        let rootFailed = linkedGaps.has(cid);
+        if (!rootFailed)
+            try {
+                await fetchIds([cid]);
+            }
+            catch (error) {
+                rootFailed = true;
+                warning(`Root ${cid} unavailable`, error);
+            }
+        const root = records.get(cid)?.tweet;
+        // Parent lookups below skip ids in `unavailable`, so each gap is reported once.
+        const unavailable = new Set([...linkedGaps, ...(root ? [] : [cid])]);
+        if (!root && !rootFailed)
+            warning(`Root ${cid} not found`);
+        else if (root) {
+            const archived = !!conversation?.complete && followed.has(root.author.userName.toLowerCase());
+            incompleteWarning('the root author');
+            if (!archived)
+                await searchAuthor(root, cid);
+        }
+        const candidates = () => [linked, ...[...records.values()].map((r) => r.tweet)
+                .filter((tweet) => tweet.id !== linked.id && (tweet.conversationId || cid) === cid &&
+                (tweet.id === cid || (!!root && sameAuthor(tweet, root)) || followed.has(tweet.author.userName.toLowerCase())))
+                .sort((a, b) => compareTweetIds(a.id, b.id))];
+        // Fill parents missing from every candidate's chain, one batch per depth.
+        let lookups = 0;
+        for (let round = 0;; round++) {
+            const missing = new Set();
+            const visited = new Set();
+            for (const candidate of candidates()) {
+                let tweet = candidate;
+                while (tweet?.inReplyToId && !visited.has(tweet.id)) {
+                    visited.add(tweet.id);
+                    const parent = records.get(tweet.inReplyToId)?.tweet;
+                    if (!parent && !unavailable.has(tweet.inReplyToId))
+                        missing.add(tweet.inReplyToId);
+                    tweet = parent;
+                }
+            }
+            if (!missing.size)
+                break;
+            if (round === THREAD_CHAIN_LIMIT || lookups === DIGEST_PARENT_LOOKUP_LIMIT) {
+                warning(round === THREAD_CHAIN_LIMIT ? `Tweet ancestors reached the ${THREAD_CHAIN_LIMIT}-tweet limit` : `Thread parent lookups reached the ${DIGEST_PARENT_LOOKUP_LIMIT}-tweet limit`);
+                break;
+            }
+            const batch = [...missing].slice(0, DIGEST_PARENT_LOOKUP_LIMIT - lookups);
+            lookups += batch.length;
+            let failed = false;
+            try {
+                await fetchIds(batch);
+            }
+            catch (error) {
+                failed = true;
+                warning(`Parent lookup of ${batch.length} tweets failed`, error);
+            }
+            for (const parentId of batch) {
+                if (records.has(parentId))
+                    continue;
+                unavailable.add(parentId);
+                if (!failed)
+                    warning(`Parent ${parentId} not found`);
+            }
+        }
+        // Each candidate brings its chain; the linked chain always, others while they fit.
+        const include = new Map();
+        const linkedChain = new Set();
+        for (const candidate of candidates()) {
+            const chain = [];
+            const local = new Set();
+            let tweet = candidate;
+            while (tweet && !include.has(tweet.id)) {
+                if (local.has(tweet.id)) {
+                    warning('Tweet ancestor cycle detected');
+                    break;
+                }
+                if (chain.length > THREAD_CHAIN_LIMIT) {
+                    warning(`Tweet ancestors reached the ${THREAD_CHAIN_LIMIT}-tweet limit`);
+                    break;
+                }
+                local.add(tweet.id);
+                chain.push(tweet);
+                tweet = tweet.inReplyToId ? records.get(tweet.inReplyToId)?.tweet : undefined;
+            }
+            if (candidate === linked)
+                for (const member of chain)
+                    linkedChain.add(member.id);
+            else if (include.size + chain.length > DIGEST_THREAD_TWEET_LIMIT) {
+                warning(`Thread reached the ${DIGEST_THREAD_TWEET_LIMIT}-tweet limit; later replies omitted`);
+                break;
+            }
+            for (const member of chain)
+                include.set(member.id, member);
+        }
+        const members = [...include.values()]
+            .sort((a, b) => a.id === cid ? -1 : b.id === cid ? 1 : compareTweetIds(a.id, b.id));
+        // Replies follow their parents in id order, so one pass finds the
+        // linked author's contiguous self-replies.
+        const selfReplies = new Set([linked.id]);
+        for (const tweet of members) {
+            if (tweet.inReplyToId && selfReplies.has(tweet.inReplyToId) && sameAuthor(tweet, linked) && !linkedChain.has(tweet.id))
+                selfReplies.add(tweet.id);
+        }
+        return {
+            members,
+            roleOf: (tweet) => tweet.id === id ? 'linked'
+                : linkedChain.has(tweet.id) || tweet.id === cid ? 'ancestor'
+                    : selfReplies.has(tweet.id) ? 'self-reply' : 'reply',
+        };
+    };
+    const { members, roleOf } = options.replies ? await digestThread(options.replies) : await authorThread();
     const quoteSeen = new Set(members.map((t) => t.id));
     const articleIds = new Set();
+    const append = (item, role, source) => {
+        if (item.article)
+            articleIds.add(item.id);
+        const { article: _rawArticle, quoted_tweet: _nestedQuote, ...rest } = item;
+        result.tweets.push({
+            ...rest, text: expandTweetUrls(item), url: `https://x.com/${item.author.userName || 'i'}/status/${item.id}`,
+            role, source, links: tweetLinks(item), ...(item.articleBody ? { article: item.articleBody } : {}),
+        });
+        if (source === 'mirror2')
+            result.stats.mirror2++;
+        else
+            result.stats.twitterApi++;
+    };
     for (const tweet of members) {
         const source = records.get(tweet.id).source;
-        const role = tweet.id === id ? 'linked' : ancestors.includes(tweet) ? 'ancestor' : 'self-reply';
-        const append = (item, itemRole, itemSource) => {
-            if (item.article)
-                articleIds.add(item.id);
-            const { article: _rawArticle, quoted_tweet: _nestedQuote, ...rest } = item;
-            result.tweets.push({ ...rest, text: expandTweetUrls(item), url: `https://x.com/${item.author.userName || 'i'}/status/${item.id}`, role: itemRole, source: itemSource, ...(item.articleBody ? { article: item.articleBody } : {}) });
-            if (itemSource === 'mirror2')
-                result.stats.mirror2++;
-            else
-                result.stats.twitterApi++;
-        };
-        append(tweet, role, source);
+        append(tweet, roleOf(tweet), source);
         if (tweet.quoted_tweet && !quoteSeen.has(tweet.quoted_tweet.id)) {
             quoteSeen.add(tweet.quoted_tweet.id);
             const archivedQuote = records.get(tweet.quoted_tweet.id);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steve31415/baselib",
-  "version": "3.36.2",
+  "version": "3.38.0",
   "description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
   "type": "module",
   "license": "MIT",