@steve31415/baselib 3.7.0 → 3.8.0

package/README.md

@@ -6,7 +6,7 @@ What it provides and why: `docs/SPEC.md`. How it's put together:
 `~/migration/research/base-services-design.md` (step 5).
 
 Server subpath exports: `config`, `log`, `auth`, `s2s`, `db`, `http`, `sync`,
-`app-update`, `llm`, and `email`. Browser exports: `log-browser`, `rum`,
+`app-update`, `llm`, `email`, and `twitter`. Browser exports: `log-browser`, `rum`,
 `slow-fetch`, `sync-browser`, and `app-update-browser`.
 
 `sync` provides the Postgres event-log and server protocol primitives;
@@ -34,6 +34,14 @@ deploy (`~/plasticine-way/docs/OPERATIONS.md`, "Deployment safety").
 first step of any production investigation (same doc, "Investigating
 production state").
 
+`twitter` provides Mirror2-first tweet lookup, TwitterAPI.io batch/search/
+article calls, and thread assembly through direct reply edges. It includes
+one level of quotes, article bodies and URL expansions. Use one shared
+`MirrorTweetClient(mirrorUrl, serviceFetch)` and `TwitterApiIoClient(key)`
+per run; `assembleThread(id, { api, mirror, ancestors: 'all' })` returns
+role-tagged tweets, source counts and warnings. Batch prefetching Mirror
+ids seeds the shared cache. Reader2 and Snip retain their copies for now.
+
 Releasing: bump `version`, commit, and run `npm run deploy` (Coder runs it
 automatically when integrating a baselib branch) — it pushes the `v<version>`
 tag, whose workflow gates on verify and publishes via OIDC, and waits for the

package/dist/twitter-client.d.ts

@@ -0,0 +1,75 @@
+import type { Logger } from './log.js';
+export interface TweetAuthor {
+    id: string;
+    userName: string;
+    name: string;
+}
+export interface TweetEntityUrl {
+    url: string;
+    expanded_url?: string;
+    display_url?: string;
+}
+export interface TwitterApiIoTweet {
+    id: string;
+    text: string;
+    createdAt: string;
+    author: TweetAuthor;
+    conversationId?: string | null;
+    inReplyToId?: string | null;
+    quoted_tweet?: TwitterApiIoTweet | null;
+    entities?: {
+        urls?: TweetEntityUrl[];
+    };
+    article?: unknown;
+    articleBody?: {
+        title: string;
+        text: string;
+    };
+}
+export interface TwitterApiIoResponse {
+    tweets: TwitterApiIoTweet[];
+    apiCalls?: number;
+    has_next_page?: boolean;
+    next_cursor?: string | null;
+}
+export interface TwitterApiIoArticleResponse {
+    apiCalls?: number;
+    article: {
+        title: string;
+        contents?: {
+            text: string;
+        }[];
+    } | null;
+}
+export interface TwitterClient {
+    getTweetsByIds(ids: string[]): Promise<TwitterApiIoResponse>;
+    advancedSearch(query: string, cursor?: string): Promise<TwitterApiIoResponse>;
+    getArticle(tweetId: string): Promise<TwitterApiIoArticleResponse>;
+}
+export declare class TweetNotFoundError extends Error {
+    readonly tweetId: string;
+    constructor(tweetId: string);
+}
+export declare class TwitterApiError extends Error {
+    readonly apiCalls: number;
+    constructor(message: string, apiCalls: number);
+}
+export declare function assertTweetId(id: string): void;
+/** Harvested from Reader2/Mirror2: bounded retries and timeout, four active
+ * requests per shared client. Credentials and upstream bodies are never logged. */
+export declare class TwitterApiIoClient implements TwitterClient {
+    private readonly apiKey;
+    private readonly logger?;
+    private readonly fetchFn;
+    private active;
+    private readonly waiters;
+    constructor(apiKey: string, logger?: Pick<Logger, "info" | "warn"> | undefined, fetchFn?: typeof fetch);
+    private request;
+    private tweets;
+    getTweetsByIds(ids: string[]): Promise<TwitterApiIoResponse>;
+    getTweetById(id: string): Promise<{
+        tweet: TwitterApiIoTweet;
+    }>;
+    advancedSearch(query: string, cursor?: string): Promise<TwitterApiIoResponse>;
+    getArticle(id: string): Promise<TwitterApiIoArticleResponse>;
+}

package/dist/twitter-client.js

@@ -0,0 +1,117 @@
+export class TweetNotFoundError extends Error {
+    tweetId;
+    constructor(tweetId) {
+        super(`Tweet ${tweetId} not found`);
+        this.tweetId = tweetId;
+        this.name = 'TweetNotFoundError';
+    }
+}
+export class TwitterApiError extends Error {
+    apiCalls;
+    constructor(message, apiCalls) {
+        super(message);
+        this.apiCalls = apiCalls;
+    }
+}
+export function assertTweetId(id) {
+    if (!/^\d{1,25}$/.test(id))
+        throw new Error('Invalid tweet id');
+}
+/** Harvested from Reader2/Mirror2: bounded retries and timeout, four active
+ * requests per shared client. Credentials and upstream bodies are never logged. */
+export class TwitterApiIoClient {
+    apiKey;
+    logger;
+    fetchFn;
+    active = 0;
+    waiters = [];
+    constructor(apiKey, logger, fetchFn = globalThis.fetch) {
+        this.apiKey = apiKey;
+        this.logger = logger;
+        this.fetchFn = fetchFn;
+    }
+    async request(path, params) {
+        if (this.active >= 4)
+            await new Promise((resolve) => this.waiters.push(resolve));
+        else
+            this.active++;
+        try {
+            const url = new URL(`https://api.twitterapi.io/twitter${path}`);
+            for (const [key, value] of Object.entries(params))
+                url.searchParams.set(key, value);
+            for (let attempt = 0;; attempt++) {
+                let response;
+                try {
+                    response = await this.fetchFn(url.toString(), {
+                        headers: { 'X-API-Key': this.apiKey }, signal: AbortSignal.timeout(30_000),
+                    });
+                }
+                catch {
+                    if (attempt >= 3)
+                        throw new TwitterApiError('TwitterAPI.io network request failed', attempt + 1);
+                    this.logger?.warn('twitter_api_retry', { path, attempt: attempt + 1 });
+                    await new Promise((resolve) => setTimeout(resolve, 1000 * 2 ** attempt));
+                    continue;
+                }
+                this.logger?.info('twitter_api_response', { path, status: response.status, attempt });
+                if ((response.status === 429 || response.status >= 500) && attempt < 3) {
+                    const header = response.headers.get('retry-after');
+                    const seconds = header ? Number(header) : NaN;
+                    const parsed = header && Number.isNaN(seconds) ? Date.parse(header) - Date.now() : seconds * 1000;
+                    const delay = Number.isFinite(parsed) && parsed >= 0 ? Math.min(parsed, 30_000) : 1000 * 2 ** attempt;
+                    await response.body?.cancel();
+                    await new Promise((resolve) => setTimeout(resolve, delay));
+                    continue;
+                }
+                if (!response.ok) {
+                    await response.body?.cancel();
+                    throw new TwitterApiError(`TwitterAPI.io HTTP ${response.status}`, attempt + 1);
+                }
+                const data = await response.json();
+                if (data.status === 'error' || data.status === 'failed' || (data.code && data.code !== 200)) {
+                    throw new TwitterApiError('TwitterAPI.io returned an API error', attempt + 1);
+                }
+                return Object.assign(data, { apiCalls: attempt + 1 });
+            }
+        }
+        finally {
+            const next = this.waiters.shift();
+            if (next)
+                next();
+            else
+                this.active--;
+        }
+    }
+    async tweets(path, params) {
+        const raw = await this.request(path, params);
+        const tweets = raw.data?.tweets ?? raw.tweets;
+        if (!Array.isArray(tweets))
+            throw new Error('TwitterAPI.io response has no tweets array');
+        return { tweets, has_next_page: raw.has_next_page, next_cursor: raw.next_cursor, apiCalls: raw.apiCalls };
+    }
+    async getTweetsByIds(ids) {
+        ids.forEach(assertTweetId);
+        const tweets = [];
+        let apiCalls = 0;
+        const unique = [...new Set(ids)];
+        for (let i = 0; i < unique.length; i += 100) {
+            const result = await this.tweets('/tweets', { tweet_ids: unique.slice(i, i + 100).join(',') });
+            tweets.push(...result.tweets);
+            apiCalls += result.apiCalls ?? 1;
+        }
+        return { tweets, apiCalls };
+    }
+    async getTweetById(id) {
+        const tweet = (await this.getTweetsByIds([id])).tweets.find((t) => t.id === id);
+        if (!tweet)
+            throw new TweetNotFoundError(id);
+        return { tweet };
+    }
+    advancedSearch(query, cursor) {
+        return this.tweets('/tweet/advanced_search', { query, queryType: 'Latest', ...(cursor ? { cursor } : {}) });
+    }
+    getArticle(id) {
+        assertTweetId(id);
+        return this.request('/article', { tweet_id: id });
+    }
+}

package/dist/twitter-mirror.d.ts

@@ -0,0 +1,42 @@
+import type { serviceFetch } from './s2s.js';
+import { type TwitterApiIoTweet, type TweetEntityUrl } from './twitter-client.js';
+export interface MirrorTweetRecord {
+    tweet_id: string;
+    conversation_id?: string | null;
+    in_reply_to_tweet_id?: string | null;
+    author_id?: string | null;
+    author_handle?: string | null;
+    author_name?: string | null;
+    tweet_text?: string | null;
+    quoted_tweet_json?: string | null;
+    quoted_tweet_id?: string | null;
+    article_title?: string | null;
+    article_content?: string | null;
+    record_timestamp?: number | null;
+    urls?: string | TweetEntityUrl[] | null;
+    is_deleted?: boolean | number;
+    deleted_at?: string | null;
+}
+export interface MirrorConversation {
+    tweets: TwitterApiIoTweet[];
+    complete: boolean;
+}
+export interface MirrorTwitterClient {
+    getTweetsByIds(ids: string[]): Promise<TwitterApiIoTweet[]>;
+    getConversation(id: string): Promise<MirrorConversation>;
+}
+/** Mirror snapshots store URL expansions under `url`, without the original
+ * t.co token. Keep both shapes; rendering handles the positional fallback. */
+export declare function tweetFromMirror(r: MirrorTweetRecord): TwitterApiIoTweet;
+/** Shared per run: batch prefetch seeds the id cache, conversation requests
+ * are coalesced, and failed requests are evicted so a later link can retry. */
+export declare class MirrorTweetClient implements MirrorTwitterClient {
+    private readonly mirrorUrl;
+    private readonly fetchFn;
+    private readonly byId;
+    private readonly conversations;
+    constructor(mirrorUrl: string, fetchFn: typeof serviceFetch);
+    private query;
+    getTweetsByIds(ids: string[]): Promise<TwitterApiIoTweet[]>;
+    getConversation(id: string): Promise<MirrorConversation>;
+}

package/dist/twitter-mirror.js

@@ -0,0 +1,94 @@
+import { assertTweetId } from './twitter-client.js';
+/** Mirror snapshots store URL expansions under `url`, without the original
+ * t.co token. Keep both shapes; rendering handles the positional fallback. */
+export function tweetFromMirror(r) {
+    let urls = [];
+    if (r.urls) {
+        const parsed = typeof r.urls === 'string' ? JSON.parse(r.urls) : r.urls;
+        if (Array.isArray(parsed))
+            urls = parsed.filter((u) => typeof u?.url === 'string');
+    }
+    let quoted;
+    if (r.quoted_tweet_json) {
+        const q = JSON.parse(r.quoted_tweet_json);
+        if (typeof q.id === 'string')
+            quoted = {
+                id: q.id, text: typeof q.text === 'string' ? q.text : '',
+                createdAt: typeof q.created_at === 'string' ? q.created_at : '',
+                author: { id: '', userName: String(q.author_handle ?? ''), name: String(q.author_name ?? '') },
+                ...(typeof q.article_content === 'string' && q.article_content ? {
+                    articleBody: { title: String(q.article_title ?? ''), text: q.article_content },
+                } : {}),
+                ...(Array.isArray(q.urls) ? { entities: { urls: q.urls } } : {}),
+            };
+    }
+    return {
+        id: r.tweet_id, conversationId: r.conversation_id, inReplyToId: r.in_reply_to_tweet_id,
+        author: { id: r.author_id ?? '', userName: r.author_handle ?? '', name: r.author_name ?? '' },
+        text: r.tweet_text ?? '',
+        createdAt: r.record_timestamp ? new Date(r.record_timestamp * 1000).toISOString() : '',
+        entities: { urls }, quoted_tweet: quoted,
+        ...(r.article_content ? { articleBody: { title: r.article_title ?? '', text: r.article_content } } : {}),
+    };
+}
+/** Shared per run: batch prefetch seeds the id cache, conversation requests
+ * are coalesced, and failed requests are evicted so a later link can retry. */
+export class MirrorTweetClient {
+    mirrorUrl;
+    fetchFn;
+    byId = new Map();
+    conversations = new Map();
+    constructor(mirrorUrl, fetchFn) {
+        this.mirrorUrl = mirrorUrl;
+        this.fetchFn = fetchFn;
+    }
+    async query(filter) {
+        const response = await this.fetchFn(`${this.mirrorUrl.replace(/\/$/, '')}/api/records/tweets/query`, {
+            audience: this.mirrorUrl.replace(/\/$/, ''), method: 'POST',
+            headers: { 'content-type': 'application/json' },
+            body: JSON.stringify({ filter_sql: filter, columns: '*', limit: 1000 }),
+            signal: AbortSignal.timeout(30_000),
+        });
+        if (!response.ok)
+            throw new Error(`Mirror2 tweets query HTTP ${response.status}`);
+        const data = await response.json();
+        if (!Array.isArray(data.records))
+            throw new Error('Mirror2 response has no records array');
+        const tweets = new Map();
+        for (const r of data.records) {
+            if (r.is_deleted || r.deleted_at)
+                continue;
+            const tweet = tweetFromMirror(r);
+            if (!tweets.has(tweet.id))
+                tweets.set(tweet.id, tweet);
+        }
+        for (const [id, tweet] of tweets)
+            this.byId.set(id, tweet);
+        return { tweets: [...tweets.values()], complete: !data.has_more && data.records.length < 1000 };
+    }
+    async getTweetsByIds(ids) {
+        ids.forEach(assertTweetId);
+        const unique = [...new Set(ids)];
+        const missing = unique.filter((id) => !this.byId.has(id));
+        for (let i = 0; i < missing.length; i += 100) {
+            const batch = missing.slice(i, i + 100);
+            await this.query(`tweet_id IN (${batch.map((id) => `'${id}'`).join(',')})`);
+            for (const id of batch)
+                if (!this.byId.has(id))
+                    this.byId.set(id, null);
+        }
+        return unique.map((id) => this.byId.get(id)).filter((t) => !!t);
+    }
+    getConversation(id) {
+        assertTweetId(id);
+        let pending = this.conversations.get(id);
+        if (!pending) {
+            pending = this.query(`conversation_id = '${id}'`).catch((error) => {
+                this.conversations.delete(id);
+                throw error;
+            });
+            this.conversations.set(id, pending);
+        }
+        return pending;
+    }
+}

package/dist/twitter.d.ts

@@ -0,0 +1,35 @@
+import { type TwitterClient, type TwitterApiIoTweet } from './twitter-client.js';
+import type { MirrorTwitterClient } from './twitter-mirror.js';
+export * from './twitter-client.js';
+export * from './twitter-mirror.js';
+export declare function parseTweetId(value: string): string | null;
+export declare function isTwitterUrl(value: string): boolean;
+export type TweetRole = 'ancestor' | 'linked' | 'self-reply' | 'quoted';
+export interface ThreadTweet extends TwitterApiIoTweet {
+    url: string;
+    role: TweetRole;
+    source: 'mirror2' | 'twitterapi';
+    article?: {
+        title: string;
+        text: string;
+    };
+}
+export interface TwitterThread {
+    tweets: ThreadTweet[];
+    stats: {
+        mirror2: number;
+        twitterApi: number;
+        apiCalls: number;
+    };
+    warnings: string[];
+}
+export interface AssembleThreadOptions {
+    api: TwitterClient;
+    mirror?: MirrorTwitterClient;
+    ancestors?: 'same-author' | 'all';
+}
+export declare function expandTweetUrls(tweet: TwitterApiIoTweet): string;
+/** Conversation search discovers candidates; only direct parent edges decide
+ * membership. The linked author's replies to strangers can never become
+ * descendants. Missing ancestors degrade visibly and preserve known text. */
+export declare function assembleThread(id: string, options: AssembleThreadOptions): Promise<TwitterThread>;

package/dist/twitter.js

@@ -0,0 +1,203 @@
+import { assertTweetId, TweetNotFoundError, TwitterApiError } from './twitter-client.js';
+export * from './twitter-client.js';
+export * from './twitter-mirror.js';
+export function parseTweetId(value) {
+    try {
+        const u = new URL(value);
+        if (!['http:', 'https:'].includes(u.protocol) || !/^(?:(?:www|mobile)\.)?(?:x\.com|twitter\.com)$/.test(u.hostname))
+            return null;
+        return u.pathname.match(/^\/(?:\w+|i\/web)\/status\/(\d{1,25})(?:\/|$)/)?.[1] ?? null;
+    }
+    catch {
+        return null;
+    }
+}
+export function isTwitterUrl(value) { return parseTweetId(value) !== null; }
+const sameAuthor = (a, b) => !!(a.author.id && b.author.id && a.author.id === b.author.id) ||
+    !!(a.author.userName && b.author.userName && a.author.userName.toLowerCase() === b.author.userName.toLowerCase());
+export function expandTweetUrls(tweet) {
+    let text = tweet.text;
+    const unmatched = [];
+    for (const u of tweet.entities?.urls ?? []) {
+        if (u.expanded_url && u.url && text.includes(u.url))
+            text = text.split(u.url).join(u.expanded_url);
+        else if (u.url && !/^https?:\/\/t\.co\//.test(u.url) && !text.includes(u.url))
+            unmatched.push(u.expanded_url ?? u.url);
+    }
+    // Mirror does not retain t.co keys. Only substitute when unambiguous;
+    // otherwise append expansions without assigning the wrong target.
+    const tokens = [...new Set(text.match(/https?:\/\/t\.co\/[\w]+/g) ?? [])];
+    if (tokens.length === 1 && unmatched.length === 1)
+        text = text.split(tokens[0]).join(unmatched[0]);
+    else if (unmatched.length)
+        text += `\nLinks: ${unmatched.join(' ')}`;
+    return text.replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>');
+}
+/** Conversation search discovers candidates; only direct parent edges decide
+ * membership. The linked author's replies to strangers can never become
+ * descendants. Missing ancestors degrade visibly and preserve known text. */
+export async function assembleThread(id, options) {
+    assertTweetId(id);
+    const { api, mirror } = options;
+    const result = { tweets: [], stats: { mirror2: 0, twitterApi: 0, apiCalls: 0 }, warnings: [] };
+    const records = new Map();
+    const add = (tweets, source) => {
+        for (const tweet of tweets)
+            if (!records.has(tweet.id) || source === 'mirror2')
+                records.set(tweet.id, { tweet, source });
+    };
+    const warning = (message, error) => result.warnings.push(`${message}${error instanceof Error ? `: ${error.message}` : ''}`);
+    const callApi = async (run) => {
+        try {
+            const response = await run();
+            result.stats.apiCalls += response.apiCalls ?? 1;
+            return response;
+        }
+        catch (error) {
+            result.stats.apiCalls += error instanceof TwitterApiError ? error.apiCalls : 1;
+            throw error;
+        }
+    };
+    const fetchIds = async (ids) => {
+        const missing = [...new Set(ids)].filter((key) => !records.has(key));
+        if (!missing.length)
+            return;
+        if (mirror)
+            try {
+                add(await mirror.getTweetsByIds(missing), 'mirror2');
+            }
+            catch (error) {
+                warning('Mirror2 lookup failed', error);
+            }
+        const absent = missing.filter((key) => !records.has(key));
+        if (absent.length)
+            add((await callApi(() => api.getTweetsByIds(absent))).tweets, 'twitterapi');
+    };
+    await fetchIds([id]);
+    const linked = records.get(id)?.tweet;
+    if (!linked)
+        throw new TweetNotFoundError(id);
+    const cid = linked.conversationId ?? linked.id;
+    let archivedAuthor = false;
+    if (mirror)
+        try {
+            const conversation = await mirror.getConversation(cid);
+            add(conversation.tweets, 'mirror2');
+            archivedAuthor = conversation.complete && conversation.tweets.some((tweet) => sameAuthor(tweet, linked));
+            if (!conversation.complete)
+                warning('Mirror2 conversation reached its row limit; searching the author via API');
+        }
+        catch (error) {
+            warning('Mirror2 conversation failed', error);
+        }
+    if (!archivedAuthor && /^\w+$/.test(linked.author.userName)) {
+        let cursor;
+        const cursors = new Set();
+        for (let page = 0; page < 5; page++) {
+            try {
+                const search = await callApi(() => api.advancedSearch(`conversation_id:${cid} from:${linked.author.userName}`, cursor));
+                add(search.tweets.filter((tweet) => sameAuthor(tweet, linked) && (!tweet.conversationId || tweet.conversationId === cid)), 'twitterapi');
+                if (!search.tweets.length || !search.has_next_page)
+                    break;
+                if (!search.next_cursor || cursors.has(search.next_cursor)) {
+                    warning('Twitter conversation pagination cursor missing or repeated');
+                    break;
+                }
+                if (page === 4)
+                    warning('Twitter conversation reached the five-page limit');
+                cursors.add(search.next_cursor);
+                cursor = search.next_cursor;
+            }
+            catch (error) {
+                warning('Twitter conversation search failed', error);
+                break;
+            }
+        }
+    }
+    else if (!archivedAuthor)
+        warning('Twitter conversation author handle missing or invalid');
+    const seen = new Set([id]);
+    const ancestors = [];
+    let current = linked;
+    while (current.inReplyToId && ancestors.length < 50) {
+        if (seen.has(current.inReplyToId)) {
+            warning('Tweet ancestor cycle detected');
+            break;
+        }
+        try {
+            await fetchIds([current.inReplyToId]);
+        }
+        catch (error) {
+            warning(`Parent ${current.inReplyToId} unavailable`, error);
+            break;
+        }
+        const parent = records.get(current.inReplyToId)?.tweet;
+        if (!parent) {
+            warning(`Parent ${current.inReplyToId} not found`);
+            break;
+        }
+        if ((options.ancestors ?? 'same-author') === 'same-author' && !sameAuthor(parent, linked))
+            break;
+        seen.add(parent.id);
+        ancestors.unshift(parent);
+        current = parent;
+    }
+    if (ancestors.length === 50 && current.inReplyToId)
+        warning('Tweet ancestors reached the 50-tweet limit');
+    const descendants = [];
+    current = linked;
+    while (descendants.length < 50) {
+        const candidates = [...records.values()].map((r) => r.tweet)
+            .filter((tweet) => tweet.inReplyToId === current.id && sameAuthor(tweet, linked) && !seen.has(tweet.id))
+            .sort((a, b) => a.id.length - b.id.length || a.id.localeCompare(b.id));
+        if (!candidates.length)
+            break;
+        if (candidates.length > 1)
+            warning(`Multiple self-replies to ${current.id}; selected earliest branch`);
+        current = candidates[0];
+        seen.add(current.id);
+        descendants.push(current);
+    }
+    if (descendants.length === 50)
+        warning('Tweet self-replies reached the 50-tweet limit');
+    const members = [...ancestors, linked, ...descendants];
+    const quoteSeen = new Set(members.map((t) => t.id));
+    const articleIds = new Set();
+    for (const tweet of members) {
+        const source = records.get(tweet.id).source;
+        const role = tweet.id === id ? 'linked' : ancestors.includes(tweet) ? 'ancestor' : 'self-reply';
+        const append = (item, itemRole, itemSource) => {
+            if (item.article)
+                articleIds.add(item.id);
+            const { article: _rawArticle, quoted_tweet: _nestedQuote, ...rest } = item;
+            result.tweets.push({ ...rest, text: expandTweetUrls(item), url: `https://x.com/${item.author.userName || 'i'}/status/${item.id}`, role: itemRole, source: itemSource, ...(item.articleBody ? { article: item.articleBody } : {}) });
+            if (itemSource === 'mirror2')
+                result.stats.mirror2++;
+            else
+                result.stats.twitterApi++;
+        };
+        append(tweet, role, source);
+        if (tweet.quoted_tweet && !quoteSeen.has(tweet.quoted_tweet.id)) {
+            quoteSeen.add(tweet.quoted_tweet.id);
+            const archivedQuote = records.get(tweet.quoted_tweet.id);
+            append(archivedQuote?.tweet ?? tweet.quoted_tweet, 'quoted', archivedQuote?.source ?? source);
+        }
+    }
+    for (const tweet of result.tweets) {
+        const isArticle = articleIds.has(tweet.id) || /https?:\/\/(?:x\.com|twitter\.com)\/i\/article\/\d+/.test(tweet.text);
+        if (tweet.article || !isArticle)
+            continue;
+        try {
+            const { article } = await callApi(() => api.getArticle(tweet.id));
+            const text = article?.contents?.map((part) => part.text).join('\n\n') ?? '';
+            if (text.trim())
+                tweet.article = { title: article?.title ?? '', text };
+            else
+                warning(`X Article ${tweet.id} returned no body`);
+        }
+        catch (error) {
+            warning(`X Article ${tweet.id} unavailable`, error);
+        }
+    }
+    return result;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@steve31415/baselib",
-  "version": "3.7.0",
+  "version": "3.8.0",
   "description": "Plasticine new-world shared platform library: logging, auth, service-to-service auth, db, HTTP, sync, app updates",
   "type": "module",
   "license": "MIT",
@@ -9,6 +9,10 @@
     "dist"
   ],
   "exports": {
+    "./twitter": {
+      "types": "./dist/twitter.d.ts",
+      "default": "./dist/twitter.js"
+    },
     ".": {
       "types": "./dist/index.d.ts",
       "default": "./dist/index.js"