gloomberb 0.11.2 → 0.12.0

package/src/plugins/builtin/news/wire/article-search.ts

@@ -1,140 +0,0 @@
-import type { NewsArticle, NewsQuery } from "../../../../news/types";
-import { getSharedNewsService } from "../../../../news/hooks";
-
-const ARTICLE_SEARCH_LIMIT = 8;
-
-const STOPWORDS = new Set([
-  "a",
-  "an",
-  "the",
-  "on",
-  "of",
-  "in",
-  "to",
-  "for",
-  "and",
-  "or",
-  "from",
-  "with",
-  "about",
-  "by",
-  "at",
-  "as",
-  "is",
-  "are",
-  "was",
-  "be",
-]);
-
-const INTENT_WORDS = new Set([
-  "article",
-  "articles",
-  "news",
-  "headline",
-  "headlines",
-  "story",
-  "stories",
-  "rss",
-  "feed",
-  "feeds",
-  "press",
-]);
-
-const COMMAND_WORDS = new Set([
-  "open",
-  "read",
-  "show",
-  "find",
-  "search",
-  "please",
-  "me",
-  "latest",
-  "recent",
-  "lookup",
-  "look",
-  "up",
-]);
-
-/** Pulls the freshest slice of the global feed to search over. */
-export const ARTICLE_SEARCH_QUERY: NewsQuery = { feed: "latest", limit: 200 };
-
-export function tokenizeArticleQuery(query: string): string[] {
-  // `ART trump` is the command prefix plus a topic — "art" is not a search term.
-  const withoutArtPrefix = query.replace(/^\s*art\b/i, " ");
-  return withoutArtPrefix
-    .toLowerCase()
-    .split(/[^a-z0-9]+/)
-    .filter((token) => (
-      token.length >= 2
-      && !STOPWORDS.has(token)
-      && !INTENT_WORDS.has(token)
-      && !COMMAND_WORDS.has(token)
-    ));
-}
-
-export function looksLikeArticleQuery(query: string): boolean {
-  const tokens = query.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
-  if (tokens.some((token) => INTENT_WORDS.has(token))) return true;
-  return /^\s*art\b/i.test(query);
-}
-
-export function scoreArticleMatch(article: NewsArticle, tokens: readonly string[]): number {
-  if (tokens.length === 0) return 0;
-  const title = article.title.toLowerCase();
-  const source = article.source.toLowerCase();
-  const haystack = [
-    title,
-    source,
-    article.summary ?? "",
-    ...article.topics,
-    ...article.categories,
-    ...article.tickers,
-  ].join(" ").toLowerCase();
-
-  let score = 0;
-  for (const token of tokens) {
-    if (title.includes(token)) score += 10;
-    else if (source.includes(token)) score += 6;
-    else if (haystack.includes(token)) score += 3;
-    else return 0;
-  }
-  return score;
-}
-
-export function searchNewsArticles(
-  articles: readonly NewsArticle[],
-  query: string,
-  limit = ARTICLE_SEARCH_LIMIT,
-): NewsArticle[] {
-  const tokens = tokenizeArticleQuery(query);
-  if (tokens.length === 0) {
-    return [...articles]
-      .sort((left, right) => right.publishedAt.getTime() - left.publishedAt.getTime())
-      .slice(0, limit);
-  }
-
-  return articles
-    .map((article) => ({ article, score: scoreArticleMatch(article, tokens) }))
-    .filter((entry) => entry.score > 0)
-    .sort((left, right) => {
-      if (right.score !== left.score) return right.score - left.score;
-      return right.article.publishedAt.getTime() - left.article.publishedAt.getTime();
-    })
-    .slice(0, limit)
-    .map((entry) => entry.article);
-}
-
-/** Headlines already sitting in the shared news pool (firehose/RSS/breaking). */
-export function cachedNewsArticles(): NewsArticle[] {
-  const service = getSharedNewsService();
-  if (!service) return [];
-  const pooled = service.getFirehose(undefined, 500);
-  if (pooled.length > 0) return pooled;
-  return service.getQueryState(ARTICLE_SEARCH_QUERY).articles;
-}
-
-export async function loadNewsArticles(): Promise<NewsArticle[]> {
-  const service = getSharedNewsService();
-  if (!service) return [];
-  return (await service.load(ARTICLE_SEARCH_QUERY)).articles;
-}

package/src/plugins/builtin/substack/api/auth.ts

@@ -1,185 +0,0 @@
-import type { SubstackAuthState } from "./types";
-import {
-  parseErrorText,
-  resolveSubstackUrl,
-  storeSubstackAuth,
-  substackFetch,
-  SUBSTACK_ORIGIN,
-} from "./store";
-
-export async function requestSubstackMagicLink(email: string): Promise<void> {
-  const normalizedEmail = email.trim();
-  if (!/^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(normalizedEmail)) {
-    throw new Error("Enter a valid email address");
-  }
-  const response = await substackFetch(`${SUBSTACK_ORIGIN}/api/v1/email-login`, {
-    method: "POST",
-    headers: {
-      "content-type": "application/json",
-    },
-    body: JSON.stringify({ email: normalizedEmail }),
-  });
-  if (!response.ok) {
-    const detail = await parseErrorText(response);
-    throw new Error(`Substack login link failed (${response.status})${detail ? `: ${detail}` : ""}`);
-  }
-}
-
-export function splitSetCookieHeader(header: string): string[] {
-  const parts: string[] = [];
-  let start = 0;
-  let inExpires = false;
-  for (let i = 0; i < header.length; i += 1) {
-    const char = header[i];
-    if (char === ",") {
-      const fragment = header.slice(start, i);
-      inExpires = /expires=/i.test(fragment) && !/;\s*(?:max-age|domain|path|secure|httponly|samesite)\b/i.test(fragment);
-      if (!inExpires) {
-        parts.push(fragment.trim());
-        start = i + 1;
-      }
-    } else if (char === ";") {
-      inExpires = false;
-    }
-  }
-  const tail = header.slice(start).trim();
-  if (tail) parts.push(tail);
-  return parts;
-}
-
-function getSetCookieHeaders(headers: Headers): string[] {
-  const withGetSetCookie = headers as Headers & { getSetCookie?: () => string[] };
-  const direct = withGetSetCookie.getSetCookie?.();
-  if (direct && direct.length > 0) return direct;
-  const header = headers.get("set-cookie");
-  return header ? splitSetCookieHeader(header) : [];
-}
-
-export function parseCookiesFromSetCookie(headers: string[]): Map<string, string> {
-  const cookies = new Map<string, string>();
-  for (const header of headers) {
-    const first = header.split(";")[0] ?? "";
-    const eq = first.indexOf("=");
-    if (eq <= 0) continue;
-    const name = first.slice(0, eq).trim();
-    const value = first.slice(eq + 1).trim();
-    if (!name) continue;
-    cookies.set(name, value);
-  }
-  return cookies;
-}
-
-function cookieHeaderFromJar(jar: Map<string, string>): string {
-  return [...jar.entries()].map(([name, value]) => `${name}=${value}`).join("; ");
-}
-
-function mergeSetCookies(cookieJar: Map<string, string>, response: Response): void {
-  for (const [name, value] of parseCookiesFromSetCookie(getSetCookieHeaders(response.headers))) {
-    cookieJar.set(name, value);
-  }
-}
-
-async function followRedirectsForCookies(startUrl: string, cookieJar: Map<string, string>): Promise<void> {
-  let currentUrl = startUrl;
-  for (let redirectCount = 0; redirectCount < 8; redirectCount += 1) {
-    const response = await substackFetch(currentUrl, {
-      method: "GET",
-      redirect: "manual",
-      headers: cookieJar.size > 0 ? { cookie: cookieHeaderFromJar(cookieJar) } : undefined,
-    });
-    mergeSetCookies(cookieJar, response);
-
-    if (response.status < 300 || response.status >= 400) break;
-    const location = response.headers.get("location");
-    if (!location) break;
-    const nextUrl = resolveSubstackUrl(location, currentUrl);
-    if (!nextUrl) break;
-    currentUrl = nextUrl;
-  }
-}
-
-export async function completeSubstackMagicLink(link: string, email: string): Promise<SubstackAuthState> {
-  const currentUrl = resolveSubstackUrl(link);
-  if (!currentUrl) throw new Error("Paste the full Substack magic-link URL");
-
-  const cookieJar = new Map<string, string>();
-  await followRedirectsForCookies(currentUrl, cookieJar);
-
-  const sid = cookieJar.get("substack.sid");
-  if (!sid) {
-    throw new Error("Substack did not return a session cookie. Request a new login link and paste the full URL.");
-  }
-  const auth: SubstackAuthState = {
-    email: email.trim(),
-    sid,
-    lli: cookieJar.get("substack.lli") ?? "1",
-    loggedInAt: Date.now(),
-  };
-  storeSubstackAuth(auth);
-  return auth;
-}
-
-export async function completeSubstackOtpLogin(code: string, email: string): Promise<SubstackAuthState> {
-  const normalizedCode = code.trim();
-  const normalizedEmail = email.trim();
-  if (!/^\d{6}$/.test(normalizedCode)) {
-    throw new Error("Enter the 6-digit Substack email code");
-  }
-  if (!/^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(normalizedEmail)) {
-    throw new Error("Enter a valid email address");
-  }
-
-  const cookieJar = new Map<string, string>();
-  const response = await substackFetch(`${SUBSTACK_ORIGIN}/api/v1/email-otp-login/complete`, {
-    method: "POST",
-    redirect: "manual",
-    credentials: "include",
-    headers: {
-      "content-type": "application/json",
-    },
-    body: JSON.stringify({
-      code: normalizedCode,
-      email: normalizedEmail,
-      redirect: "",
-      for_pub: null,
-      island_magic_signin: false,
-    }),
-  });
-  mergeSetCookies(cookieJar, response);
-
-  const isRedirect = response.status >= 300 && response.status < 400;
-  if (!response.ok && !isRedirect) {
-    const detail = await parseErrorText(response);
-    throw new Error(`Substack code login failed (${response.status})${detail ? `: ${detail}` : ""}`);
-  }
-
-  let redirectUrl = resolveSubstackUrl(response.headers.get("location"));
-  if (!redirectUrl && response.ok) {
-    try {
-      const payload = await response.clone().json() as { redirect?: unknown };
-      redirectUrl = resolveSubstackUrl(payload.redirect);
-    } catch {
-      redirectUrl = null;
-    }
-  }
-
-  if (!cookieJar.get("substack.sid") && redirectUrl) {
-    await followRedirectsForCookies(redirectUrl, cookieJar);
-  }
-
-  const sid = cookieJar.get("substack.sid");
-  if (!sid) {
-    const capturedCookieNames = [...cookieJar.keys()];
-    throw new Error(
-      `Substack accepted the code but did not return a session cookie. status=${response.status} redirect=${redirectUrl ? "yes" : "no"} cookies=${capturedCookieNames.length ? capturedCookieNames.join(",") : "none"}`,
-    );
-  }
-  const auth: SubstackAuthState = {
-    email: normalizedEmail,
-    sid,
-    lli: cookieJar.get("substack.lli") ?? "1",
-    loggedInAt: Date.now(),
-  };
-  storeSubstackAuth(auth);
-  return auth;
-}

package/src/plugins/builtin/substack/api/cache.ts

@@ -1,82 +0,0 @@
-import type { PersistedResourceValue } from "../../../../types/persistence";
-import {
-  sortSubscriptionsByLatest,
-} from "../normalize";
-import type {
-  SubstackArticleDetail,
-  SubstackArticleSummary,
-  SubstackPublication,
-} from "../types";
-import type { SubstackCachedData, SubstackHomeData, SubstackPublicationFeedPage } from "./types";
-import { readResource } from "./store";
-
-const PUBLICATION_ARCHIVE_MAX_CACHED_PAGES = 20;
-
-export function publicationCacheKey(publication: SubstackPublication, offset: number): string {
-  return `${publication.baseUrl ?? publication.subdomain ?? publication.id}:offset:${offset}`;
-}
-
-function cachedDataFromResource<T>(entry: PersistedResourceValue<T> | null): SubstackCachedData<T> | null {
-  if (!entry) return null;
-  return {
-    data: entry.value,
-    fetchedAt: entry.fetchedAt,
-    stale: Boolean(entry.stale),
-  };
-}
-
-export function getCachedSubstackHome(): SubstackHomeData | null {
-  const subscriptionsEntry = readResource<SubstackPublication[]>("subscriptions", "me", false);
-  const feedEntry = readResource<SubstackArticleSummary[] | { items: SubstackArticleSummary[]; nextCursor?: string | null }>("feed", "subscribed", false);
-  if (!subscriptionsEntry || !feedEntry) return null;
-  const feed = Array.isArray(feedEntry.value) ? feedEntry.value : feedEntry.value.items;
-  const nextCursor = Array.isArray(feedEntry.value) ? null : feedEntry.value.nextCursor ?? null;
-  return {
-    subscriptions: sortSubscriptionsByLatest(subscriptionsEntry.value, feed),
-    feed,
-    fetchedAt: Math.max(subscriptionsEntry.fetchedAt, feedEntry.fetchedAt),
-    stale: Boolean(subscriptionsEntry.stale) || Boolean(feedEntry.stale),
-    hasMore: !!nextCursor,
-    nextCursor,
-  };
-}
-
-export function getCachedSubstackPublicationFeed(
-  publication: SubstackPublication,
-): SubstackCachedData<SubstackPublicationFeedPage> | null {
-  let offset: number | null = 0;
-  let pageCount = 0;
-  let merged: SubstackPublicationFeedPage | null = null;
-  let fetchedAt = 0;
-  let stale = false;
-
-  while (offset != null && pageCount < PUBLICATION_ARCHIVE_MAX_CACHED_PAGES) {
-    const entry: PersistedResourceValue<SubstackPublicationFeedPage> | null = readResource<SubstackPublicationFeedPage>(
-      "publication",
-      publicationCacheKey(publication, offset),
-      false,
-    );
-    if (!entry) break;
-    const itemsById = new Map<string, SubstackArticleSummary>();
-    for (const article of merged?.items ?? []) itemsById.set(article.id, article);
-    for (const article of entry.value.items) itemsById.set(article.id, article);
-    merged = {
-      items: [...itemsById.values()],
-      nextOffset: entry.value.nextOffset,
-      hasMore: entry.value.hasMore,
-    };
-    fetchedAt = Math.max(fetchedAt, entry.fetchedAt);
-    stale = stale || Boolean(entry.stale);
-    offset = entry.value.nextOffset;
-    pageCount += 1;
-  }
-
-  if (!merged) return null;
-  return { data: merged, fetchedAt, stale };
-}
-
-export function getCachedSubstackArticleDetail(
-  article: Pick<SubstackArticleSummary, "id">,
-): SubstackCachedData<SubstackArticleDetail> | null {
-  return cachedDataFromResource(readResource<SubstackArticleDetail>("post", article.id, false));
-}

package/src/plugins/builtin/substack/api/loaders.ts

@@ -1,170 +0,0 @@
-import {
-  normalizeFeedItems,
-  normalizePostDetail,
-  normalizeSubscriptions,
-  sortSubscriptionsByLatest,
-} from "../normalize";
-import type {
-  SubstackArticleSummary,
-  SubstackPublication,
-} from "../types";
-import { SubstackAuthError, type SubstackCachedData, type SubstackHomeData, type SubstackPublicationFeedPage } from "./types";
-import { publicationCacheKey } from "./cache";
-import {
-  fetchJsonAuthenticated,
-  loadCachedResource,
-  requireAuth,
-  SUBSTACK_ORIGIN,
-} from "./store";
-
-const READER_FEED_TARGET_ITEMS = 50;
-const READER_FEED_MAX_PAGES = 12;
-const PUBLICATION_ARCHIVE_PAGE_LIMIT = 12;
-
-export async function loadSubstackHome(force = false): Promise<SubstackHomeData> {
-  const auth = requireAuth();
-  const [subscriptionsEntry, feedEntry] = await Promise.all([
-    loadCachedResource("subscriptions", "me", force, async () => {
-      const payload = await fetchJsonAuthenticated<unknown>(`${SUBSTACK_ORIGIN}/api/v1/subscriptions/page_v2`, auth);
-      return normalizeSubscriptions(payload);
-    }),
-    loadCachedResource("feed", "subscribed", force, async () => {
-      return fetchReaderFeedItems(auth, "subscribed");
-    }),
-  ]);
-  return {
-    subscriptions: sortSubscriptionsByLatest(subscriptionsEntry.data, feedEntry.data.items),
-    feed: feedEntry.data.items,
-    fetchedAt: Math.max(subscriptionsEntry.fetchedAt, feedEntry.fetchedAt),
-    stale: subscriptionsEntry.stale || feedEntry.stale,
-    hasMore: !!feedEntry.data.nextCursor,
-    nextCursor: feedEntry.data.nextCursor,
-  };
-}
-
-export async function loadSubstackHomeMore(cursor: string): Promise<{ items: SubstackArticleSummary[]; nextCursor: string | null }> {
-  return fetchReaderFeedPage(requireAuth(), "subscribed", cursor);
-}
-
-function payloadCursor(payload: unknown): string | null {
-  const record = payload && typeof payload === "object" && !Array.isArray(payload)
-    ? payload as Record<string, unknown>
-    : null;
-  const cursor = record?.nextCursor ?? record?.next_cursor;
-  return typeof cursor === "string" && cursor.trim() ? cursor : null;
-}
-
-async function fetchReaderFeedPage(
-  auth: ReturnType<typeof requireAuth>,
-  tab: string,
-  cursor: string | null = null,
-): Promise<{ items: SubstackArticleSummary[]; nextCursor: string | null }> {
-  const url = new URL(`${SUBSTACK_ORIGIN}/api/v1/reader/feed`);
-  url.searchParams.set("tab", tab);
-  if (cursor) url.searchParams.set("cursor", cursor);
-  const payload = await fetchJsonAuthenticated<unknown>(url.toString(), auth);
-  const nextCursor = payloadCursor(payload);
-  return {
-    items: normalizeFeedItems(payload),
-    nextCursor: nextCursor && nextCursor !== cursor ? nextCursor : null,
-  };
-}
-
-async function fetchReaderFeedItems(
-  auth: ReturnType<typeof requireAuth>,
-  tab: string,
-): Promise<{ items: SubstackArticleSummary[]; nextCursor: string | null }> {
-  const collected = new Map<string, SubstackArticleSummary>();
-  let cursor: string | null = null;
-  for (let page = 0; page < READER_FEED_MAX_PAGES && collected.size < READER_FEED_TARGET_ITEMS; page += 1) {
-    const result = await fetchReaderFeedPage(auth, tab, cursor);
-    for (const article of result.items) {
-      collected.set(article.id, article);
-      if (collected.size >= READER_FEED_TARGET_ITEMS) break;
-    }
-    if (!result.nextCursor) {
-      cursor = null;
-      break;
-    }
-    cursor = result.nextCursor;
-  }
-  return { items: [...collected.values()], nextCursor: cursor };
-}
-
-export async function loadSubstackPublicationFeed(
-  publication: SubstackPublication,
-  force = false,
-  offset = 0,
-): Promise<SubstackCachedData<SubstackPublicationFeedPage>> {
-  const auth = requireAuth();
-  const baseUrl = publication.baseUrl;
-  if (!baseUrl) {
-    return {
-      data: { items: [], nextOffset: null, hasMore: false },
-      fetchedAt: Date.now(),
-      stale: false,
-    };
-  }
-  return loadCachedResource("publication", publicationCacheKey(publication, offset), force, async () => {
-    const url = new URL("/api/v1/archive", baseUrl);
-    url.searchParams.set("sort", "new");
-    url.searchParams.set("limit", String(PUBLICATION_ARCHIVE_PAGE_LIMIT));
-    if (offset > 0) url.searchParams.set("offset", String(offset));
-    const payload = await fetchJsonAuthenticated<unknown>(url.toString(), auth);
-    const items = normalizeFeedItems(payload, publication);
-    const hasMore = items.length >= PUBLICATION_ARCHIVE_PAGE_LIMIT;
-    return {
-      items,
-      nextOffset: hasMore ? offset + PUBLICATION_ARCHIVE_PAGE_LIMIT : null,
-      hasMore,
-    };
-  });
-}
-
-function detailCandidateUrls(article: SubstackArticleSummary): string[] {
-  const urls: string[] = [];
-  if (/^\d+$/.test(article.id)) {
-    urls.push(new URL(`/api/v1/posts/by-id/${article.id}`, SUBSTACK_ORIGIN).toString());
-  }
-  const baseUrl = article.publicationBaseUrl
-    ?? (article.url ? new URL(article.url).origin : null);
-  if (baseUrl) {
-    if (/^\d+$/.test(article.id)) {
-      urls.push(new URL(`/api/v1/posts/by-id/${article.id}`, baseUrl).toString());
-    }
-    if (article.slug) {
-      urls.push(new URL(`/api/v1/posts/${article.slug}`, baseUrl).toString());
-    }
-    if (article.url) {
-      try {
-        const pathSlug = new URL(article.url).pathname.match(/\/p\/([^/?#]+)/)?.[1];
-        if (pathSlug) urls.push(new URL(`/api/v1/posts/${decodeURIComponent(pathSlug)}`, baseUrl).toString());
-      } catch {
-        // Ignore malformed article URLs; the direct post id/slug paths above still apply.
-      }
-    }
-  }
-  return [...new Set(urls)];
-}
-
-export async function loadSubstackArticleDetail(
-  article: SubstackArticleSummary,
-  force = false,
-): Promise<SubstackCachedData<ReturnType<typeof normalizePostDetail>>> {
-  const auth = requireAuth();
-  return loadCachedResource("post", article.id, force, async () => {
-    const urls = detailCandidateUrls(article);
-    const errors: string[] = [];
-    for (const url of urls) {
-      try {
-        const payload = await fetchJsonAuthenticated<unknown>(url, auth);
-        return normalizePostDetail(payload, article);
-      } catch (error) {
-        if (error instanceof SubstackAuthError) throw error;
-        errors.push(error instanceof Error ? error.message : String(error));
-      }
-    }
-    if (article.bodyHtml || urls.length === 0) return normalizePostDetail(article, article);
-    throw new Error(errors[0] ?? "Substack post content unavailable");
-  });
-}