gloomberb 0.8.2 → 0.9.0

package/src/plugins/builtin/substack/api.test.ts

@@ -0,0 +1,298 @@
+import { afterEach, describe, expect, test } from "bun:test";
+import { MemoryPluginPersistence } from "../../../test-support/plugin-persistence";
+import {
+  attachSubstackPersistence,
+  resetSubstackPersistence,
+  setSubstackFetchTransportForTests,
+} from "./api/store";
+import {
+  completeSubstackMagicLink,
+  completeSubstackOtpLogin,
+  parseCookiesFromSetCookie,
+  splitSetCookieHeader,
+} from "./api/auth";
+import {
+  getCachedSubstackArticleDetail,
+  getCachedSubstackHome,
+  getCachedSubstackPublicationFeed,
+} from "./api/cache";
+import {
+  loadSubstackArticleDetail,
+  loadSubstackHome,
+  loadSubstackPublicationFeed,
+} from "./api/loaders";
+import type { SubstackArticleSummary, SubstackPublication } from "./types";
+
+const EMAIL = "reader@example.com";
+const LOGIN_URL = "https://substack.com/login-token";
+
+afterEach(() => {
+  setSubstackFetchTransportForTests(null);
+  resetSubstackPersistence();
+});
+
+function publication(overrides: Partial<SubstackPublication> = {}): SubstackPublication {
+  return {
+    id: "alpha",
+    name: "Alpha Research",
+    subdomain: null,
+    baseUrl: "https://alpha.example",
+    description: null,
+    logoUrl: null,
+    latestPublishedAt: null,
+    ...overrides,
+  };
+}
+
+function article(overrides: Partial<SubstackArticleSummary> = {}): SubstackArticleSummary {
+  const pub = publication();
+  return {
+    id: "1",
+    title: "Markets and $SPY",
+    publicationId: pub.id,
+    publicationName: pub.name,
+    publicationSubdomain: pub.subdomain,
+    publicationBaseUrl: pub.baseUrl,
+    url: `${pub.baseUrl}/p/markets`,
+    slug: "markets",
+    publishedAt: "2026-06-05T10:00:00Z",
+    subtitle: null,
+    previewText: null,
+    bodyHtml: null,
+    imageUrls: [],
+    wordCount: 220,
+    readMinutes: 1,
+    ...overrides,
+  };
+}
+
+function installAuthenticatedTransport(
+  handler: (url: string, init?: RequestInit) => Response | undefined | Promise<Response | undefined>,
+): string[] {
+  const requests: string[] = [];
+  setSubstackFetchTransportForTests(async (url, init) => {
+    requests.push(url);
+    if (url === LOGIN_URL) {
+      return new Response(null, {
+        status: 302,
+        headers: {
+          location: "/reader",
+          "set-cookie": "substack.sid=sid123; Path=/; HttpOnly",
+        },
+      });
+    }
+    if (url === "https://substack.com/reader") {
+      return new Response("ok", { status: 200 });
+    }
+    return await handler(url, init) ?? new Response(`unexpected ${url}`, { status: 500 });
+  });
+  return requests;
+}
+
+describe("Substack auth", () => {
+  test("parses combined cookie headers and follows OTP redirects for the session cookie", async () => {
+    const headers = splitSetCookieHeader(
+      "foo=bar; Expires=Wed, 21 Oct 2026 07:28:00 GMT; Path=/, substack.sid=sid123; Path=/; HttpOnly, substack.lli=1; Path=/",
+    );
+    expect(parseCookiesFromSetCookie(headers).get("substack.sid")).toBe("sid123");
+
+    attachSubstackPersistence(new MemoryPluginPersistence());
+    const requests: string[] = [];
+    setSubstackFetchTransportForTests(async (url, init) => {
+      requests.push(url);
+      if (url === "https://substack.com/api/v1/email-otp-login/complete") {
+        expect(init?.method).toBe("POST");
+        expect(JSON.parse(String(init?.body))).toMatchObject({ code: "123456", email: EMAIL });
+        return Response.json({ redirect: "/reader" });
+      }
+      if (url === "https://substack.com/reader") {
+        return new Response(null, {
+          status: 302,
+          headers: {
+            location: "/home",
+            "set-cookie": "substack.sid=sid789; Path=/; HttpOnly",
+          },
+        });
+      }
+      if (url === "https://substack.com/home") {
+        return new Response("ok", {
+          headers: { "set-cookie": "substack.lli=1; Path=/" },
+        });
+      }
+      return new Response(`unexpected ${url}`, { status: 500 });
+    });
+
+    const auth = await completeSubstackOtpLogin("123456", EMAIL);
+
+    expect(auth).toMatchObject({ sid: "sid789", lli: "1", email: EMAIL });
+    expect(requests).toEqual([
+      "https://substack.com/api/v1/email-otp-login/complete",
+      "https://substack.com/reader",
+      "https://substack.com/home",
+    ]);
+  });
+});
+
+describe("Substack cached data loading", () => {
+  test("loads subscriptions plus paginated feed and reuses the cached result", async () => {
+    attachSubstackPersistence(new MemoryPluginPersistence());
+    const feedUrls: string[] = [];
+    const requests = installAuthenticatedTransport((url) => {
+      if (url === "https://substack.com/api/v1/subscriptions/page_v2") {
+        return Response.json({
+          result: {
+            subscriptions: [{ id: "sub-alpha", publication_id: "alpha" }],
+            publicationMap: {
+              alpha: { id: "alpha", name: "Alpha Research", base_url: "https://alpha.example" },
+            },
+          },
+        });
+      }
+      if (url.startsWith("https://substack.com/api/v1/reader/feed?")) {
+        feedUrls.push(url);
+        const parsed = new URL(url);
+        return Response.json(parsed.searchParams.has("cursor")
+          ? { items: [{ post: { id: "second", title: "Second", post_date: "2026-06-05T11:00:00Z", publication: { id: "alpha", name: "Alpha Research", base_url: "https://alpha.example" } } }] }
+          : {
+            items: [
+              { type: "comment", comment: { id: 1, body: "not a post" } },
+              { post: { id: "first", title: "First", post_date: "2026-06-05T10:00:00Z", publication: { id: "alpha", name: "Alpha Research", base_url: "https://alpha.example" } } },
+            ],
+            nextCursor: "cursor-2",
+          });
+      }
+    });
+
+    await completeSubstackMagicLink(LOGIN_URL, EMAIL);
+    const first = await loadSubstackHome(false);
+    const requestCountAfterFirstLoad = requests.length;
+    const second = await loadSubstackHome(false);
+
+    expect(first.subscriptions.map((item) => item.name)).toEqual(["Alpha Research"]);
+    expect(first.feed.map((item) => item.id)).toEqual(["first", "second"]);
+    expect(second.feed.map((item) => item.id)).toEqual(["first", "second"]);
+    expect(feedUrls).toHaveLength(2);
+    expect(new URL(feedUrls[1]!).searchParams.get("cursor")).toBe("cursor-2");
+    expect(requests).toHaveLength(requestCountAfterFirstLoad);
+  });
+
+  test("hydrates cached home, merged publication pages, and article detail synchronously", () => {
+    const persistence = new MemoryPluginPersistence();
+    attachSubstackPersistence(persistence);
+    const pub = publication({ name: "Alpha" });
+    const firstArticle = article({ id: "1", title: "Archive one", publicationName: "Alpha" });
+    const secondArticle = article({
+      id: "2",
+      title: "Archive two",
+      publicationName: "Alpha",
+      url: "https://alpha.example/p/archive-two",
+      slug: "archive-two",
+      publishedAt: "2026-06-04T10:00:00Z",
+      readMinutes: 2,
+      wordCount: 440,
+    });
+    const cacheOptions = { sourceKey: "substack", schemaVersion: 3 };
+
+    persistence.seedResource("subscriptions", "me", [pub], cacheOptions);
+    persistence.seedResource("feed", "subscribed", [firstArticle], cacheOptions);
+    persistence.seedResource("publication", "https://alpha.example:offset:0", {
+      items: [firstArticle],
+      nextOffset: 12,
+      hasMore: true,
+    }, cacheOptions);
+    persistence.seedResource("publication", "https://alpha.example:offset:12", {
+      items: [secondArticle],
+      nextOffset: null,
+      hasMore: false,
+    }, cacheOptions);
+    persistence.seedResource("post", "1", {
+      ...firstArticle,
+      bodyHtml: "<p>Cached body</p>",
+      contentText: "Cached body",
+      contentBlocks: [{ type: "paragraph", text: "Cached body" }],
+      linkUrls: [],
+    }, cacheOptions);
+
+    expect(getCachedSubstackHome()?.feed.map((item) => item.title)).toEqual(["Archive one"]);
+    expect(getCachedSubstackPublicationFeed(pub)?.data.items.map((item) => item.title)).toEqual([
+      "Archive one",
+      "Archive two",
+    ]);
+    expect(getCachedSubstackArticleDetail({ id: "1" })?.data.contentText).toBe("Cached body");
+  });
+});
+
+describe("Substack publication archive loading", () => {
+  test("uses archive page limits and offset pagination", async () => {
+    attachSubstackPersistence(new MemoryPluginPersistence());
+    const requests = installAuthenticatedTransport((url) => {
+      if (url === "https://alpha.example/api/v1/archive?sort=new&limit=12") {
+        return Response.json(Array.from({ length: 12 }, (_, index) => ({
+          id: index + 1,
+          title: `Archive ${index + 1}`,
+          post_date: "2026-06-05T10:00:00Z",
+          slug: `archive-${index + 1}`,
+        })));
+      }
+      if (url === "https://alpha.example/api/v1/archive?sort=new&limit=12&offset=12") {
+        return Response.json(Array.from({ length: 3 }, (_, index) => ({
+          id: index + 13,
+          title: `Archive ${index + 13}`,
+          post_date: "2026-06-05T10:00:00Z",
+          slug: `archive-${index + 13}`,
+        })));
+      }
+    });
+
+    await completeSubstackMagicLink(LOGIN_URL, EMAIL);
+    const first = await loadSubstackPublicationFeed(publication(), false);
+    const second = await loadSubstackPublicationFeed(publication(), false, first.data.nextOffset ?? 0);
+
+    expect(first.data).toMatchObject({ hasMore: true, nextOffset: 12 });
+    expect(first.data.items).toHaveLength(12);
+    expect(second.data.items.map((item) => item.id)).toEqual(["13", "14", "15"]);
+    expect(second.data.hasMore).toBe(false);
+    expect(requests).toContain("https://alpha.example/api/v1/archive?sort=new&limit=12");
+    expect(requests).toContain("https://alpha.example/api/v1/archive?sort=new&limit=12&offset=12");
+  });
+});
+
+describe("Substack article detail loading", () => {
+  test("prefers the Substack origin post endpoint over custom-domain teaser content", async () => {
+    attachSubstackPersistence(new MemoryPluginPersistence());
+    const requests = installAuthenticatedTransport((url) => {
+      if (url === "https://substack.com/api/v1/posts/by-id/42") {
+        return Response.json({
+          id: 42,
+          title: "Full post",
+          body_html: "<p>Preview.</p><p>Paid section with $NVDA and more detail.</p>",
+          publication: { id: "alpha", name: "Alpha", base_url: "https://alpha.example" },
+        });
+      }
+      if (url === "https://alpha.example/api/v1/posts/by-id/42") {
+        return Response.json({
+          id: 42,
+          title: "Full post",
+          body_html: "<p>Preview.</p>",
+          publication: { id: "alpha", name: "Alpha", base_url: "https://alpha.example" },
+        });
+      }
+    });
+
+    await completeSubstackMagicLink(LOGIN_URL, EMAIL);
+    const detail = await loadSubstackArticleDetail(article({
+      id: "42",
+      title: "Full post",
+      url: "https://alpha.example/p/full-post",
+      slug: "full-post",
+      previewText: "Preview.",
+      bodyHtml: "<p>Preview.</p>",
+      wordCount: 1,
+    }));
+
+    expect(detail.data.contentText).toContain("Paid section");
+    expect(detail.data.wordCount).toBeGreaterThan(1);
+    expect(requests).toContain("https://substack.com/api/v1/posts/by-id/42");
+    expect(requests).not.toContain("https://alpha.example/api/v1/posts/by-id/42");
+  });
+});

package/src/plugins/builtin/substack/article-detail.tsx

@@ -0,0 +1,359 @@
+import { useCallback, type RefObject } from "react";
+import { Box, ScrollBox, Text, TextAttributes, useRendererHost, type ScrollBoxRenderable } from "../../../ui";
+import { Spinner } from "../../../components";
+import { RemoteImage } from "../../../components/ui";
+import { TickerBadgeText } from "../../../components/ticker/badge/text";
+import { useInlineTickers } from "../../../state/hooks/inline-tickers";
+import { colors } from "../../../theme/colors";
+import type {
+  SubstackArticleDetail,
+  SubstackArticleSummary,
+  SubstackContentBlock,
+} from "./types";
+
+function articleBlockText(block: SubstackContentBlock): string {
+  switch (block.type) {
+    case "heading":
+    case "paragraph":
+    case "quote":
+    case "listItem":
+      return block.text;
+    case "embed":
+      return [block.text, block.url].filter(Boolean).join("\n");
+    default:
+      return "";
+  }
+}
+
+function wrappedTextProps(width: number) {
+  return {
+    width,
+    wrapText: true,
+    wrapMode: "word",
+    style: {
+      minWidth: 0,
+      whiteSpace: "pre-wrap",
+      overflowWrap: "anywhere",
+    },
+  } as const;
+}
+
+function normalizedTwitterUsername(username: string | null | undefined): string | null {
+  const normalized = username?.trim().replace(/^@/, "") ?? "";
+  return /^[A-Za-z0-9_]{1,15}$/.test(normalized) ? normalized : null;
+}
+
+function TweetEmbedView({
+  block,
+  lineWidth,
+  imageWidth,
+  imageHeight,
+  catalog,
+  openTicker,
+  openLink,
+  openUsername,
+}: {
+  block: Extract<SubstackContentBlock, { type: "embed"; kind: "tweet" }>;
+  lineWidth: number;
+  imageWidth: number;
+  imageHeight: number;
+  catalog: ReturnType<typeof useInlineTickers>["catalog"];
+  openTicker: ReturnType<typeof useInlineTickers>["openTicker"];
+  openLink: (url: string) => void;
+  openUsername: (username: string) => void;
+}) {
+  const username = normalizedTwitterUsername(block.username);
+  const contentWidth = Math.max(1, lineWidth - 3);
+
+  return (
+    <Box flexDirection="column" width={lineWidth} paddingX={1}>
+      <Box flexDirection="row" width={Math.max(1, lineWidth - 2)}>
+        <Text fg={colors.textDim} attributes={TextAttributes.BOLD}>Tweet</Text>
+        {username ? (
+          <Text
+            fg={colors.borderFocused}
+            onMouseDown={() => openUsername(username)}
+          >
+            {` @${username}`}
+          </Text>
+        ) : null}
+        {block.dateLabel ? <Text fg={colors.textDim}>{` | ${block.dateLabel}`}</Text> : null}
+      </Box>
+      <Box flexDirection="row" width={Math.max(1, lineWidth - 2)}>
+        <Text fg={colors.borderFocused}>| </Text>
+        <Box width={contentWidth}>
+          <TickerBadgeText
+            text={block.text}
+            lineWidth={contentWidth}
+            catalog={catalog}
+            textColor={colors.text}
+            openTicker={openTicker}
+            openLink={openLink}
+            openUsername={openUsername}
+          />
+        </Box>
+      </Box>
+      {block.imageUrls.length > 0 ? (
+        <Box flexDirection="column" paddingLeft={2}>
+          {block.imageUrls.slice(0, 2).map((url, index) => (
+            <RemoteImage
+              key={url}
+              src={url}
+              alt={`Tweet image ${index + 1}`}
+              width={Math.max(1, imageWidth - 2)}
+              height={Math.max(4, Math.min(imageHeight, 10))}
+              label={block.imageUrls.length > 1 ? `tweet image ${index + 1}` : "tweet image"}
+            />
+          ))}
+        </Box>
+      ) : null}
+      {block.url ? (
+        <Text
+          fg={colors.borderFocused}
+          onMouseDown={() => openLink(block.url!)}
+        >
+          open tweet
+        </Text>
+      ) : null}
+    </Box>
+  );
+}
+
+function ArticleBlockView({
+  block,
+  lineWidth,
+  imageWidth,
+  imageHeight,
+  catalog,
+  openTicker,
+  openLink,
+  openUsername,
+}: {
+  block: SubstackContentBlock;
+  lineWidth: number;
+  imageWidth: number;
+  imageHeight: number;
+  catalog: ReturnType<typeof useInlineTickers>["catalog"];
+  openTicker: ReturnType<typeof useInlineTickers>["openTicker"];
+  openLink: (url: string) => void;
+  openUsername: (username: string) => void;
+}) {
+  switch (block.type) {
+    case "heading":
+      return (
+        <Text
+          fg={colors.textBright}
+          attributes={TextAttributes.BOLD}
+          {...wrappedTextProps(lineWidth)}
+        >
+          {block.text}
+        </Text>
+      );
+    case "quote":
+      return (
+        <Box flexDirection="row" width={lineWidth}>
+          <Text fg={colors.warning}>| </Text>
+          <Box width={Math.max(1, lineWidth - 2)}>
+            <TickerBadgeText
+              text={block.text}
+              lineWidth={Math.max(1, lineWidth - 2)}
+              catalog={catalog}
+              textColor={colors.textDim}
+              openTicker={openTicker}
+              openLink={openLink}
+              openUsername={openUsername}
+            />
+          </Box>
+        </Box>
+      );
+    case "listItem":
+      return (
+        <Box flexDirection="row" width={lineWidth}>
+          <Text fg={colors.textDim}>- </Text>
+          <Box width={Math.max(1, lineWidth - 2)}>
+            <TickerBadgeText
+              text={block.text}
+              lineWidth={Math.max(1, lineWidth - 2)}
+              catalog={catalog}
+              textColor={colors.text}
+              openTicker={openTicker}
+              openLink={openLink}
+              openUsername={openUsername}
+            />
+          </Box>
+        </Box>
+      );
+    case "image":
+      return (
+        <RemoteImage
+          src={block.url}
+          alt={block.alt ?? "Substack image"}
+          width={imageWidth}
+          height={imageHeight}
+          label={block.alt ?? "image"}
+        />
+      );
+    case "embed":
+      if (block.kind === "tweet") {
+        return (
+          <TweetEmbedView
+            block={block}
+            lineWidth={lineWidth}
+            imageWidth={imageWidth}
+            imageHeight={imageHeight}
+            catalog={catalog}
+            openTicker={openTicker}
+            openLink={openLink}
+            openUsername={openUsername}
+          />
+        );
+      }
+      return (
+        <Box flexDirection="column" width={lineWidth} paddingX={1}>
+          <Text fg={colors.textDim} attributes={TextAttributes.BOLD}>
+            {block.kind === "media" ? "Media" : "Link"}
+          </Text>
+          <TickerBadgeText
+            text={block.text}
+            lineWidth={Math.max(1, lineWidth - 2)}
+            catalog={catalog}
+            textColor={colors.text}
+            openTicker={openTicker}
+            openLink={openLink}
+            openUsername={openUsername}
+          />
+          {block.url ? (
+            <Text
+              fg={colors.borderFocused}
+              onMouseDown={() => openLink(block.url!)}
+              {...wrappedTextProps(Math.max(1, lineWidth - 2))}
+            >
+              {block.url}
+            </Text>
+          ) : null}
+        </Box>
+      );
+    case "divider":
+      return <Text fg={colors.border}>{"-".repeat(Math.max(1, Math.min(lineWidth, 96)))}</Text>;
+    case "paragraph":
+    default:
+      return (
+        <TickerBadgeText
+          text={block.text}
+          lineWidth={lineWidth}
+          catalog={catalog}
+          textColor={colors.text}
+          openTicker={openTicker}
+          openLink={openLink}
+          openUsername={openUsername}
+        />
+      );
+  }
+}
+
+function ArticleRichContent({
+  blocks,
+  fallbackText,
+  fallbackImageUrls,
+  articleTitle,
+  lineWidth,
+  imageWidth,
+  imageHeight,
+}: {
+  blocks: SubstackContentBlock[];
+  fallbackText: string;
+  fallbackImageUrls: string[];
+  articleTitle: string;
+  lineWidth: number;
+  imageWidth: number;
+  imageHeight: number;
+}) {
+  const rendererHost = useRendererHost();
+  const resolvedBlocks = blocks.length > 0
+    ? blocks
+    : [
+      ...(fallbackText ? [{ type: "paragraph" as const, text: fallbackText }] : []),
+      ...fallbackImageUrls.map((url): SubstackContentBlock => ({ type: "image", url, alt: articleTitle })),
+    ];
+  const tickerText = resolvedBlocks.map(articleBlockText).filter(Boolean).join("\n");
+  const { catalog, openTicker } = useInlineTickers([tickerText], { liveQuotes: false });
+  const openLink = useCallback((url: string) => {
+    void rendererHost.openExternal(url);
+  }, [rendererHost]);
+  const openUsername = useCallback((username: string) => {
+    const normalized = normalizedTwitterUsername(username);
+    if (!normalized) return;
+    void rendererHost.openExternal(`https://x.com/${normalized}`);
+  }, [rendererHost]);
+
+  if (resolvedBlocks.length === 0) {
+    return <Text fg={colors.textDim}>No article text returned.</Text>;
+  }
+
+  return (
+    <Box flexDirection="column" width={lineWidth} gap={1}>
+      {resolvedBlocks.map((block, index) => (
+        <ArticleBlockView
+          key={`${block.type}:${index}:${articleBlockText(block).slice(0, 32)}`}
+          block={block}
+          lineWidth={lineWidth}
+          imageWidth={imageWidth}
+          imageHeight={imageHeight}
+          catalog={catalog}
+          openTicker={openTicker}
+          openLink={openLink}
+          openUsername={openUsername}
+        />
+      ))}
+    </Box>
+  );
+}
+
+export function ArticleDetail({
+  article,
+  detail,
+  width,
+  loading,
+  error,
+  scrollRef,
+  onOpenArticle,
+}: {
+  article: SubstackArticleSummary;
+  detail: SubstackArticleDetail | null;
+  width: number;
+  loading: boolean;
+  error: string | null;
+  scrollRef: RefObject<ScrollBoxRenderable | null>;
+  onOpenArticle: () => void;
+}) {
+  const resolved = detail ?? null;
+  const text = resolved?.contentText || article.previewText || article.subtitle || "";
+  const blocks = resolved?.contentBlocks ?? [];
+  const fallbackImageUrls = resolved?.imageUrls.length ? resolved.imageUrls : article.imageUrls;
+  const lineWidth = Math.max(1, width - 2);
+  const imageWidth = Math.min(lineWidth, 86);
+  const imageHeight = Math.max(6, Math.min(14, Math.floor(imageWidth * 0.32)));
+
+  return (
+    <ScrollBox ref={scrollRef} scrollY focusable={false} flexGrow={1} paddingX={1}>
+      <Box flexDirection="column" width={lineWidth} gap={1}>
+        {loading && !resolved ? <Spinner label="Loading article..." /> : null}
+        {error ? <Text fg={colors.negative}>{error}</Text> : null}
+        <ArticleRichContent
+          blocks={blocks}
+          fallbackText={text}
+          fallbackImageUrls={fallbackImageUrls}
+          articleTitle={article.title}
+          lineWidth={lineWidth}
+          imageWidth={imageWidth}
+          imageHeight={imageHeight}
+        />
+        {article.url ? (
+          <Box height={1}>
+            <Text fg={colors.textDim} onMouseDown={onOpenArticle}>Open source: O</Text>
+          </Box>
+        ) : null}
+      </Box>
+    </ScrollBox>
+  );
+}