gloomberb 0.11.3 → 0.12.0

package/src/plugins/builtin/substack/api/store.ts

@@ -1,186 +0,0 @@
-import type { PluginPersistence } from "../../../../types/plugin";
-import type { CachePolicy, PersistedResourceValue } from "../../../../types/persistence";
-import { createThrottledFetch, type ThrottledFetchTransport } from "../../../../utils/throttled-fetch";
-import { normalizedHttpUrl } from "../../../../utils/url";
-import { SubstackAuthError, type SubstackAuthState, type SubstackCachedData } from "./types";
-
-export const SUBSTACK_ORIGIN = "https://substack.com";
-
-const AUTH_STATE_KEY = "auth";
-const AUTH_SCHEMA_VERSION = 1;
-export const CACHE_SOURCE = "substack";
-export const CACHE_SCHEMA_VERSION = 3;
-const DEFAULT_HEADERS = {
-  accept: "application/json,text/plain,*/*",
-  "accept-language": "en-US,en;q=0.9",
-  "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
-};
-const CACHE_POLICIES = {
-  subscriptions: { staleMs: 30 * 60 * 1000, expireMs: 14 * 24 * 60 * 60 * 1000 },
-  feed: { staleMs: 5 * 60 * 1000, expireMs: 3 * 24 * 60 * 60 * 1000 },
-  publication: { staleMs: 10 * 60 * 1000, expireMs: 7 * 24 * 60 * 60 * 1000 },
-  post: { staleMs: 30 * 24 * 60 * 60 * 1000, expireMs: 365 * 24 * 60 * 60 * 1000 },
-} satisfies Record<string, CachePolicy>;
-
-let substackPersistence: PluginPersistence | null = null;
-let substackClient = createThrottledFetch({
-  requestsPerMinute: 60,
-  maxRetries: 1,
-  timeoutMs: 15_000,
-  defaultHeaders: DEFAULT_HEADERS,
-  dedupeGetRequests: false,
-});
-const activeFetches = new Map<string, Promise<SubstackCachedData<unknown>>>();
-
-export function attachSubstackPersistence(persistence: PluginPersistence): void {
-  substackPersistence = persistence;
-}
-
-export function resetSubstackPersistence(): void {
-  substackPersistence = null;
-  activeFetches.clear();
-}
-
-export function setSubstackFetchTransportForTests(transport: ThrottledFetchTransport | null): void {
-  substackClient = createThrottledFetch({
-    requestsPerMinute: 10_000,
-    maxRetries: 0,
-    timeoutMs: 15_000,
-    defaultHeaders: DEFAULT_HEADERS,
-    dedupeGetRequests: false,
-    transport: transport ?? undefined,
-  });
-  activeFetches.clear();
-}
-
-function isAuthState(value: unknown): value is SubstackAuthState {
-  const record = value as Partial<SubstackAuthState> | null;
-  return !!record
-    && typeof record === "object"
-    && typeof record.email === "string"
-    && typeof record.sid === "string"
-    && record.sid.length > 0
-    && typeof record.loggedInAt === "number";
-}
-
-export function getStoredSubstackAuth(): SubstackAuthState | null {
-  const auth = substackPersistence?.getState<unknown>(AUTH_STATE_KEY, { schemaVersion: AUTH_SCHEMA_VERSION });
-  return isAuthState(auth) ? auth : null;
-}
-
-export function storeSubstackAuth(auth: SubstackAuthState): void {
-  substackPersistence?.setState(AUTH_STATE_KEY, auth, { schemaVersion: AUTH_SCHEMA_VERSION });
-}
-
-export function clearSubstackAuth(): void {
-  substackPersistence?.deleteState(AUTH_STATE_KEY);
-}
-
-export function requireAuth(): SubstackAuthState {
-  const auth = getStoredSubstackAuth();
-  if (!auth) throw new SubstackAuthError();
-  return auth;
-}
-
-function authHeaders(auth: SubstackAuthState): Record<string, string> {
-  return {
-    cookie: `substack.sid=${auth.sid}; substack.lli=${auth.lli?.trim() || "1"}`,
-  };
-}
-
-export async function parseErrorText(response: Response): Promise<string> {
-  try {
-    const text = await response.text();
-    return text.replace(/\s+/g, " ").trim().slice(0, 240);
-  } catch {
-    return "";
-  }
-}
-
-export async function substackFetch(url: string, init?: RequestInit): Promise<Response> {
-  return substackClient.fetch(url, init);
-}
-
-export async function fetchJsonAuthenticated<T = unknown>(url: string, auth: SubstackAuthState): Promise<T> {
-  const response = await substackClient.fetch(url, {
-    headers: authHeaders(auth),
-  });
-  if (response.status === 401 || response.status === 403) {
-    clearSubstackAuth();
-    throw new SubstackAuthError("Substack session expired");
-  }
-  if (!response.ok) {
-    const detail = await parseErrorText(response);
-    throw new Error(`Substack HTTP ${response.status}${detail ? `: ${detail}` : ""}`);
-  }
-  return response.json() as Promise<T>;
-}
-
-export function readResource<T>(kind: string, key: string, allowExpired = false): PersistedResourceValue<T> | null {
-  return substackPersistence?.getResource<T>(kind, key, {
-    sourceKey: CACHE_SOURCE,
-    schemaVersion: CACHE_SCHEMA_VERSION,
-    allowExpired,
-  }) ?? null;
-}
-
-function writeResource<T>(kind: string, key: string, value: T, cachePolicy: CachePolicy): PersistedResourceValue<T> | null {
-  return substackPersistence?.setResource<T>(kind, key, value, {
-    sourceKey: CACHE_SOURCE,
-    schemaVersion: CACHE_SCHEMA_VERSION,
-    cachePolicy,
-  }) ?? null;
-}
-
-export async function loadCachedResource<T>(
-  kind: keyof typeof CACHE_POLICIES,
-  key: string,
-  force: boolean,
-  loader: () => Promise<T>,
-): Promise<SubstackCachedData<T>> {
-  const fresh = readResource<T>(kind, key, false);
-  if (!force && fresh && !fresh.stale) {
-    return { data: fresh.value, fetchedAt: fresh.fetchedAt, stale: false };
-  }
-
-  const activeKey = `${kind}:${key}`;
-  const active = activeFetches.get(activeKey);
-  if (active) return active as Promise<SubstackCachedData<T>>;
-
-  const fallback = fresh ?? readResource<T>(kind, key, true);
-  const promise = loader()
-    .then((data) => {
-      const record = writeResource(kind, key, data, CACHE_POLICIES[kind]);
-      return {
-        data,
-        fetchedAt: record?.fetchedAt ?? Date.now(),
-        stale: false,
-      };
-    })
-    .catch((error) => {
-      if (fallback) {
-        return {
-          data: fallback.value,
-          fetchedAt: fallback.fetchedAt,
-          stale: true,
-        };
-      }
-      throw error;
-    })
-    .finally(() => {
-      activeFetches.delete(activeKey);
-    });
-  activeFetches.set(activeKey, promise as Promise<SubstackCachedData<unknown>>);
-  return promise;
-}
-
-export function resolveSubstackUrl(value: unknown, baseUrl = SUBSTACK_ORIGIN): string | null {
-  if (typeof value !== "string" || !value.trim()) return null;
-  const direct = normalizedHttpUrl(value);
-  if (direct) return direct;
-  try {
-    return normalizedHttpUrl(new URL(value, baseUrl).toString());
-  } catch {
-    return null;
-  }
-}

package/src/plugins/builtin/substack/api/types.ts

@@ -1,42 +0,0 @@
-import type {
-  SubstackArticleDetail,
-  SubstackArticleSummary,
-  SubstackPublication,
-} from "../types";
-
-export interface SubstackAuthState {
-  email: string;
-  sid: string;
-  lli: string | null;
-  loggedInAt: number;
-}
-
-export interface SubstackHomeData {
-  subscriptions: SubstackPublication[];
-  feed: SubstackArticleSummary[];
-  fetchedAt: number;
-  stale: boolean;
-  hasMore?: boolean;
-  nextCursor?: string | null;
-}
-
-export interface SubstackPublicationFeedPage {
-  items: SubstackArticleSummary[];
-  nextOffset: number | null;
-  hasMore: boolean;
-}
-
-export interface SubstackCachedData<T> {
-  data: T;
-  fetchedAt: number;
-  stale: boolean;
-}
-
-export class SubstackAuthError extends Error {
-  constructor(message = "Substack login required") {
-    super(message);
-    this.name = "SubstackAuthError";
-  }
-}
-
-export type { SubstackArticleDetail, SubstackArticleSummary, SubstackPublication };

package/src/plugins/builtin/substack/api.test.ts

@@ -1,298 +0,0 @@
-import { afterEach, describe, expect, test } from "bun:test";
-import { MemoryPluginPersistence } from "../../../test-support/plugin-persistence";
-import {
-  attachSubstackPersistence,
-  resetSubstackPersistence,
-  setSubstackFetchTransportForTests,
-} from "./api/store";
-import {
-  completeSubstackMagicLink,
-  completeSubstackOtpLogin,
-  parseCookiesFromSetCookie,
-  splitSetCookieHeader,
-} from "./api/auth";
-import {
-  getCachedSubstackArticleDetail,
-  getCachedSubstackHome,
-  getCachedSubstackPublicationFeed,
-} from "./api/cache";
-import {
-  loadSubstackArticleDetail,
-  loadSubstackHome,
-  loadSubstackPublicationFeed,
-} from "./api/loaders";
-import type { SubstackArticleSummary, SubstackPublication } from "./types";
-
-const EMAIL = "reader@example.com";
-const LOGIN_URL = "https://substack.com/login-token";
-
-afterEach(() => {
-  setSubstackFetchTransportForTests(null);
-  resetSubstackPersistence();
-});
-
-function publication(overrides: Partial<SubstackPublication> = {}): SubstackPublication {
-  return {
-    id: "alpha",
-    name: "Alpha Research",
-    subdomain: null,
-    baseUrl: "https://alpha.example",
-    description: null,
-    logoUrl: null,
-    latestPublishedAt: null,
-    ...overrides,
-  };
-}
-
-function article(overrides: Partial<SubstackArticleSummary> = {}): SubstackArticleSummary {
-  const pub = publication();
-  return {
-    id: "1",
-    title: "Markets and $SPY",
-    publicationId: pub.id,
-    publicationName: pub.name,
-    publicationSubdomain: pub.subdomain,
-    publicationBaseUrl: pub.baseUrl,
-    url: `${pub.baseUrl}/p/markets`,
-    slug: "markets",
-    publishedAt: "2026-06-05T10:00:00Z",
-    subtitle: null,
-    previewText: null,
-    bodyHtml: null,
-    imageUrls: [],
-    wordCount: 220,
-    readMinutes: 1,
-    ...overrides,
-  };
-}
-
-function installAuthenticatedTransport(
-  handler: (url: string, init?: RequestInit) => Response | undefined | Promise<Response | undefined>,
-): string[] {
-  const requests: string[] = [];
-  setSubstackFetchTransportForTests(async (url, init) => {
-    requests.push(url);
-    if (url === LOGIN_URL) {
-      return new Response(null, {
-        status: 302,
-        headers: {
-          location: "/reader",
-          "set-cookie": "substack.sid=sid123; Path=/; HttpOnly",
-        },
-      });
-    }
-    if (url === "https://substack.com/reader") {
-      return new Response("ok", { status: 200 });
-    }
-    return await handler(url, init) ?? new Response(`unexpected ${url}`, { status: 500 });
-  });
-  return requests;
-}
-
-describe("Substack auth", () => {
-  test("parses combined cookie headers and follows OTP redirects for the session cookie", async () => {
-    const headers = splitSetCookieHeader(
-      "foo=bar; Expires=Wed, 21 Oct 2026 07:28:00 GMT; Path=/, substack.sid=sid123; Path=/; HttpOnly, substack.lli=1; Path=/",
-    );
-    expect(parseCookiesFromSetCookie(headers).get("substack.sid")).toBe("sid123");
-
-    attachSubstackPersistence(new MemoryPluginPersistence());
-    const requests: string[] = [];
-    setSubstackFetchTransportForTests(async (url, init) => {
-      requests.push(url);
-      if (url === "https://substack.com/api/v1/email-otp-login/complete") {
-        expect(init?.method).toBe("POST");
-        expect(JSON.parse(String(init?.body))).toMatchObject({ code: "123456", email: EMAIL });
-        return Response.json({ redirect: "/reader" });
-      }
-      if (url === "https://substack.com/reader") {
-        return new Response(null, {
-          status: 302,
-          headers: {
-            location: "/home",
-            "set-cookie": "substack.sid=sid789; Path=/; HttpOnly",
-          },
-        });
-      }
-      if (url === "https://substack.com/home") {
-        return new Response("ok", {
-          headers: { "set-cookie": "substack.lli=1; Path=/" },
-        });
-      }
-      return new Response(`unexpected ${url}`, { status: 500 });
-    });
-
-    const auth = await completeSubstackOtpLogin("123456", EMAIL);
-
-    expect(auth).toMatchObject({ sid: "sid789", lli: "1", email: EMAIL });
-    expect(requests).toEqual([
-      "https://substack.com/api/v1/email-otp-login/complete",
-      "https://substack.com/reader",
-      "https://substack.com/home",
-    ]);
-  });
-});
-
-describe("Substack cached data loading", () => {
-  test("loads subscriptions plus paginated feed and reuses the cached result", async () => {
-    attachSubstackPersistence(new MemoryPluginPersistence());
-    const feedUrls: string[] = [];
-    const requests = installAuthenticatedTransport((url) => {
-      if (url === "https://substack.com/api/v1/subscriptions/page_v2") {
-        return Response.json({
-          result: {
-            subscriptions: [{ id: "sub-alpha", publication_id: "alpha" }],
-            publicationMap: {
-              alpha: { id: "alpha", name: "Alpha Research", base_url: "https://alpha.example" },
-            },
-          },
-        });
-      }
-      if (url.startsWith("https://substack.com/api/v1/reader/feed?")) {
-        feedUrls.push(url);
-        const parsed = new URL(url);
-        return Response.json(parsed.searchParams.has("cursor")
-          ? { items: [{ post: { id: "second", title: "Second", post_date: "2026-06-05T11:00:00Z", publication: { id: "alpha", name: "Alpha Research", base_url: "https://alpha.example" } } }] }
-          : {
-            items: [
-              { type: "comment", comment: { id: 1, body: "not a post" } },
-              { post: { id: "first", title: "First", post_date: "2026-06-05T10:00:00Z", publication: { id: "alpha", name: "Alpha Research", base_url: "https://alpha.example" } } },
-            ],
-            nextCursor: "cursor-2",
-          });
-      }
-    });
-
-    await completeSubstackMagicLink(LOGIN_URL, EMAIL);
-    const first = await loadSubstackHome(false);
-    const requestCountAfterFirstLoad = requests.length;
-    const second = await loadSubstackHome(false);
-
-    expect(first.subscriptions.map((item) => item.name)).toEqual(["Alpha Research"]);
-    expect(first.feed.map((item) => item.id)).toEqual(["first", "second"]);
-    expect(second.feed.map((item) => item.id)).toEqual(["first", "second"]);
-    expect(feedUrls).toHaveLength(2);
-    expect(new URL(feedUrls[1]!).searchParams.get("cursor")).toBe("cursor-2");
-    expect(requests).toHaveLength(requestCountAfterFirstLoad);
-  });
-
-  test("hydrates cached home, merged publication pages, and article detail synchronously", () => {
-    const persistence = new MemoryPluginPersistence();
-    attachSubstackPersistence(persistence);
-    const pub = publication({ name: "Alpha" });
-    const firstArticle = article({ id: "1", title: "Archive one", publicationName: "Alpha" });
-    const secondArticle = article({
-      id: "2",
-      title: "Archive two",
-      publicationName: "Alpha",
-      url: "https://alpha.example/p/archive-two",
-      slug: "archive-two",
-      publishedAt: "2026-06-04T10:00:00Z",
-      readMinutes: 2,
-      wordCount: 440,
-    });
-    const cacheOptions = { sourceKey: "substack", schemaVersion: 3 };
-
-    persistence.seedResource("subscriptions", "me", [pub], cacheOptions);
-    persistence.seedResource("feed", "subscribed", [firstArticle], cacheOptions);
-    persistence.seedResource("publication", "https://alpha.example:offset:0", {
-      items: [firstArticle],
-      nextOffset: 12,
-      hasMore: true,
-    }, cacheOptions);
-    persistence.seedResource("publication", "https://alpha.example:offset:12", {
-      items: [secondArticle],
-      nextOffset: null,
-      hasMore: false,
-    }, cacheOptions);
-    persistence.seedResource("post", "1", {
-      ...firstArticle,
-      bodyHtml: "<p>Cached body</p>",
-      contentText: "Cached body",
-      contentBlocks: [{ type: "paragraph", text: "Cached body" }],
-      linkUrls: [],
-    }, cacheOptions);
-
-    expect(getCachedSubstackHome()?.feed.map((item) => item.title)).toEqual(["Archive one"]);
-    expect(getCachedSubstackPublicationFeed(pub)?.data.items.map((item) => item.title)).toEqual([
-      "Archive one",
-      "Archive two",
-    ]);
-    expect(getCachedSubstackArticleDetail({ id: "1" })?.data.contentText).toBe("Cached body");
-  });
-});
-
-describe("Substack publication archive loading", () => {
-  test("uses archive page limits and offset pagination", async () => {
-    attachSubstackPersistence(new MemoryPluginPersistence());
-    const requests = installAuthenticatedTransport((url) => {
-      if (url === "https://alpha.example/api/v1/archive?sort=new&limit=12") {
-        return Response.json(Array.from({ length: 12 }, (_, index) => ({
-          id: index + 1,
-          title: `Archive ${index + 1}`,
-          post_date: "2026-06-05T10:00:00Z",
-          slug: `archive-${index + 1}`,
-        })));
-      }
-      if (url === "https://alpha.example/api/v1/archive?sort=new&limit=12&offset=12") {
-        return Response.json(Array.from({ length: 3 }, (_, index) => ({
-          id: index + 13,
-          title: `Archive ${index + 13}`,
-          post_date: "2026-06-05T10:00:00Z",
-          slug: `archive-${index + 13}`,
-        })));
-      }
-    });
-
-    await completeSubstackMagicLink(LOGIN_URL, EMAIL);
-    const first = await loadSubstackPublicationFeed(publication(), false);
-    const second = await loadSubstackPublicationFeed(publication(), false, first.data.nextOffset ?? 0);
-
-    expect(first.data).toMatchObject({ hasMore: true, nextOffset: 12 });
-    expect(first.data.items).toHaveLength(12);
-    expect(second.data.items.map((item) => item.id)).toEqual(["13", "14", "15"]);
-    expect(second.data.hasMore).toBe(false);
-    expect(requests).toContain("https://alpha.example/api/v1/archive?sort=new&limit=12");
-    expect(requests).toContain("https://alpha.example/api/v1/archive?sort=new&limit=12&offset=12");
-  });
-});
-
-describe("Substack article detail loading", () => {
-  test("prefers the Substack origin post endpoint over custom-domain teaser content", async () => {
-    attachSubstackPersistence(new MemoryPluginPersistence());
-    const requests = installAuthenticatedTransport((url) => {
-      if (url === "https://substack.com/api/v1/posts/by-id/42") {
-        return Response.json({
-          id: 42,
-          title: "Full post",
-          body_html: "<p>Preview.</p><p>Paid section with $NVDA and more detail.</p>",
-          publication: { id: "alpha", name: "Alpha", base_url: "https://alpha.example" },
-        });
-      }
-      if (url === "https://alpha.example/api/v1/posts/by-id/42") {
-        return Response.json({
-          id: 42,
-          title: "Full post",
-          body_html: "<p>Preview.</p>",
-          publication: { id: "alpha", name: "Alpha", base_url: "https://alpha.example" },
-        });
-      }
-    });
-
-    await completeSubstackMagicLink(LOGIN_URL, EMAIL);
-    const detail = await loadSubstackArticleDetail(article({
-      id: "42",
-      title: "Full post",
-      url: "https://alpha.example/p/full-post",
-      slug: "full-post",
-      previewText: "Preview.",
-      bodyHtml: "<p>Preview.</p>",
-      wordCount: 1,
-    }));
-
-    expect(detail.data.contentText).toContain("Paid section");
-    expect(detail.data.wordCount).toBeGreaterThan(1);
-    expect(requests).toContain("https://substack.com/api/v1/posts/by-id/42");
-    expect(requests).not.toContain("https://alpha.example/api/v1/posts/by-id/42");
-  });
-});