gloomberb 0.8.2 → 0.9.1

package/src/plugins/builtin/substack/api/cache.ts

@@ -0,0 +1,78 @@
+import type { PersistedResourceValue } from "../../../../types/persistence";
+import {
+  sortSubscriptionsByLatest,
+} from "../normalize";
+import type {
+  SubstackArticleDetail,
+  SubstackArticleSummary,
+  SubstackPublication,
+} from "../types";
+import type { SubstackCachedData, SubstackHomeData, SubstackPublicationFeedPage } from "./types";
+import { readResource } from "./store";
+
+const PUBLICATION_ARCHIVE_MAX_CACHED_PAGES = 20;
+
+export function publicationCacheKey(publication: SubstackPublication, offset: number): string {
+  return `${publication.baseUrl ?? publication.subdomain ?? publication.id}:offset:${offset}`;
+}
+
+function cachedDataFromResource<T>(entry: PersistedResourceValue<T> | null): SubstackCachedData<T> | null {
+  if (!entry) return null;
+  return {
+    data: entry.value,
+    fetchedAt: entry.fetchedAt,
+    stale: Boolean(entry.stale),
+  };
+}
+
+export function getCachedSubstackHome(): SubstackHomeData | null {
+  const subscriptionsEntry = readResource<SubstackPublication[]>("subscriptions", "me", false);
+  const feedEntry = readResource<SubstackArticleSummary[]>("feed", "subscribed", false);
+  if (!subscriptionsEntry || !feedEntry) return null;
+  return {
+    subscriptions: sortSubscriptionsByLatest(subscriptionsEntry.value, feedEntry.value),
+    feed: feedEntry.value,
+    fetchedAt: Math.max(subscriptionsEntry.fetchedAt, feedEntry.fetchedAt),
+    stale: Boolean(subscriptionsEntry.stale) || Boolean(feedEntry.stale),
+  };
+}
+
+export function getCachedSubstackPublicationFeed(
+  publication: SubstackPublication,
+): SubstackCachedData<SubstackPublicationFeedPage> | null {
+  let offset: number | null = 0;
+  let pageCount = 0;
+  let merged: SubstackPublicationFeedPage | null = null;
+  let fetchedAt = 0;
+  let stale = false;
+
+  while (offset != null && pageCount < PUBLICATION_ARCHIVE_MAX_CACHED_PAGES) {
+    const entry: PersistedResourceValue<SubstackPublicationFeedPage> | null = readResource<SubstackPublicationFeedPage>(
+      "publication",
+      publicationCacheKey(publication, offset),
+      false,
+    );
+    if (!entry) break;
+    const itemsById = new Map<string, SubstackArticleSummary>();
+    for (const article of merged?.items ?? []) itemsById.set(article.id, article);
+    for (const article of entry.value.items) itemsById.set(article.id, article);
+    merged = {
+      items: [...itemsById.values()],
+      nextOffset: entry.value.nextOffset,
+      hasMore: entry.value.hasMore,
+    };
+    fetchedAt = Math.max(fetchedAt, entry.fetchedAt);
+    stale = stale || Boolean(entry.stale);
+    offset = entry.value.nextOffset;
+    pageCount += 1;
+  }
+
+  if (!merged) return null;
+  return { data: merged, fetchedAt, stale };
+}
+
+export function getCachedSubstackArticleDetail(
+  article: Pick<SubstackArticleSummary, "id">,
+): SubstackCachedData<SubstackArticleDetail> | null {
+  return cachedDataFromResource(readResource<SubstackArticleDetail>("post", article.id, false));
+}

package/src/plugins/builtin/substack/api/loaders.ts

@@ -0,0 +1,146 @@
+import {
+  normalizeFeedItems,
+  normalizePostDetail,
+  normalizeSubscriptions,
+  sortSubscriptionsByLatest,
+} from "../normalize";
+import type {
+  SubstackArticleSummary,
+  SubstackPublication,
+} from "../types";
+import { SubstackAuthError, type SubstackCachedData, type SubstackHomeData, type SubstackPublicationFeedPage } from "./types";
+import { publicationCacheKey } from "./cache";
+import {
+  fetchJsonAuthenticated,
+  loadCachedResource,
+  requireAuth,
+  SUBSTACK_ORIGIN,
+} from "./store";
+
+const READER_FEED_TARGET_ITEMS = 50;
+const READER_FEED_MAX_PAGES = 12;
+const PUBLICATION_ARCHIVE_PAGE_LIMIT = 12;
+
+export async function loadSubstackHome(force = false): Promise<SubstackHomeData> {
+  const auth = requireAuth();
+  const [subscriptionsEntry, feedEntry] = await Promise.all([
+    loadCachedResource("subscriptions", "me", force, async () => {
+      const payload = await fetchJsonAuthenticated<unknown>(`${SUBSTACK_ORIGIN}/api/v1/subscriptions/page_v2`, auth);
+      return normalizeSubscriptions(payload);
+    }),
+    loadCachedResource("feed", "subscribed", force, async () => {
+      return fetchReaderFeedItems(auth, "subscribed");
+    }),
+  ]);
+  return {
+    subscriptions: sortSubscriptionsByLatest(subscriptionsEntry.data, feedEntry.data),
+    feed: feedEntry.data,
+    fetchedAt: Math.max(subscriptionsEntry.fetchedAt, feedEntry.fetchedAt),
+    stale: subscriptionsEntry.stale || feedEntry.stale,
+  };
+}
+
+function payloadCursor(payload: unknown): string | null {
+  const record = payload && typeof payload === "object" && !Array.isArray(payload)
+    ? payload as Record<string, unknown>
+    : null;
+  const cursor = record?.nextCursor ?? record?.next_cursor;
+  return typeof cursor === "string" && cursor.trim() ? cursor : null;
+}
+
+async function fetchReaderFeedItems(auth: ReturnType<typeof requireAuth>, tab: string): Promise<SubstackArticleSummary[]> {
+  const collected = new Map<string, SubstackArticleSummary>();
+  let cursor: string | null = null;
+  for (let page = 0; page < READER_FEED_MAX_PAGES && collected.size < READER_FEED_TARGET_ITEMS; page += 1) {
+    const url = new URL(`${SUBSTACK_ORIGIN}/api/v1/reader/feed`);
+    url.searchParams.set("tab", tab);
+    if (cursor) url.searchParams.set("cursor", cursor);
+    const payload = await fetchJsonAuthenticated<unknown>(url.toString(), auth);
+    for (const article of normalizeFeedItems(payload)) {
+      collected.set(article.id, article);
+      if (collected.size >= READER_FEED_TARGET_ITEMS) break;
+    }
+    const nextCursor = payloadCursor(payload);
+    if (!nextCursor || nextCursor === cursor) break;
+    cursor = nextCursor;
+  }
+  return [...collected.values()];
+}
+
+export async function loadSubstackPublicationFeed(
+  publication: SubstackPublication,
+  force = false,
+  offset = 0,
+): Promise<SubstackCachedData<SubstackPublicationFeedPage>> {
+  const auth = requireAuth();
+  const baseUrl = publication.baseUrl;
+  if (!baseUrl) {
+    return {
+      data: { items: [], nextOffset: null, hasMore: false },
+      fetchedAt: Date.now(),
+      stale: false,
+    };
+  }
+  return loadCachedResource("publication", publicationCacheKey(publication, offset), force, async () => {
+    const url = new URL("/api/v1/archive", baseUrl);
+    url.searchParams.set("sort", "new");
+    url.searchParams.set("limit", String(PUBLICATION_ARCHIVE_PAGE_LIMIT));
+    if (offset > 0) url.searchParams.set("offset", String(offset));
+    const payload = await fetchJsonAuthenticated<unknown>(url.toString(), auth);
+    const items = normalizeFeedItems(payload, publication);
+    const hasMore = items.length >= PUBLICATION_ARCHIVE_PAGE_LIMIT;
+    return {
+      items,
+      nextOffset: hasMore ? offset + PUBLICATION_ARCHIVE_PAGE_LIMIT : null,
+      hasMore,
+    };
+  });
+}
+
+function detailCandidateUrls(article: SubstackArticleSummary): string[] {
+  const urls: string[] = [];
+  if (/^\d+$/.test(article.id)) {
+    urls.push(new URL(`/api/v1/posts/by-id/${article.id}`, SUBSTACK_ORIGIN).toString());
+  }
+  const baseUrl = article.publicationBaseUrl
+    ?? (article.url ? new URL(article.url).origin : null);
+  if (baseUrl) {
+    if (/^\d+$/.test(article.id)) {
+      urls.push(new URL(`/api/v1/posts/by-id/${article.id}`, baseUrl).toString());
+    }
+    if (article.slug) {
+      urls.push(new URL(`/api/v1/posts/${article.slug}`, baseUrl).toString());
+    }
+    if (article.url) {
+      try {
+        const pathSlug = new URL(article.url).pathname.match(/\/p\/([^/?#]+)/)?.[1];
+        if (pathSlug) urls.push(new URL(`/api/v1/posts/${decodeURIComponent(pathSlug)}`, baseUrl).toString());
+      } catch {
+        // Ignore malformed article URLs; the direct post id/slug paths above still apply.
+      }
+    }
+  }
+  return [...new Set(urls)];
+}
+
+export async function loadSubstackArticleDetail(
+  article: SubstackArticleSummary,
+  force = false,
+): Promise<SubstackCachedData<ReturnType<typeof normalizePostDetail>>> {
+  const auth = requireAuth();
+  return loadCachedResource("post", article.id, force, async () => {
+    const urls = detailCandidateUrls(article);
+    const errors: string[] = [];
+    for (const url of urls) {
+      try {
+        const payload = await fetchJsonAuthenticated<unknown>(url, auth);
+        return normalizePostDetail(payload, article);
+      } catch (error) {
+        if (error instanceof SubstackAuthError) throw error;
+        errors.push(error instanceof Error ? error.message : String(error));
+      }
+    }
+    if (article.bodyHtml || urls.length === 0) return normalizePostDetail(article, article);
+    throw new Error(errors[0] ?? "Substack post content unavailable");
+  });
+}

package/src/plugins/builtin/substack/api/store.ts

@@ -0,0 +1,186 @@
+import type { PluginPersistence } from "../../../../types/plugin";
+import type { CachePolicy, PersistedResourceValue } from "../../../../types/persistence";
+import { createThrottledFetch, type ThrottledFetchTransport } from "../../../../utils/throttled-fetch";
+import { normalizedHttpUrl } from "../../../../utils/url";
+import { SubstackAuthError, type SubstackAuthState, type SubstackCachedData } from "./types";
+
+export const SUBSTACK_ORIGIN = "https://substack.com";
+
+const AUTH_STATE_KEY = "auth";
+const AUTH_SCHEMA_VERSION = 1;
+export const CACHE_SOURCE = "substack";
+export const CACHE_SCHEMA_VERSION = 3;
+const DEFAULT_HEADERS = {
+  accept: "application/json,text/plain,*/*",
+  "accept-language": "en-US,en;q=0.9",
+  "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
+};
+const CACHE_POLICIES = {
+  subscriptions: { staleMs: 30 * 60 * 1000, expireMs: 14 * 24 * 60 * 60 * 1000 },
+  feed: { staleMs: 5 * 60 * 1000, expireMs: 3 * 24 * 60 * 60 * 1000 },
+  publication: { staleMs: 10 * 60 * 1000, expireMs: 7 * 24 * 60 * 60 * 1000 },
+  post: { staleMs: 30 * 24 * 60 * 60 * 1000, expireMs: 365 * 24 * 60 * 60 * 1000 },
+} satisfies Record<string, CachePolicy>;
+
+let substackPersistence: PluginPersistence | null = null;
+let substackClient = createThrottledFetch({
+  requestsPerMinute: 60,
+  maxRetries: 1,
+  timeoutMs: 15_000,
+  defaultHeaders: DEFAULT_HEADERS,
+  dedupeGetRequests: false,
+});
+const activeFetches = new Map<string, Promise<SubstackCachedData<unknown>>>();
+
+export function attachSubstackPersistence(persistence: PluginPersistence): void {
+  substackPersistence = persistence;
+}
+
+export function resetSubstackPersistence(): void {
+  substackPersistence = null;
+  activeFetches.clear();
+}
+
+export function setSubstackFetchTransportForTests(transport: ThrottledFetchTransport | null): void {
+  substackClient = createThrottledFetch({
+    requestsPerMinute: 10_000,
+    maxRetries: 0,
+    timeoutMs: 15_000,
+    defaultHeaders: DEFAULT_HEADERS,
+    dedupeGetRequests: false,
+    transport: transport ?? undefined,
+  });
+  activeFetches.clear();
+}
+
+function isAuthState(value: unknown): value is SubstackAuthState {
+  const record = value as Partial<SubstackAuthState> | null;
+  return !!record
+    && typeof record === "object"
+    && typeof record.email === "string"
+    && typeof record.sid === "string"
+    && record.sid.length > 0
+    && typeof record.loggedInAt === "number";
+}
+
+export function getStoredSubstackAuth(): SubstackAuthState | null {
+  const auth = substackPersistence?.getState<unknown>(AUTH_STATE_KEY, { schemaVersion: AUTH_SCHEMA_VERSION });
+  return isAuthState(auth) ? auth : null;
+}
+
+export function storeSubstackAuth(auth: SubstackAuthState): void {
+  substackPersistence?.setState(AUTH_STATE_KEY, auth, { schemaVersion: AUTH_SCHEMA_VERSION });
+}
+
+export function clearSubstackAuth(): void {
+  substackPersistence?.deleteState(AUTH_STATE_KEY);
+}
+
+export function requireAuth(): SubstackAuthState {
+  const auth = getStoredSubstackAuth();
+  if (!auth) throw new SubstackAuthError();
+  return auth;
+}
+
+function authHeaders(auth: SubstackAuthState): Record<string, string> {
+  return {
+    cookie: `substack.sid=${auth.sid}; substack.lli=${auth.lli?.trim() || "1"}`,
+  };
+}
+
+export async function parseErrorText(response: Response): Promise<string> {
+  try {
+    const text = await response.text();
+    return text.replace(/\s+/g, " ").trim().slice(0, 240);
+  } catch {
+    return "";
+  }
+}
+
+export async function substackFetch(url: string, init?: RequestInit): Promise<Response> {
+  return substackClient.fetch(url, init);
+}
+
+export async function fetchJsonAuthenticated<T = unknown>(url: string, auth: SubstackAuthState): Promise<T> {
+  const response = await substackClient.fetch(url, {
+    headers: authHeaders(auth),
+  });
+  if (response.status === 401 || response.status === 403) {
+    clearSubstackAuth();
+    throw new SubstackAuthError("Substack session expired");
+  }
+  if (!response.ok) {
+    const detail = await parseErrorText(response);
+    throw new Error(`Substack HTTP ${response.status}${detail ? `: ${detail}` : ""}`);
+  }
+  return response.json() as Promise<T>;
+}
+
+export function readResource<T>(kind: string, key: string, allowExpired = false): PersistedResourceValue<T> | null {
+  return substackPersistence?.getResource<T>(kind, key, {
+    sourceKey: CACHE_SOURCE,
+    schemaVersion: CACHE_SCHEMA_VERSION,
+    allowExpired,
+  }) ?? null;
+}
+
+function writeResource<T>(kind: string, key: string, value: T, cachePolicy: CachePolicy): PersistedResourceValue<T> | null {
+  return substackPersistence?.setResource<T>(kind, key, value, {
+    sourceKey: CACHE_SOURCE,
+    schemaVersion: CACHE_SCHEMA_VERSION,
+    cachePolicy,
+  }) ?? null;
+}
+
+export async function loadCachedResource<T>(
+  kind: keyof typeof CACHE_POLICIES,
+  key: string,
+  force: boolean,
+  loader: () => Promise<T>,
+): Promise<SubstackCachedData<T>> {
+  const fresh = readResource<T>(kind, key, false);
+  if (!force && fresh && !fresh.stale) {
+    return { data: fresh.value, fetchedAt: fresh.fetchedAt, stale: false };
+  }
+
+  const activeKey = `${kind}:${key}`;
+  const active = activeFetches.get(activeKey);
+  if (active) return active as Promise<SubstackCachedData<T>>;
+
+  const fallback = fresh ?? readResource<T>(kind, key, true);
+  const promise = loader()
+    .then((data) => {
+      const record = writeResource(kind, key, data, CACHE_POLICIES[kind]);
+      return {
+        data,
+        fetchedAt: record?.fetchedAt ?? Date.now(),
+        stale: false,
+      };
+    })
+    .catch((error) => {
+      if (fallback) {
+        return {
+          data: fallback.value,
+          fetchedAt: fallback.fetchedAt,
+          stale: true,
+        };
+      }
+      throw error;
+    })
+    .finally(() => {
+      activeFetches.delete(activeKey);
+    });
+  activeFetches.set(activeKey, promise as Promise<SubstackCachedData<unknown>>);
+  return promise;
+}
+
+export function resolveSubstackUrl(value: unknown, baseUrl = SUBSTACK_ORIGIN): string | null {
+  if (typeof value !== "string" || !value.trim()) return null;
+  const direct = normalizedHttpUrl(value);
+  if (direct) return direct;
+  try {
+    return normalizedHttpUrl(new URL(value, baseUrl).toString());
+  } catch {
+    return null;
+  }
+}

package/src/plugins/builtin/substack/api/types.ts

@@ -0,0 +1,40 @@
+import type {
+  SubstackArticleDetail,
+  SubstackArticleSummary,
+  SubstackPublication,
+} from "../types";
+
+export interface SubstackAuthState {
+  email: string;
+  sid: string;
+  lli: string | null;
+  loggedInAt: number;
+}
+
+export interface SubstackHomeData {
+  subscriptions: SubstackPublication[];
+  feed: SubstackArticleSummary[];
+  fetchedAt: number;
+  stale: boolean;
+}
+
+export interface SubstackPublicationFeedPage {
+  items: SubstackArticleSummary[];
+  nextOffset: number | null;
+  hasMore: boolean;
+}
+
+export interface SubstackCachedData<T> {
+  data: T;
+  fetchedAt: number;
+  stale: boolean;
+}
+
+export class SubstackAuthError extends Error {
+  constructor(message = "Substack login required") {
+    super(message);
+    this.name = "SubstackAuthError";
+  }
+}
+
+export type { SubstackArticleDetail, SubstackArticleSummary, SubstackPublication };