gloomberb 0.8.1 → 0.9.0

package/src/plugins/builtin/sec/index.tsx

@@ -1,10 +1,10 @@
 import { Text } from "../../../ui";
 import { useEffect, useMemo, useRef, useState } from "react";
 import type { GloomPlugin } from "../../../types/plugin";
-import type { SecFilingItem } from "../../../types/data-provider";
-import { useResolvedEntryValue, useSecFilingContent, useSecFilingsQuery } from "../../../market-data/hooks";
+import type { SecFilingDocument, SecFilingItem } from "../../../types/data-provider";
+import { useResolvedEntryValue, useSecFilingContent, useSecFilingDocuments, useSecFilingsQuery } from "../../../market-data/hooks";
 import { instrumentFromTicker } from "../../../market-data/request-types";
-import { usePluginPaneState } from "../../runtime";
+import { useDebouncedPluginPaneState } from "../../runtime";
 import { usePaneTicker } from "../../../state/app/context";
 import { colors } from "../../../theme/colors";
 import { FeedDataTableStackView, Spinner, useExternalLinkFooter, type FeedDataTableItem } from "../../../components";
@@ -17,6 +17,14 @@ import {
   formatFilingMetaDate,
   renderFilingNotice,
 } from "./filing-display";
+import {
+  documentContentKey,
+  documentContentTarget,
+  documentHeading,
+  formatCompactDocumentLabel,
+  isDefaultVisibleFilingDocument,
+  isInlineExhibitDocument,
+} from "./filing-documents";
 
 const SEC_FILING_LIMIT = 50;
 const OWNERSHIP_FORMS = new Set(["3", "4", "5"]);
@@ -80,6 +88,50 @@ function buildDetailBody(filing: SecFilingItem): string {
     : "No additional SEC filing description is available for this entry.";
 }
 
+function buildDetailBodyWithDocuments({
+  filing,
+  documents,
+  documentsLoading,
+  contentCache,
+  primaryContent,
+}: {
+  filing: SecFilingItem;
+  documents: SecFilingDocument[];
+  documentsLoading: boolean;
+  contentCache: Map<string, string | null>;
+  primaryContent: string;
+}): string {
+  const lines: string[] = [];
+  lines.push("Documents");
+  if (documentsLoading && documents.length === 0) {
+    lines.push("Loading filing documents...");
+  } else if (documents.length === 0) {
+    lines.push("No filing documents were listed for this filing.");
+  } else {
+    const visibleDocuments = documents.filter(isDefaultVisibleFilingDocument);
+    lines.push(...visibleDocuments.map(formatCompactDocumentLabel));
+    const hiddenCount = documents.length - visibleDocuments.length;
+    if (hiddenCount > 0) lines.push(`+ ${hiddenCount} support documents hidden`);
+  }
+
+  const exhibits = documents.filter(isInlineExhibitDocument);
+  if (exhibits.length > 0) {
+    lines.push("", "Inline Exhibits");
+    for (const document of exhibits) {
+      const key = documentContentKey(filing, document);
+      const hasContent = contentCache.has(key);
+      const content = contentCache.get(key);
+      lines.push("", documentHeading(document));
+      lines.push(hasContent
+        ? content || "Readable document content was not available for this exhibit."
+        : "Loading exhibit content...");
+    }
+  }
+
+  lines.push("", "Primary Filing Content", primaryContent);
+  return lines.join("\n");
+}
+
 function buildForm4Preview(content: string | null): string | null {
   if (!content) return null;
   const tx = parseForm4Xml(content);
@@ -134,6 +186,8 @@ function toFeedItems(
   selectedAccessionNumber: string | undefined,
   contentCache: Map<string, string | null>,
   loadingContent: boolean,
+  selectedDocuments: SecFilingDocument[],
+  loadingDocuments: boolean,
 ): FeedDataTableItem[] {
   return filings.map((filing) => {
     const displayTitle = getFilingDisplayTitle(filing);
@@ -152,6 +206,19 @@ function toFeedItems(
     const form4Detail = isOwnership && hasFetchedContent
       ? buildForm4Detail(fetchedContent ?? null, filing)
       : null;
+    const selected = filing.accessionNumber === selectedAccessionNumber;
+    const primaryDetailBody = loadingContent && selected
+      ? "Loading filing content..."
+      : form4Detail ?? fetchedContent ?? fallbackBody;
+    const detailBody = selected
+      ? buildDetailBodyWithDocuments({
+          filing,
+          documents: selectedDocuments,
+          documentsLoading: loadingDocuments,
+          contentCache,
+          primaryContent: primaryDetailBody,
+        })
+      : form4Detail ?? fallbackBody;
 
     const enrichedTitle = formDesc
       ? `${displayTitle} — ${formDesc}`
@@ -170,13 +237,7 @@ function toFeedItems(
         `Accession ${filing.accessionNumber}`,
         ...(filing.items ? [`Items ${filing.items}`] : []),
       ],
-      detailBody: filing.accessionNumber === selectedAccessionNumber
-        ? (
-            loadingContent
-              ? "Loading filing content..."
-              : form4Detail ?? fetchedContent ?? fallbackBody
-          )
-        : form4Detail ?? fallbackBody,
+      detailBody,
     };
   });
 }
@@ -184,7 +245,7 @@ function toFeedItems(
 function SecView({ width, height, focused }: { width: number; height: number; focused: boolean }) {
   const { ticker } = usePaneTicker();
   const selectionKey = `selectedIdx:${ticker?.metadata.ticker ?? "none"}`;
-  const [selectedIdx, setSelectedIdx] = usePluginPaneState<number>(selectionKey, 0);
+  const [selectedIdx, setSelectedIdx] = useDebouncedPluginPaneState<number>(selectionKey, 0);
   const [contentCache, setContentCache] = useState<Map<string, string | null>>(new Map());
   const [openItemId, setOpenItemId] = useState<string | null>(null);
   const contentFetchRef = useRef(0);
@@ -204,28 +265,67 @@ function SecView({ width, height, focused }: { width: number; height: number; fo
     contentFetchRef.current += 1;
   }, [eligibleTicker, ticker?.metadata.exchange, ticker?.metadata.ticker]);
 
-  const selected = filings[selectedIdx];
   const openFiling = openItemId
     ? filings.find((filing) => filing.accessionNumber === openItemId) ?? null
     : null;
-  const cachedSelectedContent = selected ? contentCache.get(selected.accessionNumber) : undefined;
+  const cachedOpenContent = openFiling ? contentCache.get(openFiling.accessionNumber) : undefined;
   const filingContentEntry = useSecFilingContent(
-    selected && cachedSelectedContent === undefined ? selected : null,
+    openFiling && cachedOpenContent === undefined ? openFiling : null,
   );
   const filingContent = useResolvedEntryValue(filingContentEntry);
   const loadingContent = filingContentEntry?.phase === "loading" || filingContentEntry?.phase === "refreshing";
+  const documentsEntry = useSecFilingDocuments(openFiling ?? null);
+  const openDocuments = useResolvedEntryValue(documentsEntry) ?? [];
+  const loadingDocuments = !!openFiling && (
+    documentsEntry?.phase === "idle"
+    || documentsEntry?.phase === "loading"
+    || documentsEntry?.phase === "refreshing"
+  );
 
   useEffect(() => {
-    if (!selected) return;
-    if (cachedSelectedContent !== undefined) return;
+    if (!openFiling) return;
+    if (cachedOpenContent !== undefined) return;
     if (filingContentEntry?.phase === "error") {
-      setContentCache((prev) => new Map(prev).set(selected.accessionNumber, null));
+      setContentCache((prev) => new Map(prev).set(openFiling.accessionNumber, null));
       return;
     }
     if (filingContent !== null) {
-      setContentCache((prev) => new Map(prev).set(selected.accessionNumber, filingContent));
+      setContentCache((prev) => new Map(prev).set(openFiling.accessionNumber, filingContent));
+    }
+  }, [cachedOpenContent, filingContent, filingContentEntry?.phase, openFiling]);
+
+  const documentFetchStartedRef = useRef(new Set<string>());
+  useEffect(() => {
+    if (!openFiling || openDocuments.length === 0) return;
+    const coordinator = getSharedMarketDataCoordinator();
+    if (!coordinator) return;
+    const gen = contentFetchRef.current;
+    const documentsToFetch = openDocuments.filter((document) => {
+      if (!isInlineExhibitDocument(document)) return false;
+      const key = documentContentKey(openFiling, document);
+      return !contentCache.has(key) && !documentFetchStartedRef.current.has(key);
+    });
+    if (documentsToFetch.length === 0) return;
+
+    for (const document of documentsToFetch) {
+      documentFetchStartedRef.current.add(documentContentKey(openFiling, document));
     }
-  }, [cachedSelectedContent, filingContent, filingContentEntry?.phase, selected]);
+
+    (async () => {
+      for (const document of documentsToFetch) {
+        if (contentFetchRef.current !== gen) return;
+        const key = documentContentKey(openFiling, document);
+        try {
+          const entry = await coordinator.loadSecFilingContent(documentContentTarget(openFiling, document));
+          if (contentFetchRef.current !== gen) return;
+          setContentCache((prev) => new Map(prev).set(key, entry.data ?? entry.lastGoodData ?? null));
+        } catch {
+          if (contentFetchRef.current !== gen) return;
+          setContentCache((prev) => new Map(prev).set(key, null));
+        }
+      }
+    })();
+  }, [contentCache, openFiling, openDocuments]);
 
   useEffect(() => {
     if (filings.length > 0 && selectedIdx >= filings.length) {
@@ -235,6 +335,11 @@ function SecView({ width, height, focused }: { width: number; height: number; fo
 
   // Background prefetch Form 4 content for enriched previews
   const prefetchStartedRef = useRef(new Set<string>());
+  useEffect(() => {
+    prefetchStartedRef.current.clear();
+    documentFetchStartedRef.current.clear();
+  }, [eligibleTicker, ticker?.metadata.exchange, ticker?.metadata.ticker]);
+
   useEffect(() => {
     if (filings.length === 0) return;
     const coordinator = getSharedMarketDataCoordinator();
@@ -288,7 +393,14 @@ function SecView({ width, height, focused }: { width: number; height: number; fo
       width={width}
       height={height}
       focused={focused}
-      items={toFeedItems(filings, selected?.accessionNumber, contentCache, loadingContent)}
+      items={toFeedItems(
+        filings,
+        openFiling?.accessionNumber,
+        contentCache,
+        loadingContent,
+        openDocuments,
+        loadingDocuments,
+      )}
       selectedIdx={selectedIdx}
       onSelect={setSelectedIdx}
       onOpenItemIdChange={setOpenItemId}

package/src/plugins/builtin/sectors/index.tsx

@@ -71,11 +71,6 @@ function SectorPerformancePane({ focused, width, height }: PaneProps) {
     label: collection.label,
     value: collection.id,
   })), []);
-  const selectedIdx = selectedEtf
-    ? sortedRows.findIndex((row) => row.etf === selectedEtf)
-    : -1;
-  const activeIdx = selectedIdx >= 0 ? selectedIdx : (sortedRows.length > 0 ? 0 : -1);
-
   const fetchAll = useCallback(() => {
     fetchGenRef.current += 1;
     const gen = fetchGenRef.current;
@@ -189,19 +184,6 @@ function SectorPerformancePane({ focused, width, height }: PaneProps) {
     setSortPreference((current) => nextSortPreference(current, columnId));
   }, [setSortPreference]);
 
-  const selectIndex = useCallback((index: number) => {
-    const row = sortedRows[index];
-    if (row) setSelectedEtf(row.etf);
-  }, [setSelectedEtf, sortedRows]);
-
-  const selectRow = useCallback((row: SectorRow) => {
-    if (row.etf === selectedEtf) {
-      openRow(row);
-      return;
-    }
-    setSelectedEtf(row.etf);
-  }, [openRow, selectedEtf, setSelectedEtf]);
-
   const handleTableKeyDown = useCallback((event: DataTableKeyEvent) => {
     if (event.name === "r") {
       event.preventDefault?.();
@@ -291,9 +273,19 @@ function SectorPerformancePane({ focused, width, height }: PaneProps) {
   return (
     <DataTableView<SectorRow, SectorColumn>
       focused={focused}
-      selectedIndex={activeIdx}
-      onSelectIndex={selectIndex}
-      onActivateIndex={(_index, row) => openRow(row)}
+      selection={{
+        kind: "id",
+        selectedId: selectedEtf,
+        getId: (row) => row.etf,
+        onChange: (id, row, _index, reason) => {
+          if (reason === "pointer" && id === selectedEtf) {
+            openRow(row);
+            return;
+          }
+          setSelectedEtf(id);
+        },
+      }}
+      onActivate={openRow}
       onRootKeyDown={handleTableKeyDown}
       rootBefore={rootBefore}
       rootWidth={width}
@@ -305,8 +297,6 @@ function SectorPerformancePane({ focused, width, height }: PaneProps) {
       sortDirection={sortPreference.direction}
       onHeaderClick={handleHeaderClick}
       getItemKey={(row) => row.etf}
-      isSelected={(row) => row.etf === selectedEtf}
-      onSelect={selectRow}
       renderCell={renderCell}
       emptyStateTitle="No sector data available"
       showHorizontalScrollbar={false}

package/src/plugins/builtin/shared/ticker-request.ts

@@ -11,6 +11,7 @@ export function useBoundTicker() {
   const { symbol, ticker } = usePaneTicker();
   return {
     symbol,
+    ticker,
     exchange: ticker?.metadata.exchange ?? "",
     currency: ticker?.metadata.currency ?? "USD",
   };

package/src/plugins/builtin/substack/api/auth.ts

@@ -0,0 +1,185 @@
+import type { SubstackAuthState } from "./types";
+import {
+  parseErrorText,
+  resolveSubstackUrl,
+  storeSubstackAuth,
+  substackFetch,
+  SUBSTACK_ORIGIN,
+} from "./store";
+
+export async function requestSubstackMagicLink(email: string): Promise<void> {
+  const normalizedEmail = email.trim();
+  if (!/^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(normalizedEmail)) {
+    throw new Error("Enter a valid email address");
+  }
+  const response = await substackFetch(`${SUBSTACK_ORIGIN}/api/v1/email-login`, {
+    method: "POST",
+    headers: {
+      "content-type": "application/json",
+    },
+    body: JSON.stringify({ email: normalizedEmail }),
+  });
+  if (!response.ok) {
+    const detail = await parseErrorText(response);
+    throw new Error(`Substack login link failed (${response.status})${detail ? `: ${detail}` : ""}`);
+  }
+}
+
+export function splitSetCookieHeader(header: string): string[] {
+  const parts: string[] = [];
+  let start = 0;
+  let inExpires = false;
+  for (let i = 0; i < header.length; i += 1) {
+    const char = header[i];
+    if (char === ",") {
+      const fragment = header.slice(start, i);
+      inExpires = /expires=/i.test(fragment) && !/;\s*(?:max-age|domain|path|secure|httponly|samesite)\b/i.test(fragment);
+      if (!inExpires) {
+        parts.push(fragment.trim());
+        start = i + 1;
+      }
+    } else if (char === ";") {
+      inExpires = false;
+    }
+  }
+  const tail = header.slice(start).trim();
+  if (tail) parts.push(tail);
+  return parts;
+}
+
+function getSetCookieHeaders(headers: Headers): string[] {
+  const withGetSetCookie = headers as Headers & { getSetCookie?: () => string[] };
+  const direct = withGetSetCookie.getSetCookie?.();
+  if (direct && direct.length > 0) return direct;
+  const header = headers.get("set-cookie");
+  return header ? splitSetCookieHeader(header) : [];
+}
+
+export function parseCookiesFromSetCookie(headers: string[]): Map<string, string> {
+  const cookies = new Map<string, string>();
+  for (const header of headers) {
+    const first = header.split(";")[0] ?? "";
+    const eq = first.indexOf("=");
+    if (eq <= 0) continue;
+    const name = first.slice(0, eq).trim();
+    const value = first.slice(eq + 1).trim();
+    if (!name) continue;
+    cookies.set(name, value);
+  }
+  return cookies;
+}
+
+function cookieHeaderFromJar(jar: Map<string, string>): string {
+  return [...jar.entries()].map(([name, value]) => `${name}=${value}`).join("; ");
+}
+
+function mergeSetCookies(cookieJar: Map<string, string>, response: Response): void {
+  for (const [name, value] of parseCookiesFromSetCookie(getSetCookieHeaders(response.headers))) {
+    cookieJar.set(name, value);
+  }
+}
+
+async function followRedirectsForCookies(startUrl: string, cookieJar: Map<string, string>): Promise<void> {
+  let currentUrl = startUrl;
+  for (let redirectCount = 0; redirectCount < 8; redirectCount += 1) {
+    const response = await substackFetch(currentUrl, {
+      method: "GET",
+      redirect: "manual",
+      headers: cookieJar.size > 0 ? { cookie: cookieHeaderFromJar(cookieJar) } : undefined,
+    });
+    mergeSetCookies(cookieJar, response);
+
+    if (response.status < 300 || response.status >= 400) break;
+    const location = response.headers.get("location");
+    if (!location) break;
+    const nextUrl = resolveSubstackUrl(location, currentUrl);
+    if (!nextUrl) break;
+    currentUrl = nextUrl;
+  }
+}
+
+export async function completeSubstackMagicLink(link: string, email: string): Promise<SubstackAuthState> {
+  const currentUrl = resolveSubstackUrl(link);
+  if (!currentUrl) throw new Error("Paste the full Substack magic-link URL");
+
+  const cookieJar = new Map<string, string>();
+  await followRedirectsForCookies(currentUrl, cookieJar);
+
+  const sid = cookieJar.get("substack.sid");
+  if (!sid) {
+    throw new Error("Substack did not return a session cookie. Request a new login link and paste the full URL.");
+  }
+  const auth: SubstackAuthState = {
+    email: email.trim(),
+    sid,
+    lli: cookieJar.get("substack.lli") ?? "1",
+    loggedInAt: Date.now(),
+  };
+  storeSubstackAuth(auth);
+  return auth;
+}
+
+export async function completeSubstackOtpLogin(code: string, email: string): Promise<SubstackAuthState> {
+  const normalizedCode = code.trim();
+  const normalizedEmail = email.trim();
+  if (!/^\d{6}$/.test(normalizedCode)) {
+    throw new Error("Enter the 6-digit Substack email code");
+  }
+  if (!/^[^@\s]+@[^@\s]+\.[^@\s]+$/.test(normalizedEmail)) {
+    throw new Error("Enter a valid email address");
+  }
+
+  const cookieJar = new Map<string, string>();
+  const response = await substackFetch(`${SUBSTACK_ORIGIN}/api/v1/email-otp-login/complete`, {
+    method: "POST",
+    redirect: "manual",
+    credentials: "include",
+    headers: {
+      "content-type": "application/json",
+    },
+    body: JSON.stringify({
+      code: normalizedCode,
+      email: normalizedEmail,
+      redirect: "",
+      for_pub: null,
+      island_magic_signin: false,
+    }),
+  });
+  mergeSetCookies(cookieJar, response);
+
+  const isRedirect = response.status >= 300 && response.status < 400;
+  if (!response.ok && !isRedirect) {
+    const detail = await parseErrorText(response);
+    throw new Error(`Substack code login failed (${response.status})${detail ? `: ${detail}` : ""}`);
+  }
+
+  let redirectUrl = resolveSubstackUrl(response.headers.get("location"));
+  if (!redirectUrl && response.ok) {
+    try {
+      const payload = await response.clone().json() as { redirect?: unknown };
+      redirectUrl = resolveSubstackUrl(payload.redirect);
+    } catch {
+      redirectUrl = null;
+    }
+  }
+
+  if (!cookieJar.get("substack.sid") && redirectUrl) {
+    await followRedirectsForCookies(redirectUrl, cookieJar);
+  }
+
+  const sid = cookieJar.get("substack.sid");
+  if (!sid) {
+    const capturedCookieNames = [...cookieJar.keys()];
+    throw new Error(
+      `Substack accepted the code but did not return a session cookie. status=${response.status} redirect=${redirectUrl ? "yes" : "no"} cookies=${capturedCookieNames.length ? capturedCookieNames.join(",") : "none"}`,
+    );
+  }
+  const auth: SubstackAuthState = {
+    email: normalizedEmail,
+    sid,
+    lli: cookieJar.get("substack.lli") ?? "1",
+    loggedInAt: Date.now(),
+  };
+  storeSubstackAuth(auth);
+  return auth;
+}

package/src/plugins/builtin/substack/api/cache.ts

@@ -0,0 +1,78 @@
+import type { PersistedResourceValue } from "../../../../types/persistence";
+import {
+  sortSubscriptionsByLatest,
+} from "../normalize";
+import type {
+  SubstackArticleDetail,
+  SubstackArticleSummary,
+  SubstackPublication,
+} from "../types";
+import type { SubstackCachedData, SubstackHomeData, SubstackPublicationFeedPage } from "./types";
+import { readResource } from "./store";
+
+const PUBLICATION_ARCHIVE_MAX_CACHED_PAGES = 20;
+
+export function publicationCacheKey(publication: SubstackPublication, offset: number): string {
+  return `${publication.baseUrl ?? publication.subdomain ?? publication.id}:offset:${offset}`;
+}
+
+function cachedDataFromResource<T>(entry: PersistedResourceValue<T> | null): SubstackCachedData<T> | null {
+  if (!entry) return null;
+  return {
+    data: entry.value,
+    fetchedAt: entry.fetchedAt,
+    stale: Boolean(entry.stale),
+  };
+}
+
+export function getCachedSubstackHome(): SubstackHomeData | null {
+  const subscriptionsEntry = readResource<SubstackPublication[]>("subscriptions", "me", false);
+  const feedEntry = readResource<SubstackArticleSummary[]>("feed", "subscribed", false);
+  if (!subscriptionsEntry || !feedEntry) return null;
+  return {
+    subscriptions: sortSubscriptionsByLatest(subscriptionsEntry.value, feedEntry.value),
+    feed: feedEntry.value,
+    fetchedAt: Math.max(subscriptionsEntry.fetchedAt, feedEntry.fetchedAt),
+    stale: Boolean(subscriptionsEntry.stale) || Boolean(feedEntry.stale),
+  };
+}
+
+export function getCachedSubstackPublicationFeed(
+  publication: SubstackPublication,
+): SubstackCachedData<SubstackPublicationFeedPage> | null {
+  let offset: number | null = 0;
+  let pageCount = 0;
+  let merged: SubstackPublicationFeedPage | null = null;
+  let fetchedAt = 0;
+  let stale = false;
+
+  while (offset != null && pageCount < PUBLICATION_ARCHIVE_MAX_CACHED_PAGES) {
+    const entry: PersistedResourceValue<SubstackPublicationFeedPage> | null = readResource<SubstackPublicationFeedPage>(
+      "publication",
+      publicationCacheKey(publication, offset),
+      false,
+    );
+    if (!entry) break;
+    const itemsById = new Map<string, SubstackArticleSummary>();
+    for (const article of merged?.items ?? []) itemsById.set(article.id, article);
+    for (const article of entry.value.items) itemsById.set(article.id, article);
+    merged = {
+      items: [...itemsById.values()],
+      nextOffset: entry.value.nextOffset,
+      hasMore: entry.value.hasMore,
+    };
+    fetchedAt = Math.max(fetchedAt, entry.fetchedAt);
+    stale = stale || Boolean(entry.stale);
+    offset = entry.value.nextOffset;
+    pageCount += 1;
+  }
+
+  if (!merged) return null;
+  return { data: merged, fetchedAt, stale };
+}
+
+export function getCachedSubstackArticleDetail(
+  article: Pick<SubstackArticleSummary, "id">,
+): SubstackCachedData<SubstackArticleDetail> | null {
+  return cachedDataFromResource(readResource<SubstackArticleDetail>("post", article.id, false));
+}