gloomberb 0.8.1 → 0.9.0

package/src/plugins/builtin/substack/model.test.ts

@@ -0,0 +1,155 @@
+import { describe, expect, test } from "bun:test";
+import { extractArticleContent } from "./content";
+import {
+  articleMatchesPublication,
+  normalizeFeedItems,
+  normalizePostDetail,
+  normalizeSubscriptions,
+  sortSubscriptionsByLatest,
+} from "./normalize";
+
+describe("Substack article extraction", () => {
+  test("keeps article structure while removing chrome and image-only links", () => {
+    const attrs = JSON.stringify({
+      url: "https://x.com/jukan05/status/2062809398284812532?s=46",
+      full_text: "* Important: SK hynix has no plans to add any new NAND flash capacity.\n\n=> Pure-play NAND manufacturers add capacity.",
+      username: "jukan05",
+      name: "Jukan @COMPUTEX",
+      date: "2026-06-05T08:11:00.000Z",
+      photos: [{ img_url: "https://pbs.substack.com/media/HKCSNYaa8AAq-06.jpg" }],
+      quoted_tweet: {
+        full_text: "[Exclusive] Jensen Huang asked Hynix to make more.",
+        username: "jukan05",
+      },
+    }).replace(/"/g, """);
+    const content = extractArticleContent(`
+      <article>
+        <script>window.bad = true</script>
+        <h1>Ignored detail title</h1>
+        <h2>Victory Giant: -11% as US weighs subsidies</h2>
+        <p>We bought <a href="/p/company">$NVDA</a> after earnings.</p>
+        <div class="twitter-embed" data-attrs="${attrs}" data-component-name="Twitter2ToDOM"></div>
+        <blockquote>Seems very knee jerkish - subsidies seem very small.</blockquote>
+        <figure>
+          <a href="https://substackcdn.com/image/fetch/$s_!abc/f_auto,q_auto:good/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fchart.png">image</a>
+          <img src="/company-card.png" alt="CQME chart" />
+        </figure>
+        <p>See <a href="https://example.com/report">the report</a>.</p>
+      </article>
+    `, {
+      baseUrl: "https://collyerbridge.substack.com",
+      title: "Ignored detail title",
+    });
+
+    expect(content.text).toContain("$NVDA");
+    expect(content.text).toContain("Seems very knee jerkish");
+    expect(content.text).not.toContain("Ignored detail title");
+    expect(content.text).not.toContain("window.bad");
+    expect(content.text).not.toContain("substackcdn.com/image/fetch");
+    expect(content.imageUrls).toEqual(["https://collyerbridge.substack.com/company-card.png"]);
+    expect(content.linkUrls).toEqual([
+      "https://collyerbridge.substack.com/p/company",
+      "https://example.com/report",
+    ]);
+    expect(content.blocks.map((block) => block.type)).toEqual([
+      "heading",
+      "paragraph",
+      "embed",
+      "quote",
+      "image",
+      "paragraph",
+    ]);
+    expect(content.blocks[2]).toEqual({
+      type: "embed",
+      kind: "tweet",
+      text: "* Important: SK hynix has no plans to add any new NAND flash capacity.\n\n=> Pure-play NAND manufacturers add capacity.\n\nQuoted: [Exclusive] Jensen Huang asked Hynix to make more.",
+      url: "https://x.com/jukan05/status/2062809398284812532?s=46",
+      username: "jukan05",
+      authorName: "Jukan @COMPUTEX",
+      dateLabel: "Jun 5, 2026",
+      imageUrls: ["https://pbs.substack.com/media/HKCSNYaa8AAq-06.jpg"],
+    });
+  });
+});
+
+describe("Substack feed normalization", () => {
+  test("normalizes reader rows and orders subscriptions by newest matching post", () => {
+    const subscriptions = normalizeSubscriptions({
+      result: {
+        subscriptions: [
+          { id: "sa", publication_id: "a" },
+          { id: "sb", publication_id: "b" },
+        ],
+        publicationMap: {
+          a: { id: "a", name: "Alpha", base_url: "https://alpha.example" },
+          b: { id: "b", name: "Beta", base_url: "https://beta.example" },
+        },
+      },
+    });
+    const feed = normalizeFeedItems({
+      items: [
+        {
+          post: {
+            id: "older",
+            title: "Older",
+            post_date: "2026-06-01T10:00:00Z",
+            publication: { id: "a", name: "Alpha", base_url: "https://alpha.example" },
+          },
+        },
+        {
+          post: {
+            id: "newer",
+            title: "Semis and $NVDA",
+            subtitle: "A quick note",
+            post_date: "2026-06-05T10:00:00Z",
+            canonical_url: "https://beta.example/p/nvda",
+            slug: "nvda",
+            wordcount: 560,
+            publication: { id: "b", name: "Beta", base_url: "https://beta.example" },
+          },
+        },
+      ],
+    });
+
+    const sorted = sortSubscriptionsByLatest(subscriptions, feed);
+    expect(feed[1]).toMatchObject({
+      id: "newer",
+      title: "Semis and $NVDA",
+      publicationName: "Beta",
+      publicationBaseUrl: "https://beta.example",
+      url: "https://beta.example/p/nvda",
+      readMinutes: 3,
+    });
+    expect(sorted.map((publication) => publication.name)).toEqual(["Beta", "Alpha"]);
+    expect(articleMatchesPublication(feed[1]!, sorted[0]!)).toBe(true);
+  });
+
+  test("merges fetched post body into an existing summary", () => {
+    const summary = normalizeFeedItems({
+      items: [
+        {
+          post: {
+            id: 99,
+            title: "Macro Links",
+            post_date: "2026-06-05T10:00:00Z",
+            publication: { id: "macro", name: "Macro", base_url: "https://macro.example" },
+          },
+        },
+      ],
+    })[0]!;
+
+    const detail = normalizePostDetail({
+      id: 99,
+      title: "Macro Links",
+      body_html: "<p>$SPY broke higher. <a href=\"/p/chart\">Chart</a></p>",
+      publication: { id: "macro", name: "Macro", base_url: "https://macro.example" },
+    }, summary);
+
+    expect(detail.contentText).toContain("$SPY");
+    expect(detail.linkUrls).toEqual(["https://macro.example/p/chart"]);
+    expect(detail.contentBlocks).toEqual([
+      { type: "paragraph", text: "$SPY broke higher. Chart" },
+    ]);
+    expect(detail.id).toBe("99");
+  });
+});

package/src/plugins/builtin/substack/normalize.ts

@@ -0,0 +1,343 @@
+import {
+  estimateReadingMinutes,
+  extractArticleContent,
+  wordCount,
+} from "./content";
+import type {
+  SubstackArticleDetail,
+  SubstackArticleSummary,
+  SubstackPublication,
+} from "./types";
+import {
+  asRecord,
+  firstNumber,
+  firstRawString,
+  firstRecord,
+  firstString,
+  firstValue,
+  maxIso,
+  normalizeUrl,
+  parseDateIso,
+  timestamp,
+  uniqueStrings,
+  type JsonRecord,
+} from "./utils";
+
+function extractPayloadArray(payload: unknown, preferredKeys: string[]): unknown[] {
+  if (Array.isArray(payload)) return payload;
+  const record = asRecord(payload);
+  if (!record) return [];
+  for (const key of preferredKeys) {
+    const value = record[key];
+    if (Array.isArray(value)) return value;
+  }
+  const data = asRecord(record.data);
+  if (data) {
+    for (const key of preferredKeys) {
+      const value = data[key];
+      if (Array.isArray(value)) return value;
+    }
+  }
+  const result = asRecord(record.result);
+  if (result) {
+    for (const key of preferredKeys) {
+      const value = result[key];
+      if (Array.isArray(value)) return value;
+    }
+  }
+  return [];
+}
+
+export function normalizePublication(raw: unknown): SubstackPublication | null {
+  const record = asRecord(raw);
+  if (!record) return null;
+  const subdomain = firstString(record, ["subdomain", "handle", "slug"]);
+  const customDomain = firstString(record, ["custom_domain", "customDomain", "domain"]);
+  const baseUrl = normalizeUrl(firstRawString(record, ["base_url", "baseUrl", "url", "web_url", "canonical_url"]))
+    ?? (customDomain ? `https://${customDomain}` : null)
+    ?? (subdomain ? `https://${subdomain}.substack.com` : null);
+  const idValue = firstValue(record, ["id", "publication_id", "publicationId", "subdomain"]);
+  const name = firstString(record, ["name", "publication_name", "publicationName", "title"]) ?? subdomain ?? "Substack";
+  const id = String(idValue ?? baseUrl ?? name).trim();
+  return {
+    id,
+    name,
+    subdomain,
+    baseUrl,
+    description: firstString(record, ["description", "hero_text", "heroText", "author_bio"]),
+    logoUrl: normalizeUrl(firstRawString(record, ["logo_url", "logoUrl", "logo", "cover_image", "coverImage"]), baseUrl),
+    latestPublishedAt: parseDateIso(firstValue(record, [
+      "latestPublishedAt",
+      "last_published_at",
+      "last_post_date",
+      "lastPostDate",
+      "updated_at",
+    ])),
+  };
+}
+
+export function normalizeSubscriptions(payload: unknown): SubstackPublication[] {
+  const rows = extractPayloadArray(payload, [
+    "subscriptions",
+    "publications",
+    "results",
+    "items",
+  ]);
+  const rootRecord = asRecord(payload);
+  const publicationLookup = buildPublicationLookup(rootRecord);
+  const deduped = new Map<string, SubstackPublication>();
+  for (const row of rows) {
+    const record = asRecord(row);
+    const publication = normalizePublication(
+      record?.publication
+        ?? record?.pub
+        ?? (record ? publicationLookup.get(String(record.publication_id ?? record.publicationId ?? "")) : null)
+        ?? row,
+    );
+    if (!publication) continue;
+    const key = publicationKey(publication);
+    const existing = deduped.get(key);
+    if (!existing) {
+      deduped.set(key, publication);
+      continue;
+    }
+    deduped.set(key, {
+      ...existing,
+      latestPublishedAt: maxIso(existing.latestPublishedAt, publication.latestPublishedAt),
+    });
+  }
+  return [...deduped.values()].sort((a, b) => a.name.localeCompare(b.name));
+}
+
+function buildPublicationLookup(record: JsonRecord | null): Map<string, unknown> {
+  const lookup = new Map<string, unknown>();
+  if (!record) return lookup;
+  const maps = [
+    asRecord(record.publicationMap),
+    asRecord(record.publication_map),
+    asRecord(asRecord(record.result)?.publicationMap),
+    asRecord(asRecord(record.result)?.publication_map),
+  ];
+  for (const map of maps) {
+    if (!map) continue;
+    for (const [key, value] of Object.entries(map)) {
+      lookup.set(key, value);
+    }
+  }
+  const arrays = [
+    record.publications,
+    asRecord(record.result)?.publications,
+  ];
+  for (const array of arrays) {
+    if (!Array.isArray(array)) continue;
+    for (const publication of array) {
+      const normalized = normalizePublication(publication);
+      if (!normalized) continue;
+      lookup.set(normalized.id, publication);
+    }
+  }
+  return lookup;
+}
+
+function articlePostRecord(row: unknown): JsonRecord | null {
+  const record = asRecord(row);
+  return firstRecord(
+    record?.post,
+    record?.item,
+    record?.entity,
+    record?.object,
+    record,
+  );
+}
+
+function articlePublicationRecord(row: JsonRecord | null, post: JsonRecord | null): JsonRecord | null {
+  return firstRecord(
+    post?.publication,
+    post?.pub,
+    row?.publication,
+    row?.pub,
+  );
+}
+
+function buildArticleUrl(post: JsonRecord | null, publication: SubstackPublication | null): string | null {
+  const direct = normalizeUrl(firstRawString(post, ["canonical_url", "canonicalUrl", "url", "web_url"]), publication?.baseUrl);
+  if (direct) return direct;
+  const slug = firstString(post, ["slug"]);
+  if (!slug || !publication?.baseUrl) return null;
+  return normalizeUrl(`/p/${slug}`, publication.baseUrl);
+}
+
+function normalizeArticle(row: unknown, sourcePublication?: SubstackPublication | null): SubstackArticleSummary | null {
+  const rowRecord = asRecord(row);
+  const post = articlePostRecord(row);
+  if (!post) return null;
+  const publication = normalizePublication(articlePublicationRecord(rowRecord, post)) ?? sourcePublication ?? null;
+  const title = firstString(post, ["title", "name", "social_title", "email_subject"]) ?? firstString(rowRecord, ["title"]);
+  const subtitle = firstString(post, ["subtitle", "description", "search_engine_description", "social_description"]);
+  const bodyHtml = firstRawString(post, ["body_html", "bodyHtml", "body"]);
+  const coverImage = normalizeUrl(firstRawString(post, [
+    "cover_image",
+    "coverImage",
+    "cover_image_url",
+    "coverImageUrl",
+    "thumbnail_url",
+    "thumbnailUrl",
+    "image",
+  ]), publication?.baseUrl);
+  const extracted = bodyHtml
+    ? extractArticleContent(bodyHtml, { baseUrl: publication?.baseUrl, imageUrls: coverImage ? [coverImage] : [], title })
+    : { text: "", blocks: [], imageUrls: uniqueStrings([coverImage]), linkUrls: [], wordCount: 0, readMinutes: 1 };
+  const previewText = firstString(post, [
+    "truncated_body_text",
+    "body_preview",
+    "preview",
+    "subtitle",
+    "description",
+    "search_engine_description",
+  ]) ?? (extracted.text ? extracted.text.slice(0, 500) : null);
+  const rawWordCount = firstNumber(post, ["wordcount", "word_count", "words"]);
+  const words = Math.max(0, Math.round(rawWordCount ?? extracted.wordCount ?? wordCount(previewText ?? "")));
+  const rawReadMinutes = firstNumber(post, [
+    "reading_time_minutes",
+    "read_time_minutes",
+    "readMinutes",
+    "readingTime",
+  ]);
+  const publishedAt = parseDateIso(firstValue(post, [
+    "post_date",
+    "published_at",
+    "publish_date",
+    "date",
+    "created_at",
+    "updated_at",
+  ]) ?? firstValue(rowRecord, ["published_at", "timestamp", "date"]));
+  const url = buildArticleUrl(post, publication);
+  const slug = firstString(post, ["slug"]);
+  const rawId = firstValue(post, ["id", "post_id", "postId", "canonical_slug"]) ?? firstValue(rowRecord, ["id", "post_id"]);
+  const id = String(rawId ?? url ?? `${publication?.id ?? "substack"}:${slug ?? title ?? publishedAt ?? Math.random()}`).trim();
+
+  if (!title && !url && !bodyHtml) return null;
+
+  return {
+    id,
+    title: title ?? "Untitled",
+    publicationId: publication?.id ?? null,
+    publicationName: publication?.name ?? null,
+    publicationSubdomain: publication?.subdomain ?? null,
+    publicationBaseUrl: publication?.baseUrl ?? null,
+    url,
+    slug,
+    publishedAt,
+    subtitle,
+    previewText,
+    bodyHtml,
+    imageUrls: extracted.imageUrls,
+    wordCount: words,
+    readMinutes: rawReadMinutes != null ? Math.max(1, Math.ceil(rawReadMinutes)) : estimateReadingMinutes(words || extracted.text),
+  };
+}
+
+export function normalizeFeedItems(payload: unknown, sourcePublication?: SubstackPublication | null): SubstackArticleSummary[] {
+  const rows = extractPayloadArray(payload, ["items", "posts", "results", "feed"]);
+  const items = rows
+    .map((row) => normalizeArticle(row, sourcePublication))
+    .filter((row): row is SubstackArticleSummary => !!row);
+  const deduped = new Map<string, SubstackArticleSummary>();
+  for (const item of items) {
+    const existing = deduped.get(item.id);
+    if (!existing || timestamp(item.publishedAt) > timestamp(existing.publishedAt)) {
+      deduped.set(item.id, item);
+    }
+  }
+  return [...deduped.values()];
+}
+
+export function normalizePostDetail(
+  payload: unknown,
+  fallback: SubstackArticleSummary,
+): SubstackArticleDetail {
+  const post = articlePostRecord(payload) ?? asRecord(payload);
+  const normalized = normalizeArticle(post ?? payload, {
+    id: fallback.publicationId ?? fallback.publicationBaseUrl ?? fallback.publicationName ?? "substack",
+    name: fallback.publicationName ?? "Substack",
+    subdomain: fallback.publicationSubdomain,
+    baseUrl: fallback.publicationBaseUrl,
+    description: null,
+    logoUrl: null,
+    latestPublishedAt: null,
+  }) ?? fallback;
+  const bodyHtml = normalized.bodyHtml ?? fallback.bodyHtml;
+  const extracted = extractArticleContent(bodyHtml, {
+    baseUrl: normalized.publicationBaseUrl ?? fallback.publicationBaseUrl,
+    imageUrls: uniqueStrings([...fallback.imageUrls, ...normalized.imageUrls]),
+    title: normalized.title || fallback.title,
+  });
+  const contentText = extracted.text || normalized.previewText || fallback.previewText || "";
+  const words = Math.max(normalized.wordCount, extracted.wordCount, wordCount(contentText));
+  return {
+    ...fallback,
+    ...normalized,
+    id: fallback.id,
+    title: normalized.title || fallback.title,
+    url: normalized.url ?? fallback.url,
+    bodyHtml,
+    imageUrls: uniqueStrings([...normalized.imageUrls, ...fallback.imageUrls, ...extracted.imageUrls]),
+    contentText,
+    contentBlocks: extracted.blocks,
+    linkUrls: extracted.linkUrls,
+    wordCount: words,
+    readMinutes: estimateReadingMinutes(words || contentText),
+  };
+}
+
+export function publicationKey(publication: Pick<SubstackPublication, "id" | "name" | "baseUrl" | "subdomain">): string {
+  return (
+    publication.baseUrl
+      ?? publication.subdomain
+      ?? publication.id
+      ?? publication.name
+  ).toLowerCase();
+}
+
+function articlePublicationKeys(article: SubstackArticleSummary): string[] {
+  return uniqueStrings([
+    article.publicationBaseUrl?.toLowerCase(),
+    article.publicationSubdomain?.toLowerCase(),
+    article.publicationId?.toLowerCase(),
+    article.publicationName?.toLowerCase(),
+  ]);
+}
+
+export function articleMatchesPublication(
+  article: SubstackArticleSummary,
+  publication: SubstackPublication,
+): boolean {
+  const keys = new Set(articlePublicationKeys(article));
+  return [
+    publication.baseUrl,
+    publication.subdomain,
+    publication.id,
+    publication.name,
+  ].some((value) => !!value && keys.has(value.toLowerCase()));
+}
+
+export function sortSubscriptionsByLatest(
+  subscriptions: SubstackPublication[],
+  feedItems: SubstackArticleSummary[],
+): SubstackPublication[] {
+  return [...subscriptions]
+    .map((publication) => {
+      const latestFeedAt = feedItems
+        .filter((article) => articleMatchesPublication(article, publication))
+        .reduce<string | null>((latest, article) => maxIso(latest, article.publishedAt), null);
+      return {
+        publication,
+        latestAt: maxIso(publication.latestPublishedAt, latestFeedAt),
+      };
+    })
+    .sort((a, b) => {
+      const byDate = timestamp(b.latestAt) - timestamp(a.latestAt);
+      return byDate !== 0 ? byDate : a.publication.name.localeCompare(b.publication.name);
+    })
+    .map((entry) => ({ ...entry.publication, latestPublishedAt: entry.latestAt }));
+}

package/src/plugins/builtin/substack/pane-footer.ts

@@ -0,0 +1,71 @@
+import { usePaneFooter } from "../../../components";
+import type { SubstackAuthState } from "./api/types";
+import {
+  formatReadTime,
+  formatWordCount,
+} from "./table";
+import {
+  SUBSTACK_PANE_ID,
+  type SubstackArticleSummary,
+} from "./types";
+import { cacheStatusLabel, type ActiveFeedState, type DetailState } from "./pane-state";
+
+export function useSubstackPaneFooter({
+  auth,
+  detailOpen,
+  activeFeedState,
+  activeDetail,
+  selectedArticle,
+  refreshActive,
+  openSelectedArticle,
+}: {
+  auth: SubstackAuthState | null;
+  detailOpen: boolean;
+  activeFeedState: ActiveFeedState;
+  activeDetail: DetailState;
+  selectedArticle: SubstackArticleSummary | null;
+  refreshActive: () => void;
+  openSelectedArticle: () => void;
+}) {
+  const statusLabel = cacheStatusLabel(activeFeedState.fetchedAt, activeFeedState.stale);
+  const articleMetaLabel = detailOpen && selectedArticle
+    ? [
+      selectedArticle.publicationName,
+      formatReadTime(activeDetail.data?.readMinutes ?? selectedArticle.readMinutes),
+      formatWordCount(activeDetail.data?.wordCount ?? selectedArticle.wordCount),
+    ].filter(Boolean).join("  |  ")
+    : null;
+
+  usePaneFooter(SUBSTACK_PANE_ID, () => ({
+    info: [
+      ...(!auth ? [{ id: "auth", parts: [{ text: "login required", tone: "warning" as const }] }] : []),
+      ...(articleMetaLabel ? [{ id: "article-meta", parts: [{ text: articleMetaLabel, tone: "muted" as const }] }] : []),
+      ...(activeFeedState.loading || activeFeedState.loadingMore ? [{ id: "loading", parts: [{ text: activeFeedState.loadingMore ? "loading more" : "loading", tone: "muted" as const }] }] : []),
+      ...(activeDetail.loading && detailOpen ? [{ id: "detail-loading", parts: [{ text: "loading article", tone: "muted" as const }] }] : []),
+      ...(statusLabel && auth ? [{ id: "cache", parts: [{ text: statusLabel, tone: activeFeedState.stale ? "warning" as const : "muted" as const }] }] : []),
+      ...(activeFeedState.error ? [{ id: "error", parts: [{ text: activeFeedState.error, tone: "warning" as const }] }] : []),
+      ...(activeDetail.error && detailOpen ? [{ id: "detail-error", parts: [{ text: activeDetail.error, tone: "warning" as const }] }] : []),
+    ],
+    hints: auth ? [
+      { id: "refresh", key: "r", label: "efresh", onPress: refreshActive },
+      { id: "open", key: "o", label: "pen", onPress: openSelectedArticle, disabled: !selectedArticle?.url },
+    ] : [],
+  }), [
+    activeDetail.error,
+    activeDetail.loading,
+    activeDetail.data?.readMinutes,
+    activeDetail.data?.wordCount,
+    activeFeedState.error,
+    activeFeedState.fetchedAt,
+    activeFeedState.loading,
+    activeFeedState.loadingMore,
+    activeFeedState.stale,
+    articleMetaLabel,
+    auth,
+    detailOpen,
+    openSelectedArticle,
+    refreshActive,
+    selectedArticle?.url,
+    statusLabel,
+  ]);
+}