gloomberb 0.11.2 → 0.12.0

package/src/plugins/builtin/substack/model.test.ts

@@ -1,155 +0,0 @@
-import { describe, expect, test } from "bun:test";
-import { extractArticleContent } from "./content";
-import {
-  articleMatchesPublication,
-  normalizeFeedItems,
-  normalizePostDetail,
-  normalizeSubscriptions,
-  sortSubscriptionsByLatest,
-} from "./normalize";
-
-describe("Substack article extraction", () => {
-  test("keeps article structure while removing chrome and image-only links", () => {
-    const attrs = JSON.stringify({
-      url: "https://x.com/jukan05/status/2062809398284812532?s=46",
-      full_text: "* Important: SK hynix has no plans to add any new NAND flash capacity.\n\n=> Pure-play NAND manufacturers add capacity.",
-      username: "jukan05",
-      name: "Jukan @COMPUTEX",
-      date: "2026-06-05T08:11:00.000Z",
-      photos: [{ img_url: "https://pbs.substack.com/media/HKCSNYaa8AAq-06.jpg" }],
-      quoted_tweet: {
-        full_text: "[Exclusive] Jensen Huang asked Hynix to make more.",
-        username: "jukan05",
-      },
-    }).replace(/"/g, """);
-    const content = extractArticleContent(`
-      <article>
-        <script>window.bad = true</script>
-        <h1>Ignored detail title</h1>
-        <h2>Victory Giant: -11% as US weighs subsidies</h2>
-        <p>We bought <a href="/p/company">$NVDA</a> after earnings.</p>
-        <div class="twitter-embed" data-attrs="${attrs}" data-component-name="Twitter2ToDOM"></div>
-        <blockquote>Seems very knee jerkish - subsidies seem very small.</blockquote>
-        <figure>
-          <a href="https://substackcdn.com/image/fetch/$s_!abc/f_auto,q_auto:good/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fchart.png">image</a>
-          <img src="/company-card.png" alt="CQME chart" />
-        </figure>
-        <p>See <a href="https://example.com/report">the report</a>.</p>
-      </article>
-    `, {
-      baseUrl: "https://collyerbridge.substack.com",
-      title: "Ignored detail title",
-    });
-
-    expect(content.text).toContain("$NVDA");
-    expect(content.text).toContain("Seems very knee jerkish");
-    expect(content.text).not.toContain("Ignored detail title");
-    expect(content.text).not.toContain("window.bad");
-    expect(content.text).not.toContain("substackcdn.com/image/fetch");
-    expect(content.imageUrls).toEqual(["https://collyerbridge.substack.com/company-card.png"]);
-    expect(content.linkUrls).toEqual([
-      "https://collyerbridge.substack.com/p/company",
-      "https://example.com/report",
-    ]);
-    expect(content.blocks.map((block) => block.type)).toEqual([
-      "heading",
-      "paragraph",
-      "embed",
-      "quote",
-      "image",
-      "paragraph",
-    ]);
-    expect(content.blocks[2]).toEqual({
-      type: "embed",
-      kind: "tweet",
-      text: "* Important: SK hynix has no plans to add any new NAND flash capacity.\n\n=> Pure-play NAND manufacturers add capacity.\n\nQuoted: [Exclusive] Jensen Huang asked Hynix to make more.",
-      url: "https://x.com/jukan05/status/2062809398284812532?s=46",
-      username: "jukan05",
-      authorName: "Jukan @COMPUTEX",
-      dateLabel: "Jun 5, 2026",
-      imageUrls: ["https://pbs.substack.com/media/HKCSNYaa8AAq-06.jpg"],
-    });
-  });
-});
-
-describe("Substack feed normalization", () => {
-  test("normalizes reader rows and orders subscriptions by newest matching post", () => {
-    const subscriptions = normalizeSubscriptions({
-      result: {
-        subscriptions: [
-          { id: "sa", publication_id: "a" },
-          { id: "sb", publication_id: "b" },
-        ],
-        publicationMap: {
-          a: { id: "a", name: "Alpha", base_url: "https://alpha.example" },
-          b: { id: "b", name: "Beta", base_url: "https://beta.example" },
-        },
-      },
-    });
-    const feed = normalizeFeedItems({
-      items: [
-        {
-          post: {
-            id: "older",
-            title: "Older",
-            post_date: "2026-06-01T10:00:00Z",
-            publication: { id: "a", name: "Alpha", base_url: "https://alpha.example" },
-          },
-        },
-        {
-          post: {
-            id: "newer",
-            title: "Semis and $NVDA",
-            subtitle: "A quick note",
-            post_date: "2026-06-05T10:00:00Z",
-            canonical_url: "https://beta.example/p/nvda",
-            slug: "nvda",
-            wordcount: 560,
-            publication: { id: "b", name: "Beta", base_url: "https://beta.example" },
-          },
-        },
-      ],
-    });
-
-    const sorted = sortSubscriptionsByLatest(subscriptions, feed);
-    expect(feed[1]).toMatchObject({
-      id: "newer",
-      title: "Semis and $NVDA",
-      publicationName: "Beta",
-      publicationBaseUrl: "https://beta.example",
-      url: "https://beta.example/p/nvda",
-      readMinutes: 3,
-    });
-    expect(sorted.map((publication) => publication.name)).toEqual(["Beta", "Alpha"]);
-    expect(articleMatchesPublication(feed[1]!, sorted[0]!)).toBe(true);
-  });
-
-  test("merges fetched post body into an existing summary", () => {
-    const summary = normalizeFeedItems({
-      items: [
-        {
-          post: {
-            id: 99,
-            title: "Macro Links",
-            post_date: "2026-06-05T10:00:00Z",
-            publication: { id: "macro", name: "Macro", base_url: "https://macro.example" },
-          },
-        },
-      ],
-    })[0]!;
-
-    const detail = normalizePostDetail({
-      id: 99,
-      title: "Macro Links",
-      body_html: "<p>$SPY broke higher. <a href=\"/p/chart\">Chart</a></p>",
-      publication: { id: "macro", name: "Macro", base_url: "https://macro.example" },
-    }, summary);
-
-    expect(detail.contentText).toContain("$SPY");
-    expect(detail.linkUrls).toEqual(["https://macro.example/p/chart"]);
-    expect(detail.contentBlocks).toEqual([
-      { type: "paragraph", text: "$SPY broke higher. Chart" },
-    ]);
-    expect(detail.id).toBe("99");
-  });
-});

package/src/plugins/builtin/substack/normalize.ts

@@ -1,343 +0,0 @@
-import {
-  estimateReadingMinutes,
-  extractArticleContent,
-  wordCount,
-} from "./content";
-import type {
-  SubstackArticleDetail,
-  SubstackArticleSummary,
-  SubstackPublication,
-} from "./types";
-import {
-  asRecord,
-  firstNumber,
-  firstRawString,
-  firstRecord,
-  firstString,
-  firstValue,
-  maxIso,
-  normalizeUrl,
-  parseDateIso,
-  timestamp,
-  uniqueStrings,
-  type JsonRecord,
-} from "./utils";
-
-function extractPayloadArray(payload: unknown, preferredKeys: string[]): unknown[] {
-  if (Array.isArray(payload)) return payload;
-  const record = asRecord(payload);
-  if (!record) return [];
-  for (const key of preferredKeys) {
-    const value = record[key];
-    if (Array.isArray(value)) return value;
-  }
-  const data = asRecord(record.data);
-  if (data) {
-    for (const key of preferredKeys) {
-      const value = data[key];
-      if (Array.isArray(value)) return value;
-    }
-  }
-  const result = asRecord(record.result);
-  if (result) {
-    for (const key of preferredKeys) {
-      const value = result[key];
-      if (Array.isArray(value)) return value;
-    }
-  }
-  return [];
-}
-
-export function normalizePublication(raw: unknown): SubstackPublication | null {
-  const record = asRecord(raw);
-  if (!record) return null;
-  const subdomain = firstString(record, ["subdomain", "handle", "slug"]);
-  const customDomain = firstString(record, ["custom_domain", "customDomain", "domain"]);
-  const baseUrl = normalizeUrl(firstRawString(record, ["base_url", "baseUrl", "url", "web_url", "canonical_url"]))
-    ?? (customDomain ? `https://${customDomain}` : null)
-    ?? (subdomain ? `https://${subdomain}.substack.com` : null);
-  const idValue = firstValue(record, ["id", "publication_id", "publicationId", "subdomain"]);
-  const name = firstString(record, ["name", "publication_name", "publicationName", "title"]) ?? subdomain ?? "Substack";
-  const id = String(idValue ?? baseUrl ?? name).trim();
-  return {
-    id,
-    name,
-    subdomain,
-    baseUrl,
-    description: firstString(record, ["description", "hero_text", "heroText", "author_bio"]),
-    logoUrl: normalizeUrl(firstRawString(record, ["logo_url", "logoUrl", "logo", "cover_image", "coverImage"]), baseUrl),
-    latestPublishedAt: parseDateIso(firstValue(record, [
-      "latestPublishedAt",
-      "last_published_at",
-      "last_post_date",
-      "lastPostDate",
-      "updated_at",
-    ])),
-  };
-}
-
-export function normalizeSubscriptions(payload: unknown): SubstackPublication[] {
-  const rows = extractPayloadArray(payload, [
-    "subscriptions",
-    "publications",
-    "results",
-    "items",
-  ]);
-  const rootRecord = asRecord(payload);
-  const publicationLookup = buildPublicationLookup(rootRecord);
-  const deduped = new Map<string, SubstackPublication>();
-  for (const row of rows) {
-    const record = asRecord(row);
-    const publication = normalizePublication(
-      record?.publication
-        ?? record?.pub
-        ?? (record ? publicationLookup.get(String(record.publication_id ?? record.publicationId ?? "")) : null)
-        ?? row,
-    );
-    if (!publication) continue;
-    const key = publicationKey(publication);
-    const existing = deduped.get(key);
-    if (!existing) {
-      deduped.set(key, publication);
-      continue;
-    }
-    deduped.set(key, {
-      ...existing,
-      latestPublishedAt: maxIso(existing.latestPublishedAt, publication.latestPublishedAt),
-    });
-  }
-  return [...deduped.values()].sort((a, b) => a.name.localeCompare(b.name));
-}
-
-function buildPublicationLookup(record: JsonRecord | null): Map<string, unknown> {
-  const lookup = new Map<string, unknown>();
-  if (!record) return lookup;
-  const maps = [
-    asRecord(record.publicationMap),
-    asRecord(record.publication_map),
-    asRecord(asRecord(record.result)?.publicationMap),
-    asRecord(asRecord(record.result)?.publication_map),
-  ];
-  for (const map of maps) {
-    if (!map) continue;
-    for (const [key, value] of Object.entries(map)) {
-      lookup.set(key, value);
-    }
-  }
-  const arrays = [
-    record.publications,
-    asRecord(record.result)?.publications,
-  ];
-  for (const array of arrays) {
-    if (!Array.isArray(array)) continue;
-    for (const publication of array) {
-      const normalized = normalizePublication(publication);
-      if (!normalized) continue;
-      lookup.set(normalized.id, publication);
-    }
-  }
-  return lookup;
-}
-
-function articlePostRecord(row: unknown): JsonRecord | null {
-  const record = asRecord(row);
-  return firstRecord(
-    record?.post,
-    record?.item,
-    record?.entity,
-    record?.object,
-    record,
-  );
-}
-
-function articlePublicationRecord(row: JsonRecord | null, post: JsonRecord | null): JsonRecord | null {
-  return firstRecord(
-    post?.publication,
-    post?.pub,
-    row?.publication,
-    row?.pub,
-  );
-}
-
-function buildArticleUrl(post: JsonRecord | null, publication: SubstackPublication | null): string | null {
-  const direct = normalizeUrl(firstRawString(post, ["canonical_url", "canonicalUrl", "url", "web_url"]), publication?.baseUrl);
-  if (direct) return direct;
-  const slug = firstString(post, ["slug"]);
-  if (!slug || !publication?.baseUrl) return null;
-  return normalizeUrl(`/p/${slug}`, publication.baseUrl);
-}
-
-function normalizeArticle(row: unknown, sourcePublication?: SubstackPublication | null): SubstackArticleSummary | null {
-  const rowRecord = asRecord(row);
-  const post = articlePostRecord(row);
-  if (!post) return null;
-  const publication = normalizePublication(articlePublicationRecord(rowRecord, post)) ?? sourcePublication ?? null;
-  const title = firstString(post, ["title", "name", "social_title", "email_subject"]) ?? firstString(rowRecord, ["title"]);
-  const subtitle = firstString(post, ["subtitle", "description", "search_engine_description", "social_description"]);
-  const bodyHtml = firstRawString(post, ["body_html", "bodyHtml", "body"]);
-  const coverImage = normalizeUrl(firstRawString(post, [
-    "cover_image",
-    "coverImage",
-    "cover_image_url",
-    "coverImageUrl",
-    "thumbnail_url",
-    "thumbnailUrl",
-    "image",
-  ]), publication?.baseUrl);
-  const extracted = bodyHtml
-    ? extractArticleContent(bodyHtml, { baseUrl: publication?.baseUrl, imageUrls: coverImage ? [coverImage] : [], title })
-    : { text: "", blocks: [], imageUrls: uniqueStrings([coverImage]), linkUrls: [], wordCount: 0, readMinutes: 1 };
-  const previewText = firstString(post, [
-    "truncated_body_text",
-    "body_preview",
-    "preview",
-    "subtitle",
-    "description",
-    "search_engine_description",
-  ]) ?? (extracted.text ? extracted.text.slice(0, 500) : null);
-  const rawWordCount = firstNumber(post, ["wordcount", "word_count", "words"]);
-  const words = Math.max(0, Math.round(rawWordCount ?? extracted.wordCount ?? wordCount(previewText ?? "")));
-  const rawReadMinutes = firstNumber(post, [
-    "reading_time_minutes",
-    "read_time_minutes",
-    "readMinutes",
-    "readingTime",
-  ]);
-  const publishedAt = parseDateIso(firstValue(post, [
-    "post_date",
-    "published_at",
-    "publish_date",
-    "date",
-    "created_at",
-    "updated_at",
-  ]) ?? firstValue(rowRecord, ["published_at", "timestamp", "date"]));
-  const url = buildArticleUrl(post, publication);
-  const slug = firstString(post, ["slug"]);
-  const rawId = firstValue(post, ["id", "post_id", "postId", "canonical_slug"]) ?? firstValue(rowRecord, ["id", "post_id"]);
-  const id = String(rawId ?? url ?? `${publication?.id ?? "substack"}:${slug ?? title ?? publishedAt ?? Math.random()}`).trim();
-
-  if (!title && !url && !bodyHtml) return null;
-
-  return {
-    id,
-    title: title ?? "Untitled",
-    publicationId: publication?.id ?? null,
-    publicationName: publication?.name ?? null,
-    publicationSubdomain: publication?.subdomain ?? null,
-    publicationBaseUrl: publication?.baseUrl ?? null,
-    url,
-    slug,
-    publishedAt,
-    subtitle,
-    previewText,
-    bodyHtml,
-    imageUrls: extracted.imageUrls,
-    wordCount: words,
-    readMinutes: rawReadMinutes != null ? Math.max(1, Math.ceil(rawReadMinutes)) : estimateReadingMinutes(words || extracted.text),
-  };
-}
-
-export function normalizeFeedItems(payload: unknown, sourcePublication?: SubstackPublication | null): SubstackArticleSummary[] {
-  const rows = extractPayloadArray(payload, ["items", "posts", "results", "feed"]);
-  const items = rows
-    .map((row) => normalizeArticle(row, sourcePublication))
-    .filter((row): row is SubstackArticleSummary => !!row);
-  const deduped = new Map<string, SubstackArticleSummary>();
-  for (const item of items) {
-    const existing = deduped.get(item.id);
-    if (!existing || timestamp(item.publishedAt) > timestamp(existing.publishedAt)) {
-      deduped.set(item.id, item);
-    }
-  }
-  return [...deduped.values()];
-}
-
-export function normalizePostDetail(
-  payload: unknown,
-  fallback: SubstackArticleSummary,
-): SubstackArticleDetail {
-  const post = articlePostRecord(payload) ?? asRecord(payload);
-  const normalized = normalizeArticle(post ?? payload, {
-    id: fallback.publicationId ?? fallback.publicationBaseUrl ?? fallback.publicationName ?? "substack",
-    name: fallback.publicationName ?? "Substack",
-    subdomain: fallback.publicationSubdomain,
-    baseUrl: fallback.publicationBaseUrl,
-    description: null,
-    logoUrl: null,
-    latestPublishedAt: null,
-  }) ?? fallback;
-  const bodyHtml = normalized.bodyHtml ?? fallback.bodyHtml;
-  const extracted = extractArticleContent(bodyHtml, {
-    baseUrl: normalized.publicationBaseUrl ?? fallback.publicationBaseUrl,
-    imageUrls: uniqueStrings([...fallback.imageUrls, ...normalized.imageUrls]),
-    title: normalized.title || fallback.title,
-  });
-  const contentText = extracted.text || normalized.previewText || fallback.previewText || "";
-  const words = Math.max(normalized.wordCount, extracted.wordCount, wordCount(contentText));
-  return {
-    ...fallback,
-    ...normalized,
-    id: fallback.id,
-    title: normalized.title || fallback.title,
-    url: normalized.url ?? fallback.url,
-    bodyHtml,
-    imageUrls: uniqueStrings([...normalized.imageUrls, ...fallback.imageUrls, ...extracted.imageUrls]),
-    contentText,
-    contentBlocks: extracted.blocks,
-    linkUrls: extracted.linkUrls,
-    wordCount: words,
-    readMinutes: estimateReadingMinutes(words || contentText),
-  };
-}
-
-export function publicationKey(publication: Pick<SubstackPublication, "id" | "name" | "baseUrl" | "subdomain">): string {
-  return (
-    publication.baseUrl
-      ?? publication.subdomain
-      ?? publication.id
-      ?? publication.name
-  ).toLowerCase();
-}
-
-function articlePublicationKeys(article: SubstackArticleSummary): string[] {
-  return uniqueStrings([
-    article.publicationBaseUrl?.toLowerCase(),
-    article.publicationSubdomain?.toLowerCase(),
-    article.publicationId?.toLowerCase(),
-    article.publicationName?.toLowerCase(),
-  ]);
-}
-
-export function articleMatchesPublication(
-  article: SubstackArticleSummary,
-  publication: SubstackPublication,
-): boolean {
-  const keys = new Set(articlePublicationKeys(article));
-  return [
-    publication.baseUrl,
-    publication.subdomain,
-    publication.id,
-    publication.name,
-  ].some((value) => !!value && keys.has(value.toLowerCase()));
-}
-
-export function sortSubscriptionsByLatest(
-  subscriptions: SubstackPublication[],
-  feedItems: SubstackArticleSummary[],
-): SubstackPublication[] {
-  return [...subscriptions]
-    .map((publication) => {
-      const latestFeedAt = feedItems
-        .filter((article) => articleMatchesPublication(article, publication))
-        .reduce<string | null>((latest, article) => maxIso(latest, article.publishedAt), null);
-      return {
-        publication,
-        latestAt: maxIso(publication.latestPublishedAt, latestFeedAt),
-      };
-    })
-    .sort((a, b) => {
-      const byDate = timestamp(b.latestAt) - timestamp(a.latestAt);
-      return byDate !== 0 ? byDate : a.publication.name.localeCompare(b.publication.name);
-    })
-    .map((entry) => ({ ...entry.publication, latestPublishedAt: entry.latestAt }));
-}

package/src/plugins/builtin/substack/pane-footer.ts

@@ -1,52 +0,0 @@
-import { usePaneFooter } from "../../../components";
-import type { SubstackAuthState } from "./api/types";
-import {
-  SUBSTACK_PANE_ID,
-  type SubstackArticleSummary,
-} from "./types";
-import { cacheStatusLabel, type ActiveFeedState, type DetailState } from "./pane-state";
-
-export function useSubstackPaneFooter({
-  auth,
-  detailOpen,
-  activeFeedState,
-  activeDetail,
-  selectedArticle,
-  openSelectedArticle,
-}: {
-  auth: SubstackAuthState | null;
-  detailOpen: boolean;
-  activeFeedState: ActiveFeedState;
-  activeDetail: DetailState;
-  selectedArticle: SubstackArticleSummary | null;
-  openSelectedArticle: () => void;
-}) {
-  const statusLabel = cacheStatusLabel(activeFeedState.fetchedAt, activeFeedState.stale);
-
-  usePaneFooter(SUBSTACK_PANE_ID, () => ({
-    info: [
-      ...(!auth ? [{ id: "auth", parts: [{ text: "login required", tone: "warning" as const }] }] : []),
-      ...(activeFeedState.loading || activeFeedState.loadingMore ? [{ id: "loading", parts: [{ text: activeFeedState.loadingMore ? "loading more" : "loading", tone: "muted" as const }] }] : []),
-      ...(activeDetail.loading && detailOpen ? [{ id: "detail-loading", parts: [{ text: "loading article", tone: "muted" as const }] }] : []),
-      ...(statusLabel && auth ? [{ id: "cache", parts: [{ text: statusLabel, tone: activeFeedState.stale ? "warning" as const : "muted" as const }] }] : []),
-      ...(activeFeedState.error ? [{ id: "error", parts: [{ text: activeFeedState.error, tone: "warning" as const }] }] : []),
-      ...(activeDetail.error && detailOpen ? [{ id: "detail-error", parts: [{ text: activeDetail.error, tone: "warning" as const }] }] : []),
-    ],
-    hints: auth
-      ? [{ id: "open", key: "o", label: "pen", onPress: openSelectedArticle, disabled: !selectedArticle?.url }]
-      : [],
-  }), [
-    activeDetail.error,
-    activeDetail.loading,
-    activeFeedState.error,
-    activeFeedState.fetchedAt,
-    activeFeedState.loading,
-    activeFeedState.loadingMore,
-    activeFeedState.stale,
-    auth,
-    detailOpen,
-    openSelectedArticle,
-    selectedArticle?.url,
-    statusLabel,
-  ]);
-}