gloomberb 0.5.1 → 0.6.0

package/src/plugins/builtin/news-wire/rss-parser.ts

@@ -0,0 +1,213 @@
+import type { MarketNewsItem } from "../../../types/news-source";
+import { hashString } from "./hash";
+
+export interface RssFeedConfig {
+  id: string;
+  url: string;
+  name: string;
+  category?: string;
+  authority: number; // 0-100
+  enabled: boolean;
+}
+
+function decodeHtmlEntities(s: string): string {
+  return s
+    .replace(/&/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&apos;/g, "'")
+    .replace(/&nbsp;/g, " ");
+}
+
+function stripCdata(s: string): string {
+  return s.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, (_, inner) => inner);
+}
+
+function stripHtml(s: string): string {
+  return s.replace(/<[^>]*>/g, "");
+}
+
+function extractText(s: string): string {
+  // Decode entities first so escaped HTML tags become real tags, then strip them
+  return stripHtml(decodeHtmlEntities(stripCdata(s))).trim();
+}
+
+function getTagContent(xml: string, tag: string): string {
+  const m = xml.match(new RegExp(`<${tag}[^>]*>([\\s\\S]*?)<\\/${tag}>`, "i"));
+  return m ? m[1]!.trim() : "";
+}
+
+function parseDate(s: string): Date {
+  if (!s) return new Date(0);
+  const d = new Date(s);
+  return isNaN(d.getTime()) ? new Date(0) : d;
+}
+
+function extractAttr(tag: string, attr: string): string {
+  const m = tag.match(new RegExp(`${attr}="([^"]*)"`, "i"));
+  return m ? m[1]! : "";
+}
+
+function extractImageUrl(block: string): string | undefined {
+  // media:content url="..." (common in RSS 2.0 with media namespace)
+  const mediaContent = block.match(/<media:content[^>]+url="([^"]+)"[^>]*(?:medium="image"|type="image\/)/i);
+  if (mediaContent) return mediaContent[1]!;
+
+  // media:content without explicit type (take first one with a url)
+  const mediaAny = block.match(/<media:content[^>]+url="([^"]+)"/i);
+  if (mediaAny) return mediaAny[1]!;
+
+  // media:thumbnail url="..."
+  const mediaThumbnail = block.match(/<media:thumbnail[^>]+url="([^"]+)"/i);
+  if (mediaThumbnail) return mediaThumbnail[1]!;
+
+  // enclosure with image type
+  const enclosure = block.match(/<enclosure[^>]+url="([^"]+)"[^>]+type="image\//i);
+  if (enclosure) return enclosure[1]!;
+
+  // img src inside description/content CDATA
+  const imgSrc = block.match(/<img[^>]+src="(https?:\/\/[^"]+)"/i);
+  if (imgSrc) return imgSrc[1]!;
+
+  return undefined;
+}
+
+function parseRss2Items(xml: string, config: RssFeedConfig): MarketNewsItem[] {
+  const itemRe = /<item>([\s\S]*?)<\/item>/gi;
+  const items: MarketNewsItem[] = [];
+  let match: RegExpExecArray | null;
+
+  while ((match = itemRe.exec(xml)) !== null) {
+    const block = match[1]!;
+
+    const title = extractText(getTagContent(block, "title"));
+    const url = extractText(getTagContent(block, "link"));
+    const pubDateRaw = extractText(getTagContent(block, "pubDate"));
+    const descRaw = getTagContent(block, "description");
+    const desc = descRaw ? extractText(descRaw) : undefined;
+    const categoryRaw = getTagContent(block, "category");
+    const category = categoryRaw ? extractText(categoryRaw) : undefined;
+
+    if (!title && !url) continue;
+
+    const summary = desc
+      ? desc.slice(0, 300) + (desc.length > 300 ? "…" : "")
+      : undefined;
+
+    const id = hashString(`${url}|${title}`);
+    const publishedAt = parseDate(pubDateRaw);
+    const categories = category ? [category] : config.category ? [config.category] : [];
+    const imageUrl = extractImageUrl(block);
+
+    items.push({
+      id,
+      title,
+      url,
+      source: config.name,
+      publishedAt,
+      summary,
+      imageUrl,
+      topic: categories[0] ?? "general",
+      topics: categories,
+      sectors: [],
+      categories,
+      tickers: [],
+      scores: {
+        importance: 0,
+        urgency: 0,
+        marketImpact: 0,
+        novelty: 0,
+        confidence: 0,
+      },
+      importance: 0,
+      isBreaking: false,
+      isDeveloping: false,
+    });
+  }
+
+  return items;
+}
+
+function parseAtomEntries(xml: string, config: RssFeedConfig): MarketNewsItem[] {
+  const entryRe = /<entry>([\s\S]*?)<\/entry>/gi;
+  const items: MarketNewsItem[] = [];
+  let match: RegExpExecArray | null;
+
+  while ((match = entryRe.exec(xml)) !== null) {
+    const block = match[1]!;
+
+    const title = extractText(getTagContent(block, "title"));
+
+    // Atom <link href="..."/> or <link>...</link>
+    const linkTagMatch = block.match(/<link([^>]*)>/i);
+    let url = "";
+    if (linkTagMatch) {
+      const href = extractAttr(linkTagMatch[0]!, "href");
+      if (href) {
+        url = href;
+      } else {
+        url = extractText(getTagContent(block, "link"));
+      }
+    }
+
+    const publishedRaw =
+      extractText(getTagContent(block, "published")) ||
+      extractText(getTagContent(block, "updated"));
+
+    const summaryRaw = getTagContent(block, "summary") || getTagContent(block, "content");
+    const summaryFull = summaryRaw ? extractText(summaryRaw) : undefined;
+    const summary = summaryFull
+      ? summaryFull.slice(0, 300) + (summaryFull.length > 300 ? "…" : "")
+      : undefined;
+
+    if (!title && !url) continue;
+
+    const id = hashString(`${url}|${title}`);
+    const publishedAt = parseDate(publishedRaw);
+    const categories = config.category ? [config.category] : [];
+    const imageUrl = extractImageUrl(block);
+
+    items.push({
+      id,
+      title,
+      url,
+      source: config.name,
+      publishedAt,
+      summary,
+      imageUrl,
+      topic: categories[0] ?? "general",
+      topics: categories,
+      sectors: [],
+      categories,
+      tickers: [],
+      scores: {
+        importance: 0,
+        urgency: 0,
+        marketImpact: 0,
+        novelty: 0,
+        confidence: 0,
+      },
+      importance: 0,
+      isBreaking: false,
+      isDeveloping: false,
+    });
+  }
+
+  return items;
+}
+
+export function parseRssFeed(xml: string, config: RssFeedConfig): MarketNewsItem[] {
+  if (!xml || !xml.trim()) return [];
+
+  try {
+    const isAtom = /<feed\b/i.test(xml);
+    if (isAtom) {
+      return parseAtomEntries(xml, config);
+    }
+    return parseRss2Items(xml, config);
+  } catch {
+    return [];
+  }
+}

package/src/plugins/builtin/news-wire/rss-source.test.ts

@@ -0,0 +1,159 @@
+import { describe, expect, mock, test } from "bun:test";
+import type { PluginPersistence } from "../../../types/plugin";
+import type { PersistedResourceValue } from "../../../types/persistence";
+import { createRssNewsCapability, RSS_FEED_CACHE_POLICY } from "./rss-source";
+import type { RssFeedConfig } from "./rss-parser";
+
+class MemoryPersistence implements PluginPersistence {
+  resources = new Map<string, PersistedResourceValue<any>>();
+
+  getState<T = unknown>(): T | null {
+    return null;
+  }
+
+  setState(): void {}
+
+  deleteState(): void {}
+
+  getResource<T = unknown>(
+    kind: string,
+    key: string,
+    options?: { sourceKey?: string; allowExpired?: boolean },
+  ): PersistedResourceValue<T> | null {
+    const record = this.resources.get(`${kind}:${key}:${options?.sourceKey ?? ""}`) as PersistedResourceValue<T> | undefined;
+    if (!record) return null;
+    const now = Date.now();
+    if (record.expiresAt < now && !options?.allowExpired) return null;
+    return {
+      ...record,
+      stale: record.staleAt <= now,
+      expired: record.expiresAt <= now,
+    };
+  }
+
+  setResource<T = unknown>(
+    kind: string,
+    key: string,
+    value: T,
+    options: { cachePolicy: { staleMs: number; expireMs: number }; sourceKey?: string },
+  ): PersistedResourceValue<T> {
+    const now = Date.now();
+    const record: PersistedResourceValue<T> = {
+      value,
+      fetchedAt: now,
+      staleAt: now + options.cachePolicy.staleMs,
+      expiresAt: now + options.cachePolicy.expireMs,
+      sourceKey: options.sourceKey ?? "",
+      schemaVersion: 0,
+      provenance: null,
+      stale: false,
+      expired: false,
+    };
+    this.resources.set(`${kind}:${key}:${options.sourceKey ?? ""}`, record);
+    return record;
+  }
+
+  deleteResource(kind: string, key: string, options?: { sourceKey?: string }): void {
+    this.resources.delete(`${kind}:${key}:${options?.sourceKey ?? ""}`);
+  }
+}
+
+const FEED: RssFeedConfig = {
+  id: "example-feed",
+  url: "https://example.com/rss.xml",
+  name: "Example",
+  category: "general",
+  authority: 80,
+  enabled: true,
+};
+
+const RSS_FIXTURE = `<rss version="2.0"><channel><item>
+  <title>Breaking: NVIDIA rallies on AI demand</title>
+  <link>https://example.com/nvda</link>
+  <pubDate>${new Date().toUTCString()}</pubDate>
+  <description>NVIDIA shares moved higher.</description>
+</item></channel></rss>`;
+
+describe("createRssNewsCapability", () => {
+  test("caches fetched feed items with feed authority scoring", async () => {
+    const persistence = new MemoryPersistence();
+    const fetchText = mock(async () => ({
+      ok: true,
+      text: async () => RSS_FIXTURE,
+    }));
+    const source = createRssNewsCapability([FEED], { persistence, fetchText });
+
+    const items = await source.provider.fetchNews({ scope: "global" });
+
+    expect(fetchText).toHaveBeenCalledTimes(1);
+    expect(items).toHaveLength(1);
+    expect(items[0]!.importance).toBeGreaterThanOrEqual(FEED.authority);
+    expect(items[0]!.isBreaking).toBe(true);
+    expect(source.provider.getCachedNews?.({ scope: "global" })).toHaveLength(1);
+  });
+
+  test("uses fresh plugin cache without refetching", async () => {
+    const persistence = new MemoryPersistence();
+    const source = createRssNewsCapability([FEED], {
+      persistence,
+      fetchText: async () => {
+        throw new Error("should not fetch");
+      },
+    });
+
+    persistence.setResource("rss-feed", FEED.id, {
+      items: [{
+        id: "cached",
+        title: "Cached headline",
+        url: "https://example.com/cached",
+        source: FEED.name,
+        publishedAt: new Date().toISOString(),
+        categories: ["general"],
+        tickers: [],
+        importance: 60,
+        isBreaking: false,
+      }],
+    }, {
+      sourceKey: FEED.url,
+      cachePolicy: RSS_FEED_CACHE_POLICY,
+    });
+
+    const items = await source.provider.fetchNews({ scope: "global" });
+
+    expect(items).toHaveLength(1);
+    expect(items[0]!.title).toBe("Cached headline");
+  });
+
+  test("falls back to stale plugin cache when refresh fails", async () => {
+    const persistence = new MemoryPersistence();
+    const stalePolicy = { staleMs: -1, expireMs: 60_000 };
+    persistence.setResource("rss-feed", FEED.id, {
+      items: [{
+        id: "stale",
+        title: "Stale headline",
+        url: "https://example.com/stale",
+        source: FEED.name,
+        publishedAt: new Date().toISOString(),
+        categories: ["general"],
+        tickers: [],
+        importance: 50,
+        isBreaking: false,
+      }],
+    }, {
+      sourceKey: FEED.url,
+      cachePolicy: stalePolicy,
+    });
+
+    const source = createRssNewsCapability([FEED], {
+      persistence,
+      fetchText: async () => {
+        throw new Error("network down");
+      },
+    });
+
+    const items = await source.provider.fetchNews({ scope: "global" });
+
+    expect(items).toHaveLength(1);
+    expect(items[0]!.title).toBe("Stale headline");
+  });
+});

package/src/plugins/builtin/news-wire/rss-source.ts

@@ -0,0 +1,181 @@
+import { createThrottledFetch } from "../../../utils/throttled-fetch";
+import { newsProvider, type NewsCapability } from "../../../capabilities";
+import type { NewsQuery, MarketNewsItem } from "../../../types/news-source";
+import type { PluginPersistence } from "../../../types/plugin";
+import { parseRssFeed, type RssFeedConfig } from "./rss-parser";
+import { enrichNewsItem } from "./categories";
+
+const RSS_CACHE_KIND = "rss-feed";
+export const RSS_FEED_CACHE_POLICY = {
+  staleMs: 2 * 60 * 1000,
+  expireMs: 7 * 24 * 60 * 60 * 1000,
+} as const;
+
+interface CachedNewsItem extends Omit<MarketNewsItem, "publishedAt"> {
+  publishedAt: string;
+}
+
+interface CachedFeedPayload {
+  items: CachedNewsItem[];
+}
+
+const rssClient = createThrottledFetch({
+  requestsPerMinute: 30,
+  maxRetries: 1,
+  timeoutMs: 10_000,
+  defaultHeaders: {
+    "User-Agent": "Gloomberb/0.4.1",
+    Accept: "application/rss+xml, application/atom+xml, application/xml, text/xml",
+  },
+});
+
+export interface RssNewsCapabilityOptions {
+  knownTickers?: Set<string>;
+  persistence?: PluginPersistence;
+  fetchText?: (url: string) => Promise<{ ok: boolean; text(): Promise<string> }>;
+}
+
+function supportsQuery(query: NewsQuery): boolean {
+  const feed = query.feed ?? (query.scope === "ticker" ? "ticker" : "latest");
+  return feed === "latest" || feed === "top";
+}
+
+function serializeItem(item: MarketNewsItem): CachedNewsItem {
+  return {
+    ...item,
+    publishedAt: item.publishedAt.toISOString(),
+  };
+}
+
+function deserializeItem(item: unknown): MarketNewsItem | null {
+  if (!item || typeof item !== "object") return null;
+  const record = item as Record<string, unknown>;
+  if (typeof record.id !== "string" || typeof record.title !== "string" || typeof record.url !== "string") return null;
+  if (typeof record.source !== "string") return null;
+  const publishedAt = new Date(String(record.publishedAt ?? ""));
+  if (Number.isNaN(publishedAt.getTime())) return null;
+
+  return {
+    id: record.id,
+    title: record.title,
+    url: record.url,
+    source: record.source,
+    publishedAt,
+    summary: typeof record.summary === "string" ? record.summary : undefined,
+    imageUrl: typeof record.imageUrl === "string" ? record.imageUrl : undefined,
+    topic: typeof record.topic === "string" ? record.topic : "general",
+    topics: Array.isArray(record.topics)
+      ? record.topics.filter((entry): entry is string => typeof entry === "string")
+      : [],
+    sectors: Array.isArray(record.sectors)
+      ? record.sectors.filter((entry): entry is string => typeof entry === "string")
+      : [],
+    categories: Array.isArray(record.categories)
+      ? record.categories.filter((entry): entry is string => typeof entry === "string")
+      : [],
+    tickers: Array.isArray(record.tickers)
+      ? record.tickers.filter((entry): entry is string => typeof entry === "string")
+      : [],
+    sentiment:
+      record.sentiment === "positive" || record.sentiment === "negative" || record.sentiment === "neutral"
+        ? record.sentiment
+        : undefined,
+    scores: {
+      importance: typeof record.importance === "number" ? record.importance : 0,
+      urgency: record.isBreaking === true ? 80 : 0,
+      marketImpact: typeof record.importance === "number" ? record.importance : 0,
+      novelty: 0,
+      confidence: 0,
+    },
+    isBreaking: record.isBreaking === true,
+    isDeveloping: record.isDeveloping === true,
+    importance: typeof record.importance === "number" ? record.importance : 0,
+  };
+}
+
+function readFeedCache(
+  persistence: PluginPersistence | undefined,
+  feed: RssFeedConfig,
+  options?: { allowExpired?: boolean; allowStale?: boolean },
+): MarketNewsItem[] | null {
+  const cached = persistence?.getResource<CachedFeedPayload>(RSS_CACHE_KIND, feed.id, {
+    sourceKey: feed.url,
+    allowExpired: options?.allowExpired,
+  });
+  if (cached?.stale && !options?.allowStale && !options?.allowExpired) return null;
+  if (!cached?.value || !Array.isArray(cached.value.items)) return null;
+  const items = cached.value.items
+    .map(deserializeItem)
+    .filter((item): item is MarketNewsItem => !!item);
+  return items.length > 0 ? items : null;
+}
+
+function writeFeedCache(
+  persistence: PluginPersistence | undefined,
+  feed: RssFeedConfig,
+  items: MarketNewsItem[],
+): void {
+  if (!persistence) return;
+  persistence.setResource<CachedFeedPayload>(RSS_CACHE_KIND, feed.id, {
+    items: items.map(serializeItem),
+  }, {
+    sourceKey: feed.url,
+    cachePolicy: RSS_FEED_CACHE_POLICY,
+    provenance: { url: feed.url, name: feed.name },
+  });
+}
+
+export function createRssNewsCapability(
+  feedsOrGetter: RssFeedConfig[] | (() => RssFeedConfig[]),
+  options: RssNewsCapabilityOptions = {},
+): NewsCapability {
+  const fetchText = options.fetchText ?? ((url: string) => rssClient.fetch(url));
+  const getFeeds = () => Array.isArray(feedsOrGetter) ? feedsOrGetter : feedsOrGetter();
+
+  async function fetchFeed(feed: RssFeedConfig): Promise<MarketNewsItem[]> {
+    const freshCache = readFeedCache(options.persistence, feed);
+    if (freshCache) return freshCache;
+
+    try {
+      const resp = await fetchText(feed.url);
+      if (!resp.ok) return readFeedCache(options.persistence, feed, { allowExpired: true }) ?? [];
+      const xml = await resp.text();
+      const items = parseRssFeed(xml, feed)
+        .map((item) => enrichNewsItem(item, feed.authority, options.knownTickers));
+      writeFeedCache(options.persistence, feed, items);
+      return items;
+    } catch {
+      return readFeedCache(options.persistence, feed, { allowExpired: true }) ?? [];
+    }
+  }
+
+  return newsProvider({
+    id: "rss",
+    name: "RSS Feeds",
+    priority: 2000,
+    provider: {
+      supports: supportsQuery,
+      getCachedNews(query: NewsQuery): MarketNewsItem[] {
+        if (!supportsQuery(query)) return [];
+        const enabledFeeds = getFeeds().filter((feed) => feed.enabled);
+        return enabledFeeds.flatMap((feed) => readFeedCache(options.persistence, feed, { allowExpired: true }) ?? []);
+      },
+      async fetchNews(query: NewsQuery): Promise<MarketNewsItem[]> {
+        if (!supportsQuery(query)) return [];
+        const enabledFeeds = getFeeds().filter((f) => f.enabled);
+        const results = await Promise.allSettled(
+          enabledFeeds.map(fetchFeed),
+        );
+
+        const allItems: MarketNewsItem[] = [];
+        for (const result of results) {
+          if (result.status === "fulfilled") {
+            allItems.push(...result.value);
+          }
+        }
+
+        return allItems;
+      },
+    },
+  });
+}

package/src/plugins/builtin/news-wire/top-pane.tsx

@@ -0,0 +1,21 @@
+import type { PaneProps } from "../../../types/plugin";
+import { NEWS_QUERY_PRESETS } from "./news-query-presets";
+import { NewsPresetPane } from "./news-preset-pane";
+import type { NewsSortPreference } from "./news-table";
+
+const DEFAULT_SORT: NewsSortPreference = { columnId: "importance", direction: "desc" };
+
+export function TopPane(props: PaneProps) {
+  return (
+    <NewsPresetPane
+      {...props}
+      paneKey="top"
+      title="Top News"
+      query={NEWS_QUERY_PRESETS.top}
+      columns={["rank", "time", "source", "title", "tickers", "importance"]}
+      defaultSort={DEFAULT_SORT}
+      emptyStateTitle="No top stories yet"
+      emptyStateHint="Try refreshing later as new headlines are ranked."
+    />
+  );
+}