@zhin.js/plugin-rss 1.1.19 → 1.1.20

package/src/admin-tasks.ts

@@ -70,15 +70,22 @@ export class AdministratorRssTasks implements PluginTaskProvider {
   readonly #outbound: OutboundHost;
   readonly #config: Readonly<RssConfig>;
   readonly #load: FeedLoader;
+  readonly #enabledFor: (target: DeliveryTarget) => boolean;
   readonly projectLocale: Locale;
   readonly #admission = createGenerationAdmissionGate();
   #stopped = false;
   #working = false;
   get [generationAdmissionSource]() { return [this.#admission]; }
 
-  constructor(state: PluginStateStore, outbound: OutboundHost, config: Readonly<RssConfig>, load: FeedLoader = fetchFeed, projectLocale: Locale = 'zh-CN') {
+  /**
+   * `enabledFor` is the ADR 0050 / 0055 feature switch of the target conversation (pushes are not
+   * messages, so dispatch gating never sees them); a switched-off target is skipped, not sent.
+   */
+  constructor(state: PluginStateStore, outbound: OutboundHost, config: Readonly<RssConfig>, load: FeedLoader = fetchFeed, projectLocale: Locale = 'zh-CN',
+    enabledFor: (target: DeliveryTarget) => boolean = () => true) {
     if (!outbound.sendTarget) throw unavailable('exactTarget');
     this.projectLocale = projectLocale; this.#state = state; this.#outbound = outbound; this.#config = config; this.#load = load;
+    this.#enabledFor = enabledFor;
   }
   stop(): void { this.#stopped = true; }
 
@@ -169,7 +176,7 @@ export class AdministratorRssTasks implements PluginTaskProvider {
           if (this.#stopped) break;
           if (task.state !== 'scheduled' || flushed.has(task.id)) continue;
           let feed: Awaited<ReturnType<FeedLoader>>;
-          try { feed = await this.#load(task.url, this.#config.timeout); }
+          try { feed = await this.#load(task.url, this.#config); }
           catch { await this.#recordCheck(task, 'fetchFailed'); continue; }
           if (this.#stopped) break;
           // Keep only bounded identifiers, never arbitrary feed bodies, in durable state.
@@ -211,7 +218,10 @@ export class AdministratorRssTasks implements PluginTaskProvider {
   async #deliver(claimed: Task, content: string): Promise<void> {
     if (this.#stopped) return;
     let state: Task['state'] = 'unknown', detail = taskDetail('unknown');
-    try {
+    let enabled: boolean;
+    try { enabled = this.#enabledFor(claimed.target); } catch { enabled = true; }
+    if (!enabled) { state = 'scheduled'; detail = taskDetail('featureDisabled'); }
+    else try {
       const id = await this.#outbound.sendTarget!(claimed.target, content, { locale: resolveLocale(claimed.locale) });
       if (id) { state = 'scheduled'; detail = taskDetail('delivered'); }
     } catch (error) {

package/src/console.ts

@@ -39,6 +39,8 @@ function resolveRssRuntime(call: PluginConsoleRpcCall, fallback?: RssRuntime): R
 
 export function createRssConsoleRpc(defaultRuntime?: RssRuntime): PluginConsoleRpc {
   const rpc = definePluginConsoleRpc({
+    // Day-to-day edits belong to operators; `:write` would otherwise infer `manage` (ADR 0049).
+    permissions: [{ id: 'rss:write', level: 'operate', description: '编辑订阅' }],
     status: {
       page: 'index',
       requiredPermissions: ['rss:read'],

package/src/feed.ts

@@ -4,6 +4,10 @@ import { rssMessages as t } from './i18n.js';
  * RSS feed fetch/preview helpers (no Plugin host).
  */
 import Parser from 'rss-parser';
+import { fetchPublic, readCappedBody, type HostLookup } from './public-fetch.js';
+
+/** Feeds larger than this are refused instead of buffered. */
+const MAX_FEED_BYTES = 5 * 1024 * 1024;
 
 export interface FeedItem {
   title: string;
@@ -20,6 +24,8 @@ export interface RssConfig {
   readonly timeout: number;
   readonly maxItemsPerPoll?: number;
   readonly fetchTimeoutMs?: number;
+  /** Let feeds live on loopback/LAN addresses (self-hosted RSSHub and the like). Default false. */
+  readonly allowPrivateNetwork?: boolean;
 }
 
 export const DEFAULT_RSS_CONFIG: RssConfig = {
@@ -79,18 +85,35 @@ export function stripHtml(html: string): string {
   return out;
 }
 
+/**
+ * Fetch and parse a feed. The URL comes from chat users, so it is requested through the
+ * public-address policy (every redirect hop re-checked), with one timeout for the whole
+ * exchange and a body cap.
+ */
 export async function fetchFeed(
   url: string,
-  timeout = DEFAULT_RSS_CONFIG.timeout,
+  options: number | Pick<RssConfig, 'timeout' | 'allowPrivateNetwork'> = DEFAULT_RSS_CONFIG,
+  lookup?: HostLookup,
 ): Promise<{ title: string; items: FeedItem[] }> {
-  const parser = new Parser({
-    timeout,
-    headers: {
-      'User-Agent': 'ZhinBot-RSS/1.0',
-      Accept: 'application/rss+xml, application/atom+xml, application/xml, text/xml',
+  const { timeout, allowPrivateNetwork = false } = typeof options === 'number' ? { timeout: options } : options;
+  const { response } = await fetchPublic(url, {
+    timeoutMs: timeout,
+    lookup,
+    allowPrivateNetwork,
+    init: {
+      headers: {
+        'User-Agent': 'ZhinBot-RSS/1.0',
+        Accept: 'application/rss+xml, application/atom+xml, application/xml, text/xml',
+      },
     },
   });
-  const feed = await parser.parseURL(url);
+  if (!response.ok) {
+    await response.body?.cancel();
+    throw new Error(`Status code ${response.status}`);
+  }
+  const body = await readCappedBody(response, MAX_FEED_BYTES);
+  if (!body) throw new Error(`Feed is larger than ${MAX_FEED_BYTES} bytes`);
+  const feed = await new Parser().parseString(body.toString('utf8'));
   const items: FeedItem[] = (feed.items || []).slice(0, 20).map((item) => ({
     title: (item.title || '').trim(),
     link: (item.link || '').trim(),

package/src/poll.ts

@@ -8,10 +8,22 @@ import { fetchFeed, type FeedItem } from './feed.js';
 import { getRssSeen, getRssSubs } from './db-store.js';
 import type { RssRuntime } from './runtime.js';
 
+/** Seen rows are kept this long after the item was last present in its feed. */
+export const SEEN_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
+/** Seen rows of items still in the feed are re-stamped at most this often. */
+const SEEN_REFRESH_MS = 24 * 60 * 60 * 1000;
+
 function ts(): string {
   return new Date().toISOString();
 }
 
+/** True for an item whose own publish date is past the retention window. Undated items are never stale. */
+function isStale(item: FeedItem, cutoffMs: number): boolean {
+  if (!item.date) return false;
+  const published = Date.parse(item.date);
+  return Number.isFinite(published) && published < cutoffMs;
+}
+
 export function formatNewItems(feedTitle: string, items: FeedItem[], maxItems: number, locale: Locale = 'zh-CN'): string {
   const lines = [t(locale, 'newItems', { title: feedTitle, count: items.length })];
   for (const item of items.slice(0, maxItems)) {
@@ -26,19 +38,28 @@ export function formatNewItems(feedTitle: string, items: FeedItem[], maxItems: n
   return lines.join('\n');
 }
 
-export async function markItemsSeen(runtime: RssRuntime, feedUrl: string, items: FeedItem[]): Promise<void> {
+export async function markItemsSeen(
+  runtime: RssRuntime,
+  feedUrl: string,
+  items: FeedItem[],
+  /** Guids already stored for this feed; loaded once when the caller does not have them. */
+  knownGuids?: ReadonlySet<string>,
+): Promise<void> {
   const Seen = getRssSeen(runtime.db);
-  if (!Seen) return;
+  if (!Seen || items.length === 0) return;
+  const known = new Set(knownGuids
+    ?? (await Seen.select().where({ feed_url: feedUrl })).map((row) => String(row.item_guid ?? '')));
+  // One timestamp per batch so cleanup can drop the whole batch with a single delete.
+  const seenAt = ts();
   for (const item of items) {
-    const dup = await Seen.select().where({ feed_url: feedUrl, item_guid: item.guid });
-    if (dup.length === 0) {
-      await Seen.insert({
-        feed_url: feedUrl,
-        item_guid: item.guid,
-        item_title: item.title,
-        seen_at: ts(),
-      });
-    }
+    if (known.has(item.guid)) continue;
+    known.add(item.guid);
+    await Seen.insert({
+      feed_url: feedUrl,
+      item_guid: item.guid,
+      item_title: item.title,
+      seen_at: seenAt,
+    });
   }
 }
 
@@ -110,15 +131,28 @@ export async function checkSubscriptions(runtime: RssRuntime, options: {
 
   for (const url of options.urls) {
     try {
-      const { title, items } = await fetchFeed(url, runtime.config.timeout);
+      const { title, items } = await fetchFeed(url, runtime.config);
       const seenRows = await Seen.select().where({ feed_url: url });
-      const seenGuids = new Set(seenRows.map((r) => String(r.item_guid ?? '')));
-      const newItems = items.filter((item) => item.guid && !seenGuids.has(item.guid));
+      const seenAt = new Map(seenRows.map((r) => [String(r.item_guid ?? ''), String(r.seen_at ?? '')]));
+      const seenGuids = new Set(seenAt.keys());
+      const now = Date.now();
+      // Keep the seen row of an item that is still listed, so it is not re-announced after cleanup.
+      const refreshBefore = new Date(now - SEEN_REFRESH_MS).toISOString();
+      const refreshed = ts();
+      for (const item of items) {
+        const stamp = seenAt.get(item.guid);
+        if (item.guid && stamp !== undefined && stamp < refreshBefore) {
+          await Seen.update({ seen_at: refreshed }).where({ feed_url: url, item_guid: item.guid });
+        }
+      }
+      // An item older than the retention window can no longer have a seen row; never treat it as new.
+      const newItems = items.filter((item) => item.guid && !seenGuids.has(item.guid)
+        && !isStale(item, now - SEEN_RETENTION_MS));
       if (newItems.length === 0) {
         parts.push(t(locale, 'noNew', { title: title || url }));
         continue;
       }
-      await markItemsSeen(runtime, url, newItems);
+      await markItemsSeen(runtime, url, newItems, seenGuids);
       totalNew += newItems.length;
       const body = formatNewItems(title || url, newItems, runtime.config.maxItems, locale);
       parts.push(body);
@@ -139,21 +173,22 @@ export async function checkSubscriptions(runtime: RssRuntime, options: {
 }
 
 /**
- * Periodically clean seen records older than 7 days.
+ * Periodically clean seen records older than 7 days. Rows of items still listed in their
+ * feed are re-stamped by checkSubscriptions and therefore survive.
  */
 export async function cleanOldSeen(runtime: RssRuntime): Promise<void> {
   const Seen = getRssSeen(runtime.db);
   if (!Seen) return;
   try {
-    const cutoff = new Date();
-    cutoff.setDate(cutoff.getDate() - 7);
-    const cutoffStr = cutoff.toISOString();
+    const cutoffStr = new Date(Date.now() - SEEN_RETENTION_MS).toISOString();
     const all = await Seen.select();
-    const expired = all.filter((r) => typeof r.seen_at === 'string' && r.seen_at < cutoffStr);
-    for (const row of expired) {
-      // 表结构没有 id 列，按业务键 (feed_url, item_guid) 删除。
-      await Seen.delete().where({ feed_url: row.feed_url, item_guid: row.item_guid });
+    // The table has no id column. Rows of one batch share (feed_url, seen_at), so one delete per batch.
+    const batches = new Map<string, { feed_url: unknown; seen_at: string }>();
+    for (const row of all) {
+      if (typeof row.seen_at !== 'string' || row.seen_at >= cutoffStr) continue;
+      batches.set(`${String(row.feed_url)}\0${row.seen_at}`, { feed_url: row.feed_url, seen_at: row.seen_at });
     }
+    for (const batch of batches.values()) await Seen.delete().where(batch);
   } catch (e) {
     // Cleanup failure must not break the main flow, but it must be visible.
     console.warn(`[rss] cleanOldSeen failed: ${e instanceof Error ? e.message : String(e)}`);

package/src/public-fetch.ts

@@ -0,0 +1,160 @@
+/**
+ * Outbound address policy for URLs that come from chat users or third-party pages (SSRF guard).
+ *
+ * This process connects to the target itself, so a URL is requested only when its host is a
+ * public name and every address it resolves to is public. Redirects are followed by hand and
+ * each hop is checked again. The check and the connection use separate DNS lookups, so a
+ * rebinding resolver can still race them; that needs a connection-level lookup hook.
+ *
+ * Plugins may not depend on each other, so this small module is duplicated in the plugins that
+ * fetch user-supplied URLs. Keep the copies identical.
+ */
+import { lookup as dnsLookup } from 'node:dns/promises';
+import { BlockList, isIP } from 'node:net';
+
+export class BlockedUrlError extends Error {
+  readonly code = 'blocked_url';
+  constructor(reason: string) {
+    super(`refusing to fetch: ${reason}`);
+    this.name = 'BlockedUrlError';
+  }
+}
+
+const blockedV4 = new BlockList();
+for (const [base, bits] of [
+  ['0.0.0.0', 8], ['10.0.0.0', 8], ['100.64.0.0', 10], ['127.0.0.0', 8], ['169.254.0.0', 16],
+  ['172.16.0.0', 12], ['192.0.0.0', 24], ['192.0.2.0', 24], ['192.88.99.0', 24], ['192.168.0.0', 16],
+  ['198.18.0.0', 15], ['198.51.100.0', 24], ['203.0.113.0', 24], ['224.0.0.0', 4], ['240.0.0.0', 4],
+] as const) blockedV4.addSubnet(base, bits, 'ipv4');
+// Host-internal endpoints that sit in otherwise public space (Alibaba metadata, Azure WireServer).
+for (const address of ['100.100.100.200', '168.63.129.16']) blockedV4.addAddress(address, 'ipv4');
+
+// IPv6 is an allow-list: global unicast only, minus the special ranges inside it.
+const globalV6 = new BlockList();
+globalV6.addSubnet('2000::', 3, 'ipv6');
+const blockedV6 = new BlockList();
+for (const [base, bits] of [
+  ['2001::', 32], ['2001:2::', 48], ['2001:10::', 28], ['2001:db8::', 32], ['2002::', 16], ['3fff::', 20],
+] as const) blockedV6.addSubnet(base, bits, 'ipv6');
+const mappedV4 = new BlockList();
+mappedV4.addSubnet('::ffff:0:0', 96, 'ipv6');
+
+/** True when a literal IP address is globally routable and not a known host-internal endpoint. */
+export function isPublicAddress(address: string): boolean {
+  const family = isIP(address);
+  if (family === 4) return !blockedV4.check(address, 'ipv4');
+  if (family !== 6) return false;
+  // BlockList matches IPv4 rules against IPv4-mapped IPv6 addresses.
+  if (mappedV4.check(address, 'ipv6')) return !blockedV4.check(address, 'ipv6');
+  return globalV6.check(address, 'ipv6') && !blockedV6.check(address, 'ipv6');
+}
+
+const BLOCKED_SUFFIXES = [
+  'localhost', 'local', 'localdomain', 'internal', 'intranet', 'private', 'corp', 'home', 'lan',
+  'arpa', 'test', 'invalid', 'example', 'onion', 'metadata.goog',
+];
+
+/** Resolves a host name to every address it maps to. */
+export type HostLookup = (hostname: string) => Promise<readonly string[]>;
+
+const systemLookup: HostLookup = async (hostname) =>
+  (await dnsLookup(hostname, { all: true, verbatim: true })).map((entry) => entry.address);
+
+/**
+ * Parse `input` and refuse anything that is not an http(s) URL on a public host.
+ * Fails closed: an unresolvable name, or a name with any non-public address, is refused.
+ */
+export async function assertPublicUrl(
+  input: string | URL,
+  lookup: HostLookup = systemLookup,
+  allowPrivateNetwork = false,
+): Promise<URL> {
+  let url: URL;
+  try {
+    url = new URL(input);
+  } catch {
+    throw new BlockedUrlError('not a valid URL');
+  }
+  if (url.protocol !== 'http:' && url.protocol !== 'https:') throw new BlockedUrlError('only http and https are allowed');
+  if (url.username || url.password) throw new BlockedUrlError('credentials in the URL are not allowed');
+  // Operator opt-in (never chat users): scheme and credential rules still apply, the address policy does not.
+  if (allowPrivateNetwork) return url;
+  // The WHATWG parser already canonicalised odd IPv4 spellings (0x7f.1, 2130706433) to dotted quads.
+  const host = url.hostname.replace(/^\[|\]$/g, '').replace(/\.$/, '').toLowerCase();
+  if (isIP(host)) {
+    if (!isPublicAddress(host)) throw new BlockedUrlError(`${host} is not a public address`);
+    return url;
+  }
+  if (!host.includes('.') || BLOCKED_SUFFIXES.some((suffix) => host === suffix || host.endsWith(`.${suffix}`))) {
+    throw new BlockedUrlError(`${host} is not a public host name`);
+  }
+  let addresses: readonly string[];
+  try {
+    addresses = await lookup(host);
+  } catch {
+    throw new BlockedUrlError(`${host} could not be resolved`);
+  }
+  if (addresses.length === 0) throw new BlockedUrlError(`${host} could not be resolved`);
+  const internal = addresses.find((address) => !isPublicAddress(address));
+  if (internal !== undefined) throw new BlockedUrlError(`${host} resolves to a non-public address`);
+  return url;
+}
+
+export interface PublicFetchOptions {
+  /** Bounds every hop and the body read of the returned response. */
+  readonly timeoutMs?: number;
+  readonly maxRedirects?: number;
+  readonly init?: Omit<RequestInit, 'redirect' | 'signal'>;
+  /** Extra per-hop rule on top of the address policy (for example a host allow-list). */
+  readonly allow?: (url: URL) => boolean;
+  readonly lookup?: HostLookup;
+  /** Skip the public-address requirement. Only for an explicit operator setting; default false. */
+  readonly allowPrivateNetwork?: boolean;
+}
+
+export interface PublicResponse {
+  readonly response: Response;
+  /** URL of the hop that produced `response`. */
+  readonly url: string;
+}
+
+/** Fetch with redirects followed by hand so that every hop passes the address policy first. */
+export async function fetchPublic(input: string, options: PublicFetchOptions = {}): Promise<PublicResponse> {
+  const signal = AbortSignal.timeout(options.timeoutMs ?? 10_000);
+  const maxRedirects = options.maxRedirects ?? 5;
+  let current = input;
+  for (let hop = 0; hop <= maxRedirects; hop += 1) {
+    const url = await assertPublicUrl(current, options.lookup, options.allowPrivateNetwork);
+    if (options.allow && !options.allow(url)) throw new BlockedUrlError(`${url.hostname} is outside the allow-list`);
+    const response = await fetch(url.href, { ...options.init, redirect: 'manual', signal });
+    const location = response.status >= 300 && response.status < 400 ? response.headers.get('location') : null;
+    if (!location) return { response, url: url.href };
+    await response.body?.cancel();
+    current = new URL(location, url).href;
+  }
+  throw new BlockedUrlError('too many redirects');
+}
+
+/** Read a body with a byte cap; the stream is cancelled and `undefined` returned once it is exceeded. */
+export async function readCappedBody(response: Response, maxBytes: number): Promise<Buffer | undefined> {
+  const declared = Number(response.headers.get('content-length'));
+  if (Number.isFinite(declared) && declared > maxBytes) {
+    await response.body?.cancel();
+    return undefined;
+  }
+  if (!response.body) return Buffer.alloc(0);
+  const chunks: Uint8Array[] = [];
+  let total = 0;
+  const reader = response.body.getReader();
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    total += value.byteLength;
+    if (total > maxBytes) {
+      await reader.cancel();
+      return undefined;
+    }
+    chunks.push(value);
+  }
+  return Buffer.concat(chunks);
+}