agentic-rss-parser 1.0.1

package/src/adapters/provider.js

@@ -0,0 +1,73 @@
+import { AnalysisSchema, validateAnalysis } from '../agent.js';
+import { z } from 'zod';
+
+async function loadProviderModel(provider, modelId, apiKey, baseURL) {
+  if (!provider || provider === 'heuristic') return null;
+
+  if (provider === 'openai') {
+    const { openai } = await import('@ai-sdk/openai');
+    return openai(modelId ?? 'gpt-4o-mini', apiKey ? { apiKey, baseURL } : baseURL ? { baseURL } : undefined);
+  }
+
+  if (provider === 'anthropic') {
+    const { anthropic } = await import('@ai-sdk/anthropic');
+    return anthropic(modelId ?? 'claude-3-5-sonnet-latest', apiKey ? { apiKey, baseURL } : baseURL ? { baseURL } : undefined);
+  }
+
+  if (provider === 'local') {
+    const { openai } = await import('@ai-sdk/openai');
+    return openai(modelId ?? 'local-model', {
+      apiKey: apiKey ?? 'local',
+      baseURL: baseURL ?? 'http://localhost:11434/v1'
+    });
+  }
+
+  throw new Error(`Unsupported provider: ${provider}`);
+}
+
+export async function createAnalyzer(config = {}) {
+  const model = await loadProviderModel(config.provider, config.model, config.apiKey, config.baseURL);
+
+  if (!model) {
+    return async ({ item, context }) => {
+      const text = `${item.title}\n${item.contentSnippet ?? ''}\n${context ?? ''}`.toLowerCase();
+      const signals = ['release', 'security', 'vulnerability', 'node', 'javascript', 'typescript', 'framework', 'api', 'breaking', 'performance', 'agent', 'rss'];
+      const score = signals.reduce((total, signal) => total + (text.includes(signal) ? 1 : 0), 0);
+      return validateAnalysis({
+        decision: score >= 3 ? 'relevant' : 'ignore',
+        confidence: Math.min(95, 35 + score * 10),
+        summary: score >= 3 ? `Likely worth reading: ${item.title}` : `Low-signal item: ${item.title}`,
+        impact: score >= 3 ? 'Could affect engineering decisions or tooling.' : 'Probably noise for a technical feed.',
+        actionItems: score >= 3 ? ['Review the source article.', 'Share with the relevant team if actionable.'] : [],
+        tags: signals.filter((signal) => text.includes(signal)).slice(0, 5)
+      });
+    };
+  }
+
+  return async ({ item, context }) => {
+    const { generateObject } = await import('ai');
+    const schema = z.object({
+      decision: z.enum(['relevant', 'ignore']),
+      confidence: z.number().int().min(0).max(100),
+      summary: z.string().min(1),
+      impact: z.string().min(1),
+      actionItems: z.array(z.string()).default([]),
+      tags: z.array(z.string()).default([])
+    });
+
+    const prompt = [
+      `Title: ${item.title}`,
+      `URL: ${item.link}`,
+      `Feed snippet: ${item.contentSnippet ?? ''}`,
+      `Expanded context: ${context ?? ''}`
+    ].join('\n');
+
+    const response = await generateObject({
+      model,
+      schema,
+      prompt
+    });
+
+    return AnalysisSchema.parse(response.object);
+  };
+}

package/src/agent.js

@@ -0,0 +1,71 @@
+import { z } from 'zod';
+import { fetchFullArticle } from './fetch-article.js';
+
+function validateAnalysis(result) {
+  const decision = result?.decision === 'relevant' ? 'relevant' : 'ignore';
+  const confidence = Number.isFinite(result?.confidence)
+    ? Math.max(0, Math.min(100, Math.trunc(result.confidence)))
+    : 0;
+  const summary = typeof result?.summary === 'string' && result.summary.trim() ? result.summary.trim() : 'No summary provided.';
+  const impact = typeof result?.impact === 'string' && result.impact.trim() ? result.impact.trim() : 'No impact provided.';
+  const actionItems = Array.isArray(result?.actionItems)
+    ? result.actionItems.filter((item) => typeof item === 'string' && item.trim()).map((item) => item.trim())
+    : [];
+  const tags = Array.isArray(result?.tags)
+    ? [...new Set(result.tags.filter((tag) => typeof tag === 'string' && tag.trim()).map((tag) => tag.trim()))]
+    : [];
+
+  return { decision, confidence, summary, impact, actionItems, tags };
+}
+
+export const AnalysisSchema = z.object({
+  decision: z.enum(['relevant', 'ignore']),
+  confidence: z.number().int().min(0).max(100),
+  summary: z.string().min(1),
+  impact: z.string().min(1),
+  actionItems: z.array(z.string()),
+  tags: z.array(z.string())
+});
+
+function heuristicAnalyze(item, context) {
+  const text = `${item.title}\n${item.contentSnippet ?? ''}\n${context ?? ''}`.toLowerCase();
+  const signals = [
+    'release',
+    'security',
+    'vulnerability',
+    'node',
+    'javascript',
+    'typescript',
+    'framework',
+    'api',
+    'breaking',
+    'performance',
+    'agent',
+    'rss'
+  ];
+  const score = signals.reduce((total, signal) => total + (text.includes(signal) ? 1 : 0), 0);
+  const confidence = Math.min(95, 35 + score * 10);
+  const decision = score >= 3 ? 'relevant' : 'ignore';
+  return validateAnalysis({
+    decision,
+    confidence,
+    summary: decision === 'relevant' ? `Likely worth reading: ${item.title}` : `Low-signal item: ${item.title}`,
+    impact: decision === 'relevant' ? 'Could affect engineering decisions or tooling.' : 'Probably noise for a technical feed.',
+    actionItems: decision === 'relevant' ? ['Review the source article.', 'Share with the relevant team if actionable.'] : [],
+    tags: [...new Set(signals.filter((signal) => text.includes(signal)).slice(0, 5))]
+  });
+}
+
+export async function analyzeFeedItem(item, options = {}) {
+  const shouldFetch = Boolean(options.fetchFullArticle) && !item.contentSnippet;
+  const context = shouldFetch ? await fetchFullArticle(item.link) : item.contentSnippet ?? item.content ?? '';
+
+  if (typeof options.analyzer === 'function') {
+    const result = await options.analyzer({ item, context });
+    return validateAnalysis(result);
+  }
+
+  return heuristicAnalyze(item, context);
+}
+
+export { validateAnalysis };

package/src/cli.js

@@ -0,0 +1,40 @@
+#!/usr/bin/env node
+import { existsSync } from 'node:fs';
+import { resolve } from 'node:path';
+import { runAgenticParser } from './parser.js';
+
+function parseArgs(argv) {
+  const args = { feeds: [], db: './data/rss-agent.db', fetchFullArticle: false };
+  for (let i = 2; i < argv.length; i += 1) {
+    const current = argv[i];
+    if (current === '--feed') args.feeds.push(argv[++i]);
+    else if (current === '--db') args.db = argv[++i];
+    else if (current === '--fetch-full-article') args.fetchFullArticle = true;
+  }
+  return args;
+}
+
+const args = parseArgs(process.argv);
+
+if (!args.feeds.length) {
+  console.error('Usage: agentic-rss-parser --feed <url> [--feed <url>] [--db <path>] [--fetch-full-article]');
+  process.exit(1);
+}
+
+const dbPath = resolve(args.db);
+if (!existsSync(resolve('.'))) {
+  console.error('Working directory not found.');
+  process.exit(1);
+}
+
+const results = await runAgenticParser({
+  feedUrls: args.feeds,
+  dbPath,
+  fetchFullArticle: args.fetchFullArticle
+});
+
+for (const { item, analysis } of results) {
+  if (analysis.decision === 'relevant') {
+    console.log(JSON.stringify({ title: item.title, link: item.link, ...analysis }, null, 2));
+  }
+}

package/src/compat.js

@@ -0,0 +1,82 @@
+import { readFile } from 'node:fs/promises';
+import { runAgenticParser } from './parser.js';
+import { parseFeedXml } from './core/parser.js';
+import { fetchTextWithRedirects } from './core/http.js';
+
+const DEFAULT_OPTIONS = {
+  normalize: true,
+  customFields: { feed: [], item: [] },
+  headers: undefined,
+  timeout: 10000,
+  maxRedirects: 5,
+  requestOptions: {},
+  defaultRSS: 2.0,
+  xml2js: {}
+};
+
+function mergeOptions(options = {}) {
+  return {
+    ...DEFAULT_OPTIONS,
+    ...options,
+    customFields: {
+      feed: [...(DEFAULT_OPTIONS.customFields.feed || []), ...(options.customFields?.feed || [])],
+      item: [...(DEFAULT_OPTIONS.customFields.item || []), ...(options.customFields?.item || [])]
+    },
+    requestOptions: {
+      ...DEFAULT_OPTIONS.requestOptions,
+      ...(options.requestOptions || {})
+    },
+    xml2js: {
+      ...DEFAULT_OPTIONS.xml2js,
+      ...(options.xml2js || {})
+    }
+  };
+}
+
+export class ParserCompat {
+  constructor(options = {}) {
+    this.options = mergeOptions(options);
+  }
+
+  parseURL(url, callback) {
+    const promise = fetchTextWithRedirects(url, this.options).then((xml) => this.parseString(xml));
+
+    return maybeCallback(promise, callback);
+  }
+
+  parseString(xml, callback) {
+    const promise = parseFeedXml(xml, this.options);
+    return maybeCallback(promise, callback);
+  }
+
+  parseFile(filePath, callback) {
+    const promise = readFile(filePath, 'utf8').then((xml) => this.parseString(xml));
+    return maybeCallback(promise, callback);
+  }
+
+  async parseFeed(urls, config = {}) {
+    return runAgenticParser({
+      feedUrls: Array.isArray(urls) ? urls : [urls],
+      dbPath: config.dbPath ?? './data/rss-agent.db',
+      fetchFullArticle: Boolean(config.fetchFullArticle),
+      concurrency: config.concurrency,
+      parserOptions: this.options,
+      analyzer: config.analyzer,
+      model: config.model
+    });
+  }
+}
+
+function maybeCallback(promise, callback) {
+  if (typeof callback === 'function') {
+    promise.then((value) => callback(null, value), (error) => callback(error));
+    return undefined;
+  }
+  return promise;
+}
+
+export function createParser(options = {}) {
+  return new ParserCompat(options);
+}
+
+export default ParserCompat;

package/src/core/http.js

@@ -0,0 +1,63 @@
+const DEFAULT_USER_AGENT = 'agentic-rss-parser/1.0.1';
+
+export async function fetchTextWithRedirects(url, options = {}) {
+  assertHttpUrl(url);
+
+  const maxRedirects = Number.isFinite(options.maxRedirects) ? Math.max(0, options.maxRedirects) : 5;
+  let currentUrl = url;
+  let redirects = 0;
+
+  while (true) {
+    const controller = new AbortController();
+    const timeoutMs = Number.isFinite(options.timeout) ? options.timeout : 10000;
+    const timeoutId = setTimeout(() => controller.abort(new Error('Request timed out')), timeoutMs);
+    const requestOptions = {
+      ...options.requestOptions,
+      signal: controller.signal,
+      redirect: 'manual',
+      headers: {
+        'user-agent': DEFAULT_USER_AGENT,
+        ...(options.headers || {}),
+        ...(options.requestOptions?.headers || {})
+      }
+    };
+
+    try {
+      const response = await fetch(currentUrl, requestOptions);
+
+      if (isRedirect(response.status)) {
+        if (redirects >= maxRedirects) {
+          throw new Error(`Too many redirects while fetching ${url}`);
+        }
+
+        const location = response.headers.get('location');
+        if (!location) {
+          throw new Error(`Redirect response missing Location header for ${currentUrl}`);
+        }
+
+        currentUrl = new URL(location, currentUrl).toString();
+        redirects += 1;
+        continue;
+      }
+
+      if (!response.ok) {
+        throw new Error(`Request failed with status ${response.status}`);
+      }
+
+      return await response.text();
+    } finally {
+      clearTimeout(timeoutId);
+    }
+  }
+}
+
+function isRedirect(status) {
+  return status >= 300 && status < 400;
+}
+
+function assertHttpUrl(url) {
+  const parsed = new URL(url);
+  if (!['http:', 'https:'].includes(parsed.protocol)) {
+    throw new Error(`Unsupported feed URL protocol: ${parsed.protocol}`);
+  }
+}

package/src/core/parser.js

@@ -0,0 +1,143 @@
+import { XMLParser } from 'fast-xml-parser';
+
+function asArray(value) {
+  if (value == null) return [];
+  return Array.isArray(value) ? value : [value];
+}
+
+function stripHtml(html = '') {
+  return String(html)
+    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
+    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/&nbsp;/gi, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+function textValue(value) {
+  if (value == null) return '';
+  if (typeof value === 'string') return value;
+  if (typeof value === 'number' || typeof value === 'boolean') return String(value);
+  if (Array.isArray(value)) return textValue(value[0]);
+  if (typeof value === 'object') {
+    if ('#text' in value) return textValue(value['#text']);
+    if ('#cdata' in value) return textValue(value['#cdata']);
+    if ('_text' in value) return textValue(value._text);
+    const first = Object.values(value)[0];
+    return textValue(first);
+  }
+  return '';
+}
+
+function extractCustomValue(source, key, keepArray = false) {
+  const value = source?.[key];
+  if (value == null) return keepArray ? [] : undefined;
+  if (keepArray) return asArray(value).map(textValue);
+  return textValue(asArray(value)[0]);
+}
+
+function applyCustomFields(target, source, fields = []) {
+  for (const field of fields) {
+    if (typeof field === 'string') {
+      const value = extractCustomValue(source, field);
+      if (value !== undefined) target[field] = value;
+      continue;
+    }
+
+    const [fromField, toField, flags = {}] = field;
+    const value = extractCustomValue(source, fromField, Boolean(flags.keepArray));
+    if (value !== undefined) {
+      target[toField] = value;
+      if (flags.includeSnippet) {
+        target[`${toField}Snippet`] = stripHtml(Array.isArray(value) ? value.join(' ') : value);
+      }
+    }
+  }
+}
+
+function getLinkValue(node) {
+  if (node == null) return '';
+  if (Array.isArray(node)) return getLinkValue(node[0]);
+  if (typeof node === 'string') return node;
+  if (typeof node !== 'object') return textValue(node);
+  if ('@_href' in node) return textValue(node['@_href']);
+  if ('href' in node) return textValue(node.href);
+  return textValue(node);
+}
+
+function normalizeItem(itemNode, options) {
+  const normalized = {
+    title: textValue(itemNode.title),
+    link: getLinkValue(itemNode.link),
+    pubDate: textValue(itemNode.pubDate || itemNode.updated || itemNode.published),
+    isoDate: textValue(itemNode.pubDate || itemNode.updated || itemNode.published),
+    guid: textValue(itemNode.guid || itemNode.id),
+    content: textValue(itemNode['content:encoded'] || itemNode.content || itemNode.summary || itemNode.description),
+    contentSnippet: stripHtml(textValue(itemNode.description || itemNode.summary || itemNode.content || itemNode['content:encoded'])),
+    categories: asArray(itemNode.category ?? itemNode.categories).map(textValue).filter(Boolean)
+  };
+
+  if (!options.normalize) {
+    Object.assign(normalized, itemNode);
+  }
+
+  applyCustomFields(normalized, itemNode, options.customFields?.item || []);
+
+  if (!normalized.creator) {
+    const creator = itemNode.creator || itemNode.author || itemNode['dc:creator'];
+    if (creator) normalized.creator = textValue(creator);
+  }
+
+  const contentSnippet = normalized.contentSnippet || stripHtml(normalized.content || '');
+  normalized.contentSnippet = contentSnippet;
+
+  return normalized;
+}
+
+function normalizeFeed(feedNode, items, options) {
+  const feed = {
+    title: textValue(feedNode.title),
+    description: textValue(feedNode.description || feedNode.subtitle),
+    link: getLinkValue(feedNode.link),
+    feedUrl: textValue(feedNode.feedUrl),
+    items
+  };
+
+  applyCustomFields(feed, feedNode, options.customFields?.feed || []);
+  return feed;
+}
+
+function pickFeedNode(parsed) {
+  if (parsed?.rss?.channel) return parsed.rss.channel;
+  if (parsed?.feed) return parsed.feed;
+  if (parsed?.channel) return parsed.channel;
+  return parsed;
+}
+
+function getFeedAndItems(parsed) {
+  const feedContainer = pickFeedNode(parsed);
+  const feedNode = Array.isArray(feedContainer) ? feedContainer[0] : feedContainer;
+  const rawItems = asArray(feedNode?.item || feedNode?.entry);
+  return { feedNode: feedNode || {}, rawItems };
+}
+
+export async function parseFeedXml(xml, options = {}) {
+  const parser = new XMLParser({
+    ignoreAttributes: false,
+    attributeNamePrefix: '@_',
+    textNodeName: '#text',
+    cdataPropName: '#cdata',
+    trimValues: true,
+    allowBooleanAttributes: true,
+    parseTagValue: false,
+    parseAttributeValue: false,
+    removeNSPrefix: false,
+    ...options.xml2js
+  });
+
+  const parsed = parser.parse(xml);
+  const { feedNode, rawItems } = getFeedAndItems(parsed);
+  const items = rawItems.map((itemNode) => normalizeItem(itemNode, options));
+  return normalizeFeed(feedNode, items, options);
+}

package/src/fetch-article.js

@@ -0,0 +1,29 @@
+const MAX_SNIPPET_CHARS = 1200;
+
+function stripHtml(html) {
+  return html
+    .replace(/<script[\s\S]*?<\/script>/gi, ' ')
+    .replace(/<style[\s\S]*?<\/style>/gi, ' ')
+    .replace(/<[^>]+>/g, ' ')
+    .replace(/&nbsp;/gi, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
+}
+
+export async function fetchFullArticle(url) {
+  const response = await fetch(url, {
+    headers: {
+      'user-agent': 'agentic-rss-parser/1.0 (+https://example.local)'
+    }
+  });
+
+  if (!response.ok) {
+    throw new Error(`Failed to fetch article: ${response.status} ${response.statusText}`);
+  }
+
+  const contentType = response.headers.get('content-type') || '';
+  const body = await response.text();
+  const plainText = contentType.includes('text/html') ? stripHtml(body) : body.replace(/\s+/g, ' ').trim();
+
+  return plainText.slice(0, MAX_SNIPPET_CHARS);
+}

package/src/index.d.ts

@@ -0,0 +1,135 @@
+export interface CustomFieldTuple {
+  0: string;
+  1: string;
+  2?: {
+    keepArray?: boolean;
+    includeSnippet?: boolean;
+  };
+}
+
+export interface CustomFieldConfig {
+  feed?: Array<string | [string, string] | CustomFieldTuple>;
+  item?: Array<string | [string, string] | CustomFieldTuple>;
+}
+
+export interface ParserOptions {
+  customFields?: CustomFieldConfig;
+  defaultRSS?: number | string;
+  headers?: Record<string, string>;
+  timeout?: number;
+  maxRedirects?: number;
+  requestOptions?: Record<string, unknown>;
+  xml2js?: Record<string, unknown>;
+  normalize?: boolean;
+}
+
+export interface ParserFeedItem {
+  title?: string;
+  link?: string;
+  pubDate?: string;
+  isoDate?: string;
+  content?: string;
+  contentSnippet?: string;
+  guid?: string;
+  categories?: string[];
+  creator?: string;
+  [key: string]: unknown;
+}
+
+export interface ParserFeed<Feed = unknown, Item = ParserFeedItem> {
+  feedUrl?: string;
+  title?: string;
+  description?: string;
+  link?: string;
+  items: Item[];
+  [key: string]: unknown;
+}
+
+export type ParserCallback<T> = (err: Error | null, result?: T) => void;
+
+export class Parser<Feed = unknown, Item = ParserFeedItem> {
+  constructor(options?: ParserOptions);
+  parseURL(url: string): Promise<ParserFeed<Feed, Item>>;
+  parseURL(url: string, callback: ParserCallback<ParserFeed<Feed, Item>>): void;
+  parseString(xml: string): Promise<ParserFeed<Feed, Item>>;
+  parseString(xml: string, callback: ParserCallback<ParserFeed<Feed, Item>>): void;
+  parseFile(path: string): Promise<ParserFeed<Feed, Item>>;
+  parseFile(path: string, callback: ParserCallback<ParserFeed<Feed, Item>>): void;
+  parseFeed(
+    urls: string | string[],
+    config?: {
+      dbPath?: string;
+      fetchFullArticle?: boolean;
+      concurrency?: number;
+      analyzer?: (input: { item: Item; context: string }) => unknown;
+      model?: {
+        provider?: 'heuristic' | 'openai' | 'anthropic' | 'local';
+        model?: string;
+        apiKey?: string;
+        baseURL?: string;
+      };
+    }
+  ): Promise<Array<{ item: Item; analysis: unknown }>>;
+}
+
+export function createParser<Feed = unknown, Item = ParserFeedItem>(
+  options?: ParserOptions
+): Parser<Feed, Item>;
+
+export function runAgenticParser(config: {
+  feedUrls: string[];
+  dbPath: string;
+  fetchFullArticle?: boolean;
+  concurrency?: number;
+  parserOptions?: ParserOptions;
+  analyzer?: (input: { item: ParserFeedItem; context: string }) => unknown;
+  model?: {
+    provider?: 'heuristic' | 'openai' | 'anthropic' | 'local';
+    model?: string;
+    apiKey?: string;
+    baseURL?: string;
+  };
+}): Promise<Array<{ item: ParserFeedItem; analysis: unknown }>>;
+
+export function analyzeFeedItem(
+  item: ParserFeedItem,
+  options?: {
+    fetchFullArticle?: boolean;
+    analyzer?: (input: { item: ParserFeedItem; context: string }) => unknown;
+  }
+): Promise<{
+  decision: 'relevant' | 'ignore';
+  confidence: number;
+  summary: string;
+  impact: string;
+  actionItems: string[];
+  tags: string[];
+}>;
+
+export function fetchFullArticle(url: string): Promise<string>;
+export function createStorage(dbPath: string): {
+  hasProcessed(id: string): boolean;
+  markProcessed(item: { id: string; feedUrl: string; title: string; link: string; publishedAt?: string | null }): void;
+  saveAnalysis(
+    itemId: string,
+    analysis: {
+      id: string;
+      decision: string;
+      confidence: number;
+      summary: string;
+      impact: string;
+      actionItems: string[];
+      tags: string[];
+    }
+  ): void;
+  close(): void;
+};
+export function createAnalyzer(config?: {
+  provider?: 'heuristic' | 'openai' | 'anthropic' | 'local';
+  model?: string;
+  apiKey?: string;
+  baseURL?: string;
+}): Promise<(input: { item: ParserFeedItem; context: string }) => Promise<unknown>>;
+
+declare const ParserDefault: typeof Parser;
+export default ParserDefault;

package/src/index.js

@@ -0,0 +1,7 @@
+export { runAgenticParser } from './parser.js';
+export { analyzeFeedItem } from './agent.js';
+export { fetchFullArticle } from './fetch-article.js';
+export { createStorage } from './storage.js';
+export { createAnalyzer } from './adapters/provider.js';
+export { ParserCompat as Parser, createParser } from './compat.js';
+export { ParserCompat as default } from './compat.js';