agentic-rss-parser 1.0.1

package/src/mcp/server.js

@@ -0,0 +1,62 @@
+#!/usr/bin/env node
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { z } from 'zod';
+import { runAgenticParser } from '../parser.js';
+import { createAnalyzer } from '../adapters/provider.js';
+import { fetchFullArticle } from '../fetch-article.js';
+
+const server = new McpServer({
+  name: 'agentic-rss-parser',
+  version: '1.0.0'
+});
+
+server.tool(
+  'fetch_rss_feed',
+  {
+    url: z.string().url().describe('The RSS or Atom feed URL to fetch.'),
+    limit: z.number().int().min(1).max(100).default(10).describe('Maximum number of items to return.'),
+    provider: z.enum(['heuristic', 'openai', 'anthropic', 'local']).default('heuristic').describe('Analysis provider to use.')
+  },
+  async ({ url, limit, provider }) => {
+    const analyzer = await createAnalyzer({ provider });
+    const results = await runAgenticParser({
+      feedUrls: [url],
+      dbPath: './data/rss-agent.db',
+      analyzer,
+      model: { provider }
+    });
+
+    return {
+      content: [
+        {
+          type: 'text',
+          text: JSON.stringify(results.slice(0, limit), null, 2)
+        }
+      ]
+    };
+  }
+);
+
+server.tool(
+  'fetch_full_article',
+  {
+    url: z.string().url().describe('The article URL to fetch.'),
+    provider: z.enum(['heuristic', 'openai', 'anthropic', 'local']).default('heuristic').describe('Analysis provider to use.')
+  },
+  async ({ url, provider }) => {
+    const text = await fetchFullArticle(url);
+
+    return {
+      content: [
+        {
+          type: 'text',
+          text: JSON.stringify({ url, text }, null, 2)
+        }
+      ]
+    };
+  }
+);
+
+const transport = new StdioServerTransport();
+await server.connect(transport);

package/src/parser.js

@@ -0,0 +1,81 @@
+import crypto from 'node:crypto';
+import { analyzeFeedItem } from './agent.js';
+import { createStorage } from './storage.js';
+import { createAnalyzer } from './adapters/provider.js';
+import { parseFeedXml } from './core/parser.js';
+import { fetchTextWithRedirects } from './core/http.js';
+
+function normalizeItem(feedUrl, item) {
+  const link = item.link || item.guid || '';
+  const id = crypto.createHash('sha256').update(`${feedUrl}:${link || item.title}`).digest('hex');
+  return {
+    id,
+    feedUrl,
+    title: item.title?.trim() || 'Untitled item',
+    link,
+    publishedAt: item.isoDate || item.pubDate || null,
+    contentSnippet: item.contentSnippet || item.content || ''
+  };
+}
+
+export async function runAgenticParser(config) {
+  const storage = createStorage(config.dbPath);
+  const results = [];
+  const analyzer = config.analyzer ?? await createAnalyzer(config.model);
+  const concurrency = normalizeConcurrency(config.concurrency);
+
+  try {
+    const feedRuns = await mapWithConcurrency(config.feedUrls, concurrency, async (feedUrl) => {
+      const xml = await fetchTextWithRedirects(feedUrl, config.parserOptions);
+      const feed = await parseFeedXml(xml, config.parserOptions);
+      const feedResults = [];
+
+      for (const item of feed.items) {
+        const normalized = normalizeItem(feedUrl, item);
+        if (storage.hasProcessed(normalized.id)) continue;
+
+        const analysis = await analyzeFeedItem(normalized, {
+          fetchFullArticle: config.fetchFullArticle,
+          analyzer
+        });
+
+        storage.markProcessed(normalized);
+        storage.saveAnalysis(normalized.id, {
+          id: crypto.randomUUID(),
+          ...analysis
+        });
+        feedResults.push({ item: normalized, analysis });
+      }
+
+      return feedResults;
+    });
+
+    results.push(...feedRuns.flat());
+    return results;
+  } finally {
+    storage.close();
+  }
+}
+
+function normalizeConcurrency(concurrency) {
+  const parsed = Number(concurrency);
+  if (!Number.isFinite(parsed) || parsed < 1) return 1;
+  return Math.min(16, Math.trunc(parsed));
+}
+
+async function mapWithConcurrency(items, limit, worker) {
+  const results = new Array(items.length);
+  let index = 0;
+
+  async function next() {
+    while (index < items.length) {
+      const currentIndex = index;
+      index += 1;
+      results[currentIndex] = await worker(items[currentIndex], currentIndex);
+    }
+  }
+
+  const workers = Array.from({ length: Math.min(limit, items.length) }, () => next());
+  await Promise.all(workers);
+  return results;
+}

package/src/storage.js

@@ -0,0 +1,62 @@
+import { DatabaseSync } from 'node:sqlite';
+import { mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+
+export function createStorage(dbPath) {
+  mkdirSync(dirname(dbPath), { recursive: true });
+  const db = new DatabaseSync(dbPath);
+
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS processed_items (
+      id TEXT PRIMARY KEY,
+      feed_url TEXT NOT NULL,
+      title TEXT NOT NULL,
+      link TEXT NOT NULL,
+      published_at TEXT,
+      processed_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+    ) STRICT;
+
+    CREATE TABLE IF NOT EXISTS analyses (
+      id TEXT PRIMARY KEY,
+      item_id TEXT NOT NULL,
+      decision TEXT NOT NULL,
+      confidence INTEGER NOT NULL,
+      summary TEXT NOT NULL,
+      impact TEXT NOT NULL,
+      action_items TEXT NOT NULL,
+      tags TEXT NOT NULL,
+      created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+    ) STRICT;
+  `);
+
+  return {
+    hasProcessed(id) {
+      const row = db.prepare('SELECT 1 FROM processed_items WHERE id = ?').get(id);
+      return Boolean(row);
+    },
+    markProcessed(item) {
+      db.prepare(
+        'INSERT OR IGNORE INTO processed_items (id, feed_url, title, link, published_at) VALUES (?, ?, ?, ?, ?)'
+      ).run(item.id, item.feedUrl, item.title, item.link, item.publishedAt ?? null);
+    },
+    saveAnalysis(itemId, analysis) {
+      db.prepare(
+        `INSERT OR REPLACE INTO analyses
+         (id, item_id, decision, confidence, summary, impact, action_items, tags)
+         VALUES (?, ?, ?, ?, ?, ?, ?, ?)`
+      ).run(
+        analysis.id,
+        itemId,
+        analysis.decision,
+        analysis.confidence,
+        analysis.summary,
+        analysis.impact,
+        JSON.stringify(analysis.actionItems),
+        JSON.stringify(analysis.tags)
+      );
+    },
+    close() {
+      db.close();
+    }
+  };
+}

package/src/tools.js

@@ -0,0 +1,2 @@
+export { analyzeFeedItem } from './agent.js';
+export { fetchFullArticle } from './fetch-article.js';