alif-digest 1.0.1

package/src/core/pipeline.ts

@@ -0,0 +1,171 @@
+import { Config } from './config-schema.js';
+import { ScraperOrchestrator } from './orchestrator.js';
+import { ScraperSource } from './scraper-types.js';
+import { KeywordScorer } from './filters/keywords.js';
+import { Deduplicator } from './filters/deduplicator.js';
+import { ArticleStore } from '../db/article-store.js';
+import { SourceHealthStore } from '../db/source-health-store.js';
+import { Database } from 'better-sqlite3';
+import { ProviderFactory } from '../providers/factory.js';
+import { LLMProvider } from '../providers/llm/index.js';
+import { DeliveryProvider, Digest } from '../providers/delivery/index.js';
+import { BASE_KEYWORDS } from './default-keywords.js';
+
+export class Pipeline {
+  private orchestrator: ScraperOrchestrator;
+  private scorer: KeywordScorer;
+  private deduplicator: Deduplicator;
+  private articleStore: ArticleStore;
+  private healthStore: SourceHealthStore;
+  private llm: LLMProvider;
+  private delivery: DeliveryProvider[];
+
+  constructor(
+    private config: Config,
+    db: Database,
+  ) {
+    this.orchestrator = new ScraperOrchestrator();
+
+    const mergedKeywords = {
+      ...BASE_KEYWORDS,
+      ...(config.preferences.customKeywords || {}),
+    };
+    this.scorer = new KeywordScorer(mergedKeywords);
+
+    this.deduplicator = new Deduplicator();
+    this.articleStore = new ArticleStore(db);
+    this.healthStore = new SourceHealthStore(db);
+    this.llm = ProviderFactory.createLLM(config);
+    this.delivery = ProviderFactory.createDelivery(config);
+  }
+
+  async run(sources: ScraperSource[], force = false) {
+    const cooldown = this.config.preferences.sourceCooldownMinutes;
+    const activeSources = sources.filter((s) => {
+      if (!force && this.healthStore.isThrottled(s.id, cooldown)) {
+        console.log(`[Pipeline] Skipping ${s.id} (last check was < ${cooldown}m ago)`);
+        return false;
+      }
+      return true;
+    });
+
+    let enrichedItems: any[] = [];
+    const digestDate = new Date().toISOString().split('T')[0];
+    let newItemsCount = 0;
+
+    if (activeSources.length > 0) {
+      console.log(`[Pipeline] Scraping ${activeSources.length} sources...`);
+      const results = await this.orchestrator.runAll(activeSources);
+
+      // Record health
+      for (const res of results) {
+        this.healthStore.record({
+          source: res.source,
+          status: res.status,
+          items_found: res.items.length,
+          error_message: res.error,
+        });
+      }
+
+      const allArticles = results.flatMap((r) => r.items);
+      console.log(`[Pipeline] Found ${allArticles.length} raw items.`);
+
+      // Filtering
+      const latestTimestamp = this.articleStore.getLatestTimestamp();
+      let newItems = allArticles;
+
+      if (latestTimestamp) {
+        const lastTime = new Date(latestTimestamp).getTime();
+        newItems = allArticles.filter((a) => {
+          if (!a.published_at) return true;
+          return new Date(a.published_at).getTime() > lastTime;
+        });
+      }
+      newItemsCount = newItems.length;
+      console.log(`[Pipeline] ${newItemsCount} new items after incremental filter.`);
+
+      // Scoring & Deduplication
+      const unique = this.deduplicator.process(newItems);
+      const highSignal = unique
+        .map((a) => ({ ...a, score: this.scorer.score(a) }))
+        .filter((a) => a.score >= this.config.preferences.signalThreshold)
+        .sort((a, b) => b.score - a.score);
+
+      console.log(`[Pipeline] ${highSignal.length} high-signal items selected.`);
+
+      if (highSignal.length > 0) {
+        // LLM Analysis
+        console.log('[Pipeline] Analyzing high-signal items with LLM...');
+        const analysisResults = await this.llm.analyze(
+          highSignal.map((a) => ({ title: a.title, content: a.content })),
+        );
+
+        enrichedItems = highSignal.map((article, idx) => ({
+          ...article,
+          summary: analysisResults[idx]?.summary || null,
+          category: analysisResults[idx]?.category || 'Uncategorized',
+        }));
+
+        // Persistence
+        for (const item of enrichedItems) {
+          this.articleStore.upsert({
+            ...item,
+            digest_date: digestDate,
+            delivered: 0,
+          } as any);
+        }
+      } else {
+        console.log('[Pipeline] No high-signal items in this batch.');
+      }
+    } else {
+      console.log('[Pipeline] All sources are cooled down. Checking database for pending items...');
+    }
+
+    // 5. Build Final Digest (Current + Pending from last 24h)
+    const pendingItems = this.articleStore.getPendingHighSignal(
+      this.config.preferences.signalThreshold,
+      24,
+    );
+
+    // Merge and deduplicate by ID (latest run wins)
+    const allToDeliverMap = new Map();
+    for (const item of pendingItems) allToDeliverMap.set(item.id, item);
+    for (const item of enrichedItems) allToDeliverMap.set(item.id, item);
+
+    const finalItemsToDeliver = Array.from(allToDeliverMap.values());
+
+    if (finalItemsToDeliver.length === 0) {
+      console.log('[Pipeline] No high-signal items to deliver.');
+      console.log('[Pipeline] Execution complete! 🥂');
+      return [];
+    }
+
+    // Delivery
+    const digest: Digest = {
+      items: finalItemsToDeliver.map((item) => ({
+        title: item.title,
+        url: item.url,
+        summary: item.summary,
+        category: item.category,
+        source: item.source,
+        score: item.score,
+      })),
+      metadata: {
+        total_new_items: newItemsCount,
+        total_selected: finalItemsToDeliver.length,
+        date: digestDate,
+      },
+    };
+
+    console.log(
+      `[Pipeline] Delivering ${finalItemsToDeliver.length} items to ${this.delivery.length} channels...`,
+    );
+    await Promise.all(this.delivery.map((d) => d.send(digest)));
+
+    // Mark as delivered
+    this.articleStore.markAsDelivered(finalItemsToDeliver.map((a) => a.id));
+
+    console.log('[Pipeline] Execution complete! 🥂');
+    return finalItemsToDeliver;
+  }
+}

package/src/core/scheduler.ts

@@ -0,0 +1,74 @@
+import { ScheduleStore } from '../db/schedule-store.js';
+import { Database } from 'better-sqlite3';
+
+export class Scheduler {
+  private store: ScheduleStore;
+
+  constructor(db: Database) {
+    this.store = new ScheduleStore(db);
+  }
+
+  async add(name: string, cron: string, scheduledTime?: string) {
+    const id = Math.random().toString(36).substring(2, 9);
+    this.store.add({
+      id,
+      name,
+      cron,
+      scheduled_time: scheduledTime,
+      active: 1,
+    });
+    return id;
+  }
+
+  list() {
+    return this.store.getAll();
+  }
+
+  remove(id: string) {
+    this.store.delete(id);
+  }
+
+  // This would be called by a background daemon or a frequent cron
+  async checkAndRun(runner: () => Promise<void>) {
+    const schedules = this.store.getAll();
+    const now = new Date();
+    const currentHHmm = `${now.getHours().toString().padStart(2, '0')}:${now.getMinutes().toString().padStart(2, '0')}`;
+
+    for (const schedule of schedules) {
+      if (!schedule.active) continue;
+
+      const lastRun = schedule.last_run ? new Date(schedule.last_run) : new Date(0);
+      const diffHours = (now.getTime() - lastRun.getTime()) / (1000 * 60 * 60);
+
+      // Simple implementation:
+      // 1. If never run, run it instantly.
+      // 2. If it's a 'daily' schedule and has a scheduled_time, check if we've passed that time today and haven't run yet.
+      // 3. Otherwise fallback to the 24h cooldown.
+
+      let shouldRun = false;
+
+      if (!schedule.last_run) {
+        shouldRun = true;
+      } else if (schedule.cron === 'daily' && schedule.scheduled_time) {
+        // Run if:
+        // - Current time >= scheduled time
+        // - Last run was NOT today
+        const lastRunDate = lastRun.toISOString().split('T')[0];
+        const todayDate = now.toISOString().split('T')[0];
+
+        if (currentHHmm >= schedule.scheduled_time && lastRunDate !== todayDate) {
+          shouldRun = true;
+        }
+      } else if (diffHours >= 24) {
+        // Fallback for non-daily or simple daily without time
+        shouldRun = true;
+      }
+
+      if (shouldRun) {
+        console.log(`[Scheduler] Running job: ${schedule.name}`);
+        await runner();
+        this.store.updateLastRun(schedule.id, now.toISOString());
+      }
+    }
+  }
+}

package/src/core/scraper-types.ts

@@ -0,0 +1,30 @@
+export interface ScrapedArticle {
+  id: string;
+  title: string;
+  url: string;
+  content?: string;
+  published_at?: string;
+  source: string;
+  metadata?: Record<string, unknown>;
+}
+
+export interface ScraperSource {
+  id: string;
+  name: string;
+  type: 'rss' | 'api' | 'scrape' | 'json';
+  url: string;
+  tier?: number;
+  tags?: string[];
+  mapping?: Record<string, string>;
+}
+
+export interface ScraperResult {
+  source: string;
+  status: 'ok' | 'error';
+  items: ScrapedArticle[];
+  error?: string;
+}
+
+export abstract class BaseScraper {
+  abstract scrape(source: ScraperSource): Promise<ScraperResult>;
+}

package/src/core/scrapers/api-scraper.ts

@@ -0,0 +1,45 @@
+import axios from 'axios';
+import { BaseScraper, ScraperSource, ScraperResult, ScrapedArticle } from '../scraper-types.js';
+
+export class ApiScraper extends BaseScraper {
+  async scrape(source: ScraperSource): Promise<ScraperResult> {
+    try {
+      const response = await axios.get(source.url);
+      const data = response.data;
+      let items: ScrapedArticle[] = [];
+
+      // Specialized logic based on source ID for common APIs
+      if (source.id === 'hn') {
+        items = data.hits.map((hit: any) => ({
+          id: hit.objectID,
+          title: hit.title,
+          url: hit.url || `https://news.ycombinator.com/item?id=${hit.objectID}`,
+          content: hit.story_text || '',
+          published_at: hit.created_at,
+          source: source.id,
+        }));
+      } else if (source.id.startsWith('arxiv')) {
+        // ArXiv returns XML/Atom, but we'll assume the URL handles it or use a specific XML parser if needed.
+        // For brevity in this generic API scraper, we'll keep it simple.
+        // In a real scenario, we'd use a dedicated library or cheerio for XML.
+        return {
+          source: source.id,
+          status: 'error',
+          items: [],
+          error: 'ArXiv requires specialized XML parsing',
+        };
+      } else {
+        return { source: source.id, status: 'error', items: [], error: 'Unsupported API source' };
+      }
+
+      return { source: source.id, status: 'ok', items };
+    } catch (error) {
+      return {
+        source: source.id,
+        status: 'error',
+        items: [],
+        error: error instanceof Error ? error.message : String(error),
+      };
+    }
+  }
+}

package/src/core/scrapers/arxiv-scraper.ts

@@ -0,0 +1,35 @@
+import axios from 'axios';
+import * as cheerio from 'cheerio';
+import { BaseScraper, ScraperSource, ScraperResult, ScrapedArticle } from '../scraper-types.js';
+
+export class ArxivScraper extends BaseScraper {
+  async scrape(source: ScraperSource): Promise<ScraperResult> {
+    try {
+      const response = await axios.get(source.url);
+      const $ = cheerio.load(response.data, { xmlMode: true });
+      const items: ScrapedArticle[] = [];
+
+      $('entry').each((_, el) => {
+        const $el = $(el);
+        const url = $el.find('id').text().trim();
+        items.push({
+          id: url,
+          title: $el.find('title').text().trim().replace(/\s+/g, ' '),
+          url,
+          content: $el.find('summary').text().trim().replace(/\s+/g, ' '),
+          published_at: $el.find('published').text().trim(),
+          source: source.id,
+        });
+      });
+
+      return { source: source.id, status: 'ok', items };
+    } catch (error) {
+      return {
+        source: source.id,
+        status: 'error',
+        items: [],
+        error: error instanceof Error ? error.message : String(error),
+      };
+    }
+  }
+}

package/src/core/scrapers/json-scraper.ts

@@ -0,0 +1,54 @@
+import axios from 'axios';
+import { BaseScraper, ScraperSource, ScraperResult, ScrapedArticle } from '../scraper-types.js';
+
+export class JsonScraper extends BaseScraper {
+  async scrape(source: ScraperSource): Promise<ScraperResult> {
+    try {
+      const response = await axios.get(source.url);
+      const data = response.data;
+      let items: ScrapedArticle[] = [];
+
+      if (source.mapping) {
+        // Generic JSON array mapping
+        const list = Array.isArray(data) ? data : data[source.mapping.items || 'items'];
+        if (Array.isArray(list)) {
+          items = list.map((item: any, index: number) => ({
+            id: item.id || `json-${source.id}-${index}`,
+            title: item[source.mapping!.title || 'title'] || 'No Title',
+            url: item[source.mapping!.url || 'url'] || source.url,
+            content: item[source.mapping!.content || 'content'] || '',
+            published_at: item[source.mapping!.published_at || 'date'] || new Date().toISOString(),
+            source: source.id,
+          }));
+        }
+      } else if (source.id.startsWith('reddit')) {
+        items = data.data.children.map((child: any) => {
+          const post = child.data;
+          return {
+            id: post.name,
+            title: post.title,
+            url: post.url.startsWith('/') ? `https://reddit.com${post.url}` : post.url,
+            content: post.selftext || '',
+            published_at: new Date(post.created_utc * 1000).toISOString(),
+            source: source.id,
+          };
+        });
+      } else {
+        return {
+          source: source.id,
+          status: 'error',
+          items: [],
+          error: 'Unsupported JSON source or missing mapping',
+        };
+      }
+      return { source: source.id, status: 'ok', items };
+    } catch (error) {
+      return {
+        source: source.id,
+        status: 'error',
+        items: [],
+        error: error instanceof Error ? error.message : String(error),
+      };
+    }
+  }
+}

package/src/core/scrapers/rss-scraper.ts

@@ -0,0 +1,34 @@
+import Parser from 'rss-parser';
+import { BaseScraper, ScraperSource, ScraperResult, ScrapedArticle } from '../scraper-types.js';
+
+export class RssScraper extends BaseScraper {
+  private parser: Parser;
+
+  constructor() {
+    super();
+    this.parser = new Parser();
+  }
+
+  async scrape(source: ScraperSource): Promise<ScraperResult> {
+    try {
+      const feed = await this.parser.parseURL(source.url);
+      const items: ScrapedArticle[] = feed.items.map((item) => ({
+        id: item.guid || item.link || `${source.id}-${item.title}`,
+        title: item.title || 'No Title',
+        url: item.link || '',
+        content: item.contentSnippet || item.content || '',
+        published_at: item.isoDate || item.pubDate,
+        source: source.id,
+      }));
+
+      return { source: source.id, status: 'ok', items };
+    } catch (error) {
+      return {
+        source: source.id,
+        status: 'error',
+        items: [],
+        error: error instanceof Error ? error.message : String(error),
+      };
+    }
+  }
+}

package/src/core/scrapers/scrape-scraper.ts

@@ -0,0 +1,50 @@
+import axios from 'axios';
+import * as cheerio from 'cheerio';
+import { BaseScraper, ScraperSource, ScraperResult, ScrapedArticle } from '../scraper-types.js';
+
+export class ScrapeScraper extends BaseScraper {
+  async scrape(source: ScraperSource): Promise<ScraperResult> {
+    try {
+      const response = await axios.get(source.url, {
+        headers: {
+          'User-Agent': 'Mozilla/5.0 (compatible; AlifBot/1.0)',
+        },
+      });
+      const $ = cheerio.load(response.data);
+      const items: ScrapedArticle[] = [];
+
+      if (source.id === 'github_trending') {
+        $('.Box-row').each((_, el) => {
+          const $el = $(el);
+          const title = $el.find('h2 a').text().trim().replace(/\s+/g, ' ');
+          const url = 'https://github.com' + $el.find('h2 a').attr('href');
+          const content = $el.find('p').text().trim();
+
+          items.push({
+            id: url,
+            title,
+            url,
+            content,
+            source: source.id,
+          });
+        });
+      } else {
+        return {
+          source: source.id,
+          status: 'error',
+          items: [],
+          error: 'Unsupported scraping source',
+        };
+      }
+
+      return { source: source.id, status: 'ok', items };
+    } catch (error) {
+      return {
+        source: source.id,
+        status: 'error',
+        items: [],
+        error: error instanceof Error ? error.message : String(error),
+      };
+    }
+  }
+}

package/src/db/article-store.ts

@@ -0,0 +1,75 @@
+import { Database } from 'better-sqlite3';
+
+export interface Article {
+  id: string;
+  title: string;
+  url: string;
+  source: string;
+  content?: string | null;
+  summary?: string | null;
+  category?: string;
+  published_at?: string;
+  score?: number;
+  digest_date: string;
+  delivered?: number;
+}
+
+export class ArticleStore {
+  constructor(private db: Database) {}
+
+  upsert(article: Article) {
+    const stmt = this.db.prepare(`
+      INSERT INTO articles (id, title, url, source, content, summary, category, published_at, score, digest_date, delivered)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(id) DO UPDATE SET
+        title = excluded.title,
+        summary = excluded.summary,
+        category = excluded.category,
+        score = excluded.score
+    `);
+
+    stmt.run(
+      article.id,
+      article.title,
+      article.url,
+      article.source,
+      article.content || null,
+      article.summary || null,
+      article.category || 'Uncategorized',
+      article.published_at || null,
+      article.score || 0,
+      article.digest_date,
+      article.delivered || 0,
+    );
+  }
+
+  getPendingHighSignal(threshold: number, hours: number = 24): Article[] {
+    const cutoff = new Date(Date.now() - hours * 60 * 60 * 1000).toISOString();
+    return this.db
+      .prepare(
+        `
+      SELECT * FROM articles 
+      WHERE score >= ? 
+      AND delivered = 0 
+      AND summary IS NOT NULL
+      AND (published_at > ? OR digest_date > ?)
+    `,
+      )
+      .all(threshold, cutoff, cutoff.split('T')[0]) as Article[];
+  }
+
+  markAsDelivered(articleIds: string[]) {
+    const stmt = this.db.prepare('UPDATE articles SET delivered = 1 WHERE id = ?');
+    const transaction = this.db.transaction((ids: string[]) => {
+      for (const id of ids) stmt.run(id);
+    });
+    transaction(articleIds);
+  }
+
+  getLatestTimestamp(): string | null {
+    const result = this.db.prepare('SELECT MAX(published_at) as latest FROM articles').get() as {
+      latest: string | null;
+    };
+    return result?.latest || null;
+  }
+}

package/src/db/connection.ts

@@ -0,0 +1,17 @@
+import Database, { Database as DatabaseType } from 'better-sqlite3';
+import path from 'path';
+import fs from 'fs';
+
+export function createDatabase(dbPath: string): DatabaseType {
+  const dbDir = path.dirname(dbPath);
+  if (!fs.existsSync(dbDir)) {
+    fs.mkdirSync(dbDir, { recursive: true });
+  }
+
+  const db = new Database(dbPath);
+  db.pragma('journal_mode = WAL');
+  db.pragma('synchronous = NORMAL');
+  db.pragma('foreign_keys = ON');
+
+  return db;
+}

package/src/db/migrate.ts

@@ -0,0 +1,68 @@
+import { Database } from 'better-sqlite3';
+
+const MIGRATIONS = [
+  `
+  CREATE TABLE IF NOT EXISTS articles (
+    id TEXT PRIMARY KEY,
+    title TEXT NOT NULL,
+    url TEXT NOT NULL,
+    source TEXT NOT NULL,
+    content TEXT,
+    summary TEXT,
+    category TEXT,
+    published_at TEXT,
+    score INTEGER,
+    digest_date TEXT NOT NULL
+  );
+  `,
+  `
+  CREATE TABLE IF NOT EXISTS source_health (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    source TEXT NOT NULL,
+    status TEXT NOT NULL,
+    items_found INTEGER DEFAULT 0,
+    error_message TEXT,
+    last_check TEXT DEFAULT CURRENT_TIMESTAMP
+  );
+  `,
+  `
+  CREATE TABLE IF NOT EXISTS schedules (
+    id TEXT PRIMARY KEY,
+    name TEXT NOT NULL,
+    cron TEXT NOT NULL,
+    active INTEGER DEFAULT 1,
+    last_run TEXT
+  );
+  `,
+  `
+  ALTER TABLE schedules ADD COLUMN scheduled_time TEXT;
+  `,
+  `
+  ALTER TABLE articles ADD COLUMN delivered INTEGER DEFAULT 0;
+  `,
+];
+
+export function runMigrations(db: Database) {
+  db.transaction(() => {
+    // Basic migration tracking
+    db.prepare(
+      `
+      CREATE TABLE IF NOT EXISTS migrations (
+        id INTEGER PRIMARY KEY,
+        executed_at TEXT DEFAULT CURRENT_TIMESTAMP
+      )
+    `,
+    ).run();
+
+    const result = db.prepare('SELECT MAX(id) as lastId FROM migrations').get() as {
+      lastId: number | null;
+    };
+    const lastId = result?.lastId ?? -1;
+
+    for (let i = lastId + 1; i < MIGRATIONS.length; i++) {
+      console.log(`[Database] Running migration ${i}...`);
+      db.prepare(MIGRATIONS[i]).run();
+      db.prepare('INSERT INTO migrations (id) VALUES (?)').run(i);
+    }
+  })();
+}