alif-digest 1.0.1

package/dist/core/default-keywords.js

@@ -0,0 +1,10 @@
+export const BASE_KEYWORDS = {
+    breakthrough: 20,
+    'gpt-5': 30,
+    o1: 20,
+    deepseek: 25,
+    'open source': 15,
+    agi: 15,
+    agent: 15,
+};
+//# sourceMappingURL=default-keywords.js.map

package/dist/core/default-keywords.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"default-keywords.js","sourceRoot":"","sources":["../../src/core/default-keywords.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,aAAa,GAA2B;IACnD,YAAY,EAAE,EAAE;IAChB,OAAO,EAAE,EAAE;IACX,EAAE,EAAE,EAAE;IACN,QAAQ,EAAE,EAAE;IACZ,aAAa,EAAE,EAAE;IACjB,GAAG,EAAE,EAAE;IACP,KAAK,EAAE,EAAE;CACV,CAAC"}

package/dist/core/filters/deduplicator.d.ts

@@ -0,0 +1,10 @@
+import { ScrapedArticle } from '../scraper-types.js';
+export interface DeduplicatorOptions {
+    similarityThreshold?: number;
+}
+export declare class Deduplicator {
+    private options;
+    constructor(options?: DeduplicatorOptions);
+    process(articles: ScrapedArticle[]): ScrapedArticle[];
+    private normalizeUrl;
+}

package/dist/core/filters/deduplicator.js

@@ -0,0 +1,34 @@
+export class Deduplicator {
+    options;
+    constructor(options = {}) {
+        this.options = options;
+    }
+    process(articles) {
+        const unique = new Map();
+        for (const article of articles) {
+            // Primary key: URL (simplified)
+            const urlKey = this.normalizeUrl(article.url);
+            if (!unique.has(urlKey)) {
+                unique.set(urlKey, article);
+            }
+            else {
+                // If we have an existing one, keep the one with content if possible
+                const existing = unique.get(urlKey);
+                if (!existing.content && article.content) {
+                    unique.set(urlKey, article);
+                }
+            }
+        }
+        return Array.from(unique.values());
+    }
+    normalizeUrl(url) {
+        try {
+            const u = new URL(url);
+            return u.origin + u.pathname.replace(/\/$/, '');
+        }
+        catch {
+            return url;
+        }
+    }
+}
+//# sourceMappingURL=deduplicator.js.map

package/dist/core/filters/deduplicator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"deduplicator.js","sourceRoot":"","sources":["../../../src/core/filters/deduplicator.ts"],"names":[],"mappings":"AAMA,MAAM,OAAO,YAAY;IACH;IAApB,YAAoB,UAA+B,EAAE;QAAjC,YAAO,GAAP,OAAO,CAA0B;IAAG,CAAC;IAEzD,OAAO,CAAC,QAA0B;QAChC,MAAM,MAAM,GAAG,IAAI,GAAG,EAA0B,CAAC;QAEjD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,gCAAgC;YAChC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YAE9C,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACxB,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,oEAAoE;gBACpE,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,CAAE,CAAC;gBACrC,IAAI,CAAC,QAAQ,CAAC,OAAO,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;oBACzC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACrC,CAAC;IAEO,YAAY,CAAC,GAAW;QAC9B,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YACvB,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,CAAC;QACb,CAAC;IACH,CAAC;CACF"}

package/dist/core/filters/keywords.d.ts

@@ -0,0 +1,6 @@
+import { ScrapedArticle } from '../scraper-types.js';
+export declare class KeywordScorer {
+    private keywords;
+    constructor(keywords: Record<string, number>);
+    score(article: ScrapedArticle): number;
+}

package/dist/core/filters/keywords.js

@@ -0,0 +1,17 @@
+export class KeywordScorer {
+    keywords;
+    constructor(keywords) {
+        this.keywords = keywords;
+    }
+    score(article) {
+        let score = 0;
+        const text = `${article.title} ${article.content || ''}`.toLowerCase();
+        for (const [keyword, weight] of Object.entries(this.keywords)) {
+            if (text.includes(keyword.toLowerCase())) {
+                score += weight;
+            }
+        }
+        return score;
+    }
+}
+//# sourceMappingURL=keywords.js.map

package/dist/core/filters/keywords.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"keywords.js","sourceRoot":"","sources":["../../../src/core/filters/keywords.ts"],"names":[],"mappings":"AAEA,MAAM,OAAO,aAAa;IACJ;IAApB,YAAoB,QAAgC;QAAhC,aAAQ,GAAR,QAAQ,CAAwB;IAAG,CAAC;IAExD,KAAK,CAAC,OAAuB;QAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,IAAI,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,IAAI,EAAE,EAAE,CAAC,WAAW,EAAE,CAAC;QAEvE,KAAK,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC9D,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;gBACzC,KAAK,IAAI,MAAM,CAAC;YAClB,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}

package/dist/core/orchestrator.d.ts

@@ -0,0 +1,6 @@
+import { ScraperSource, ScraperResult } from './scraper-types.js';
+export declare class ScraperOrchestrator {
+    private scrapers;
+    constructor();
+    runAll(sources: ScraperSource[]): Promise<ScraperResult[]>;
+}

package/dist/core/orchestrator.js

@@ -0,0 +1,44 @@
+import { RssScraper } from './scrapers/rss-scraper.js';
+import { ArxivScraper } from './scrapers/arxiv-scraper.js';
+import { ApiScraper } from './scrapers/api-scraper.js';
+import { ScrapeScraper } from './scrapers/scrape-scraper.js';
+import { JsonScraper } from './scrapers/json-scraper.js';
+export class ScraperOrchestrator {
+    scrapers = {};
+    constructor() {
+        this.scrapers.rss = new RssScraper();
+        this.scrapers.arxiv = new ArxivScraper();
+        this.scrapers.api = new ApiScraper();
+        this.scrapers.scrape = new ScrapeScraper();
+        this.scrapers.json = new JsonScraper();
+    }
+    async runAll(sources) {
+        const tasks = sources.map((source) => {
+            let scraper;
+            // Priority 1: Match by ID for specialized logic in reference
+            if (source.id === 'hn')
+                scraper = this.scrapers.api;
+            else if (source.id === 'github_trending')
+                scraper = this.scrapers.scrape;
+            else if (source.id.startsWith('arxiv'))
+                scraper = this.scrapers.arxiv;
+            else if (source.id.startsWith('reddit'))
+                scraper = this.scrapers.json;
+            // Priority 2: Use explicitly defined type
+            if (!scraper) {
+                scraper = this.scrapers[source.type];
+            }
+            if (!scraper) {
+                return Promise.resolve({
+                    source: source.id,
+                    status: 'error',
+                    items: [],
+                    error: `No scraper found for type: ${source.type}`,
+                });
+            }
+            return scraper.scrape(source);
+        });
+        return Promise.all(tasks);
+    }
+}
+//# sourceMappingURL=orchestrator.js.map

package/dist/core/orchestrator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AAC3D,OAAO,EAAE,UAAU,EAAE,MAAM,2BAA2B,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,8BAA8B,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AAEzD,MAAM,OAAO,mBAAmB;IACtB,QAAQ,GAAgC,EAAE,CAAC;IAEnD;QACE,IAAI,CAAC,QAAQ,CAAC,GAAG,GAAG,IAAI,UAAU,EAAE,CAAC;QACrC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,IAAI,YAAY,EAAE,CAAC;QACzC,IAAI,CAAC,QAAQ,CAAC,GAAG,GAAG,IAAI,UAAU,EAAE,CAAC;QACrC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QAC3C,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAwB;QACnC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE;YACnC,IAAI,OAAgC,CAAC;YAErC,6DAA6D;YAC7D,IAAI,MAAM,CAAC,EAAE,KAAK,IAAI;gBAAE,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;iBAC/C,IAAI,MAAM,CAAC,EAAE,KAAK,iBAAiB;gBAAE,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;iBACpE,IAAI,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC;gBAAE,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;iBACjE,IAAI,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC;gBAAE,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAEtE,0CAA0C;YAC1C,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACvC,CAAC;YAED,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,OAAO,OAAO,CAAC,OAAO,CAAC;oBACrB,MAAM,EAAE,MAAM,CAAC,EAAE;oBACjB,MAAM,EAAE,OAAO;oBACf,KAAK,EAAE,EAAE;oBACT,KAAK,EAAE,8BAA8B,MAAM,CAAC,IAAI,EAAE;iBAClC,CAAC,CAAC;YACtB,CAAC;YACD,OAAO,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;CACF"}

package/dist/core/pipeline.d.ts

@@ -0,0 +1,15 @@
+import { Config } from './config-schema.js';
+import { ScraperSource } from './scraper-types.js';
+import { Database } from 'better-sqlite3';
+export declare class Pipeline {
+    private config;
+    private orchestrator;
+    private scorer;
+    private deduplicator;
+    private articleStore;
+    private healthStore;
+    private llm;
+    private delivery;
+    constructor(config: Config, db: Database);
+    run(sources: ScraperSource[], force?: boolean): Promise<any[]>;
+}

package/dist/core/pipeline.js

@@ -0,0 +1,140 @@
+import { ScraperOrchestrator } from './orchestrator.js';
+import { KeywordScorer } from './filters/keywords.js';
+import { Deduplicator } from './filters/deduplicator.js';
+import { ArticleStore } from '../db/article-store.js';
+import { SourceHealthStore } from '../db/source-health-store.js';
+import { ProviderFactory } from '../providers/factory.js';
+import { BASE_KEYWORDS } from './default-keywords.js';
+export class Pipeline {
+    config;
+    orchestrator;
+    scorer;
+    deduplicator;
+    articleStore;
+    healthStore;
+    llm;
+    delivery;
+    constructor(config, db) {
+        this.config = config;
+        this.orchestrator = new ScraperOrchestrator();
+        const mergedKeywords = {
+            ...BASE_KEYWORDS,
+            ...(config.preferences.customKeywords || {}),
+        };
+        this.scorer = new KeywordScorer(mergedKeywords);
+        this.deduplicator = new Deduplicator();
+        this.articleStore = new ArticleStore(db);
+        this.healthStore = new SourceHealthStore(db);
+        this.llm = ProviderFactory.createLLM(config);
+        this.delivery = ProviderFactory.createDelivery(config);
+    }
+    async run(sources, force = false) {
+        const cooldown = this.config.preferences.sourceCooldownMinutes;
+        const activeSources = sources.filter((s) => {
+            if (!force && this.healthStore.isThrottled(s.id, cooldown)) {
+                console.log(`[Pipeline] Skipping ${s.id} (last check was < ${cooldown}m ago)`);
+                return false;
+            }
+            return true;
+        });
+        let enrichedItems = [];
+        const digestDate = new Date().toISOString().split('T')[0];
+        let newItemsCount = 0;
+        if (activeSources.length > 0) {
+            console.log(`[Pipeline] Scraping ${activeSources.length} sources...`);
+            const results = await this.orchestrator.runAll(activeSources);
+            // Record health
+            for (const res of results) {
+                this.healthStore.record({
+                    source: res.source,
+                    status: res.status,
+                    items_found: res.items.length,
+                    error_message: res.error,
+                });
+            }
+            const allArticles = results.flatMap((r) => r.items);
+            console.log(`[Pipeline] Found ${allArticles.length} raw items.`);
+            // Filtering
+            const latestTimestamp = this.articleStore.getLatestTimestamp();
+            let newItems = allArticles;
+            if (latestTimestamp) {
+                const lastTime = new Date(latestTimestamp).getTime();
+                newItems = allArticles.filter((a) => {
+                    if (!a.published_at)
+                        return true;
+                    return new Date(a.published_at).getTime() > lastTime;
+                });
+            }
+            newItemsCount = newItems.length;
+            console.log(`[Pipeline] ${newItemsCount} new items after incremental filter.`);
+            // Scoring & Deduplication
+            const unique = this.deduplicator.process(newItems);
+            const highSignal = unique
+                .map((a) => ({ ...a, score: this.scorer.score(a) }))
+                .filter((a) => a.score >= this.config.preferences.signalThreshold)
+                .sort((a, b) => b.score - a.score);
+            console.log(`[Pipeline] ${highSignal.length} high-signal items selected.`);
+            if (highSignal.length > 0) {
+                // LLM Analysis
+                console.log('[Pipeline] Analyzing high-signal items with LLM...');
+                const analysisResults = await this.llm.analyze(highSignal.map((a) => ({ title: a.title, content: a.content })));
+                enrichedItems = highSignal.map((article, idx) => ({
+                    ...article,
+                    summary: analysisResults[idx]?.summary || null,
+                    category: analysisResults[idx]?.category || 'Uncategorized',
+                }));
+                // Persistence
+                for (const item of enrichedItems) {
+                    this.articleStore.upsert({
+                        ...item,
+                        digest_date: digestDate,
+                        delivered: 0,
+                    });
+                }
+            }
+            else {
+                console.log('[Pipeline] No high-signal items in this batch.');
+            }
+        }
+        else {
+            console.log('[Pipeline] All sources are cooled down. Checking database for pending items...');
+        }
+        // 5. Build Final Digest (Current + Pending from last 24h)
+        const pendingItems = this.articleStore.getPendingHighSignal(this.config.preferences.signalThreshold, 24);
+        // Merge and deduplicate by ID (latest run wins)
+        const allToDeliverMap = new Map();
+        for (const item of pendingItems)
+            allToDeliverMap.set(item.id, item);
+        for (const item of enrichedItems)
+            allToDeliverMap.set(item.id, item);
+        const finalItemsToDeliver = Array.from(allToDeliverMap.values());
+        if (finalItemsToDeliver.length === 0) {
+            console.log('[Pipeline] No high-signal items to deliver.');
+            console.log('[Pipeline] Execution complete! 🥂');
+            return [];
+        }
+        // Delivery
+        const digest = {
+            items: finalItemsToDeliver.map((item) => ({
+                title: item.title,
+                url: item.url,
+                summary: item.summary,
+                category: item.category,
+                source: item.source,
+                score: item.score,
+            })),
+            metadata: {
+                total_new_items: newItemsCount,
+                total_selected: finalItemsToDeliver.length,
+                date: digestDate,
+            },
+        };
+        console.log(`[Pipeline] Delivering ${finalItemsToDeliver.length} items to ${this.delivery.length} channels...`);
+        await Promise.all(this.delivery.map((d) => d.send(digest)));
+        // Mark as delivered
+        this.articleStore.markAsDelivered(finalItemsToDeliver.map((a) => a.id));
+        console.log('[Pipeline] Execution complete! 🥂');
+        return finalItemsToDeliver;
+    }
+}
+//# sourceMappingURL=pipeline.js.map

package/dist/core/pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/core/pipeline.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AAEjE,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAG1D,OAAO,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAEtD,MAAM,OAAO,QAAQ;IAUT;IATF,YAAY,CAAsB;IAClC,MAAM,CAAgB;IACtB,YAAY,CAAe;IAC3B,YAAY,CAAe;IAC3B,WAAW,CAAoB;IAC/B,GAAG,CAAc;IACjB,QAAQ,CAAqB;IAErC,YACU,MAAc,EACtB,EAAY;QADJ,WAAM,GAAN,MAAM,CAAQ;QAGtB,IAAI,CAAC,YAAY,GAAG,IAAI,mBAAmB,EAAE,CAAC;QAE9C,MAAM,cAAc,GAAG;YACrB,GAAG,aAAa;YAChB,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,cAAc,IAAI,EAAE,CAAC;SAC7C,CAAC;QACF,IAAI,CAAC,MAAM,GAAG,IAAI,aAAa,CAAC,cAAc,CAAC,CAAC;QAEhD,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,EAAE,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,IAAI,iBAAiB,CAAC,EAAE,CAAC,CAAC;QAC7C,IAAI,CAAC,GAAG,GAAG,eAAe,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC7C,IAAI,CAAC,QAAQ,GAAG,eAAe,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAwB,EAAE,KAAK,GAAG,KAAK;QAC/C,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,qBAAqB,CAAC;QAC/D,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACzC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,QAAQ,CAAC,EAAE,CAAC;gBAC3D,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC,EAAE,sBAAsB,QAAQ,QAAQ,CAAC,CAAC;gBAC/E,OAAO,KAAK,CAAC;YACf,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;QAEH,IAAI,aAAa,GAAU,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1D,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,uBAAuB,aAAa,CAAC,MAAM,aAAa,CAAC,CAAC;YACtE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;YAE9D,gBAAgB;YAChB,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC1B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC;oBACtB,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;oBAClB,WAAW,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM;oBAC7B,aAAa,EAAE,GAAG,CAAC,KAAK;iBACzB,CAAC,CAAC;YACL,CAAC;YAED,MAAM,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACpD,OAAO,CAAC,GAAG,CAAC,oBAAoB,WAAW,CAAC,MAAM,aAAa,CAAC,CAAC;YAEjE,YAAY;YACZ,MAAM,eAAe,GAAG,IAAI,CAAC,YAAY,CAAC,kBAAkB,EAAE,CAAC;YAC/D,IAAI,QAAQ,GAAG,WAAW,CAAC;YAE3B,IAAI,eAAe,EAAE,CAAC;gBACpB,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,eAAe,CAAC,CAAC,OAAO,EAAE,CAAC;gBACrD,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;oBAClC,IAAI,CAAC,CAAC,CAAC,YAAY;wBAAE,OAAO,IAAI,CAAC;oBACjC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,OAAO,EAAE,GAAG,QAAQ,CAAC;gBACvD,CAAC,CAAC,CAAC;YACL,CAAC;YACD,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YAChC,OAAO,CAAC,GAAG,CAAC,cAAc,aAAa,sCAAsC,CAAC,CAAC;YAE/E,0BAA0B;YAC1B,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YACnD,MAAM,UAAU,GAAG,MAAM;iBACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;iBACnD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,eAAe,CAAC;iBACjE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YAErC,OAAO,CAAC,GAAG,CAAC,cAAc,UAAU,CAAC,MAAM,8BAA8B,CAAC,CAAC;YAE3E,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC1B,eAAe;gBACf,OAAO,CAAC,GAAG,CAAC,oDAAoD,CAAC,CAAC;gBAClE,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,OAAO,CAC5C,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAChE,CAAC;gBAEF,aAAa,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;oBAChD,GAAG,OAAO;oBACV,OAAO,EAAE,eAAe,CAAC,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI;oBAC9C,QAAQ,EAAE,eAAe,CAAC,GAAG,CAAC,EAAE,QAAQ,IAAI,eAAe;iBAC5D,CAAC,CAAC,CAAC;gBAEJ,cAAc;gBACd,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;oBACjC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;wBACvB,GAAG,IAAI;wBACP,WAAW,EAAE,UAAU;wBACvB,SAAS,EAAE,CAAC;qBACN,CAAC,CAAC;gBACZ,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;YAChE,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,gFAAgF,CAAC,CAAC;QAChG,CAAC;QAED,0DAA0D;QAC1D,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,oBAAoB,CACzD,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,eAAe,EACvC,EAAE,CACH,CAAC;QAEF,gDAAgD;QAChD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAE,CAAC;QAClC,KAAK,MAAM,IAAI,IAAI,YAAY;YAAE,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QACpE,KAAK,MAAM,IAAI,IAAI,aAAa;YAAE,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAErE,MAAM,mBAAmB,GAAG,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC;QAEjE,IAAI,mBAAmB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;YAC3D,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;YACjD,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,WAAW;QACX,MAAM,MAAM,GAAW;YACrB,KAAK,EAAE,mBAAmB,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACxC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,GAAG,EAAE,IAAI,CAAC,GAAG;gBACb,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;aAClB,CAAC,CAAC;YACH,QAAQ,EAAE;gBACR,eAAe,EAAE,aAAa;gBAC9B,cAAc,EAAE,mBAAmB,CAAC,MAAM;gBAC1C,IAAI,EAAE,UAAU;aACjB;SACF,CAAC;QAEF,OAAO,CAAC,GAAG,CACT,yBAAyB,mBAAmB,CAAC,MAAM,aAAa,IAAI,CAAC,QAAQ,CAAC,MAAM,cAAc,CACnG,CAAC;QACF,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAE5D,oBAAoB;QACpB,IAAI,CAAC,YAAY,CAAC,eAAe,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAExE,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;QACjD,OAAO,mBAAmB,CAAC;IAC7B,CAAC;CACF"}

package/dist/core/scheduler.d.ts

@@ -0,0 +1,9 @@
+import { Database } from 'better-sqlite3';
+export declare class Scheduler {
+    private store;
+    constructor(db: Database);
+    add(name: string, cron: string, scheduledTime?: string): Promise<string>;
+    list(): import("../db/schedule-store.js").Schedule[];
+    remove(id: string): void;
+    checkAndRun(runner: () => Promise<void>): Promise<void>;
+}

package/dist/core/scheduler.js

@@ -0,0 +1,64 @@
+import { ScheduleStore } from '../db/schedule-store.js';
+export class Scheduler {
+    store;
+    constructor(db) {
+        this.store = new ScheduleStore(db);
+    }
+    async add(name, cron, scheduledTime) {
+        const id = Math.random().toString(36).substring(2, 9);
+        this.store.add({
+            id,
+            name,
+            cron,
+            scheduled_time: scheduledTime,
+            active: 1,
+        });
+        return id;
+    }
+    list() {
+        return this.store.getAll();
+    }
+    remove(id) {
+        this.store.delete(id);
+    }
+    // This would be called by a background daemon or a frequent cron
+    async checkAndRun(runner) {
+        const schedules = this.store.getAll();
+        const now = new Date();
+        const currentHHmm = `${now.getHours().toString().padStart(2, '0')}:${now.getMinutes().toString().padStart(2, '0')}`;
+        for (const schedule of schedules) {
+            if (!schedule.active)
+                continue;
+            const lastRun = schedule.last_run ? new Date(schedule.last_run) : new Date(0);
+            const diffHours = (now.getTime() - lastRun.getTime()) / (1000 * 60 * 60);
+            // Simple implementation:
+            // 1. If never run, run it instantly.
+            // 2. If it's a 'daily' schedule and has a scheduled_time, check if we've passed that time today and haven't run yet.
+            // 3. Otherwise fallback to the 24h cooldown.
+            let shouldRun = false;
+            if (!schedule.last_run) {
+                shouldRun = true;
+            }
+            else if (schedule.cron === 'daily' && schedule.scheduled_time) {
+                // Run if:
+                // - Current time >= scheduled time
+                // - Last run was NOT today
+                const lastRunDate = lastRun.toISOString().split('T')[0];
+                const todayDate = now.toISOString().split('T')[0];
+                if (currentHHmm >= schedule.scheduled_time && lastRunDate !== todayDate) {
+                    shouldRun = true;
+                }
+            }
+            else if (diffHours >= 24) {
+                // Fallback for non-daily or simple daily without time
+                shouldRun = true;
+            }
+            if (shouldRun) {
+                console.log(`[Scheduler] Running job: ${schedule.name}`);
+                await runner();
+                this.store.updateLastRun(schedule.id, now.toISOString());
+            }
+        }
+    }
+}
+//# sourceMappingURL=scheduler.js.map

package/dist/core/scheduler.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scheduler.js","sourceRoot":"","sources":["../../src/core/scheduler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAGxD,MAAM,OAAO,SAAS;IACZ,KAAK,CAAgB;IAE7B,YAAY,EAAY;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,aAAa,CAAC,EAAE,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,IAAY,EAAE,IAAY,EAAE,aAAsB;QAC1D,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC;YACb,EAAE;YACF,IAAI;YACJ,IAAI;YACJ,cAAc,EAAE,aAAa;YAC7B,MAAM,EAAE,CAAC;SACV,CAAC,CAAC;QACH,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IAC7B,CAAC;IAED,MAAM,CAAC,EAAU;QACf,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACxB,CAAC;IAED,iEAAiE;IACjE,KAAK,CAAC,WAAW,CAAC,MAA2B;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACtC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,WAAW,GAAG,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;QAEpH,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,QAAQ,CAAC,MAAM;gBAAE,SAAS;YAE/B,MAAM,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;YAC9E,MAAM,SAAS,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,GAAG,CAAC,IAAI,GAAG,EAAE,GAAG,EAAE,CAAC,CAAC;YAEzE,yBAAyB;YACzB,qCAAqC;YACrC,qHAAqH;YACrH,6CAA6C;YAE7C,IAAI,SAAS,GAAG,KAAK,CAAC;YAEtB,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBACvB,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;iBAAM,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,IAAI,QAAQ,CAAC,cAAc,EAAE,CAAC;gBAChE,UAAU;gBACV,mCAAmC;gBACnC,2BAA2B;gBAC3B,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxD,MAAM,SAAS,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;gBAElD,IAAI,WAAW,IAAI,QAAQ,CAAC,cAAc,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;oBACxE,SAAS,GAAG,IAAI,CAAC;gBACnB,CAAC;YACH,CAAC;iBAAM,IAAI,SAAS,IAAI,EAAE,EAAE,CAAC;gBAC3B,sDAAsD;gBACtD,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;YAED,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,CAAC,GAAG,CAAC,4BAA4B,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;gBACzD,MAAM,MAAM,EAAE,CAAC;gBACf,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,QAAQ,CAAC,EAAE,EAAE,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;IACH,CAAC;CACF"}

package/dist/core/scraper-types.d.ts

@@ -0,0 +1,27 @@
+export interface ScrapedArticle {
+    id: string;
+    title: string;
+    url: string;
+    content?: string;
+    published_at?: string;
+    source: string;
+    metadata?: Record<string, unknown>;
+}
+export interface ScraperSource {
+    id: string;
+    name: string;
+    type: 'rss' | 'api' | 'scrape' | 'json';
+    url: string;
+    tier?: number;
+    tags?: string[];
+    mapping?: Record<string, string>;
+}
+export interface ScraperResult {
+    source: string;
+    status: 'ok' | 'error';
+    items: ScrapedArticle[];
+    error?: string;
+}
+export declare abstract class BaseScraper {
+    abstract scrape(source: ScraperSource): Promise<ScraperResult>;
+}

package/dist/core/scraper-types.js

@@ -0,0 +1,3 @@
+export class BaseScraper {
+}
+//# sourceMappingURL=scraper-types.js.map

package/dist/core/scraper-types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scraper-types.js","sourceRoot":"","sources":["../../src/core/scraper-types.ts"],"names":[],"mappings":"AA2BA,MAAM,OAAgB,WAAW;CAEhC"}

package/dist/core/scrapers/api-scraper.d.ts

@@ -0,0 +1,4 @@
+import { BaseScraper, ScraperSource, ScraperResult } from '../scraper-types.js';
+export declare class ApiScraper extends BaseScraper {
+    scrape(source: ScraperSource): Promise<ScraperResult>;
+}

package/dist/core/scrapers/api-scraper.js

@@ -0,0 +1,46 @@
+import axios from 'axios';
+import { BaseScraper } from '../scraper-types.js';
+export class ApiScraper extends BaseScraper {
+    async scrape(source) {
+        try {
+            const response = await axios.get(source.url);
+            const data = response.data;
+            let items = [];
+            // Specialized logic based on source ID for common APIs
+            if (source.id === 'hn') {
+                items = data.hits.map((hit) => ({
+                    id: hit.objectID,
+                    title: hit.title,
+                    url: hit.url || `https://news.ycombinator.com/item?id=${hit.objectID}`,
+                    content: hit.story_text || '',
+                    published_at: hit.created_at,
+                    source: source.id,
+                }));
+            }
+            else if (source.id.startsWith('arxiv')) {
+                // ArXiv returns XML/Atom, but we'll assume the URL handles it or use a specific XML parser if needed.
+                // For brevity in this generic API scraper, we'll keep it simple.
+                // In a real scenario, we'd use a dedicated library or cheerio for XML.
+                return {
+                    source: source.id,
+                    status: 'error',
+                    items: [],
+                    error: 'ArXiv requires specialized XML parsing',
+                };
+            }
+            else {
+                return { source: source.id, status: 'error', items: [], error: 'Unsupported API source' };
+            }
+            return { source: source.id, status: 'ok', items };
+        }
+        catch (error) {
+            return {
+                source: source.id,
+                status: 'error',
+                items: [],
+                error: error instanceof Error ? error.message : String(error),
+            };
+        }
+    }
+}
+//# sourceMappingURL=api-scraper.js.map

package/dist/core/scrapers/api-scraper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"api-scraper.js","sourceRoot":"","sources":["../../../src/core/scrapers/api-scraper.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,WAAW,EAAgD,MAAM,qBAAqB,CAAC;AAEhG,MAAM,OAAO,UAAW,SAAQ,WAAW;IACzC,KAAK,CAAC,MAAM,CAAC,MAAqB;QAChC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7C,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;YAC3B,IAAI,KAAK,GAAqB,EAAE,CAAC;YAEjC,uDAAuD;YACvD,IAAI,MAAM,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;gBACvB,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAQ,EAAE,EAAE,CAAC,CAAC;oBACnC,EAAE,EAAE,GAAG,CAAC,QAAQ;oBAChB,KAAK,EAAE,GAAG,CAAC,KAAK;oBAChB,GAAG,EAAE,GAAG,CAAC,GAAG,IAAI,wCAAwC,GAAG,CAAC,QAAQ,EAAE;oBACtE,OAAO,EAAE,GAAG,CAAC,UAAU,IAAI,EAAE;oBAC7B,YAAY,EAAE,GAAG,CAAC,UAAU;oBAC5B,MAAM,EAAE,MAAM,CAAC,EAAE;iBAClB,CAAC,CAAC,CAAC;YACN,CAAC;iBAAM,IAAI,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;gBACzC,sGAAsG;gBACtG,iEAAiE;gBACjE,uEAAuE;gBACvE,OAAO;oBACL,MAAM,EAAE,MAAM,CAAC,EAAE;oBACjB,MAAM,EAAE,OAAO;oBACf,KAAK,EAAE,EAAE;oBACT,KAAK,EAAE,wCAAwC;iBAChD,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,wBAAwB,EAAE,CAAC;YAC5F,CAAC;YAED,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,MAAM,EAAE,MAAM,CAAC,EAAE;gBACjB,MAAM,EAAE,OAAO;gBACf,KAAK,EAAE,EAAE;gBACT,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}

package/dist/core/scrapers/arxiv-scraper.d.ts

@@ -0,0 +1,4 @@
+import { BaseScraper, ScraperSource, ScraperResult } from '../scraper-types.js';
+export declare class ArxivScraper extends BaseScraper {
+    scrape(source: ScraperSource): Promise<ScraperResult>;
+}

package/dist/core/scrapers/arxiv-scraper.js

@@ -0,0 +1,34 @@
+import axios from 'axios';
+import * as cheerio from 'cheerio';
+import { BaseScraper } from '../scraper-types.js';
+export class ArxivScraper extends BaseScraper {
+    async scrape(source) {
+        try {
+            const response = await axios.get(source.url);
+            const $ = cheerio.load(response.data, { xmlMode: true });
+            const items = [];
+            $('entry').each((_, el) => {
+                const $el = $(el);
+                const url = $el.find('id').text().trim();
+                items.push({
+                    id: url,
+                    title: $el.find('title').text().trim().replace(/\s+/g, ' '),
+                    url,
+                    content: $el.find('summary').text().trim().replace(/\s+/g, ' '),
+                    published_at: $el.find('published').text().trim(),
+                    source: source.id,
+                });
+            });
+            return { source: source.id, status: 'ok', items };
+        }
+        catch (error) {
+            return {
+                source: source.id,
+                status: 'error',
+                items: [],
+                error: error instanceof Error ? error.message : String(error),
+            };
+        }
+    }
+}
+//# sourceMappingURL=arxiv-scraper.js.map

package/dist/core/scrapers/arxiv-scraper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"arxiv-scraper.js","sourceRoot":"","sources":["../../../src/core/scrapers/arxiv-scraper.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,WAAW,EAAgD,MAAM,qBAAqB,CAAC;AAEhG,MAAM,OAAO,YAAa,SAAQ,WAAW;IAC3C,KAAK,CAAC,MAAM,CAAC,MAAqB;QAChC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7C,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,MAAM,KAAK,GAAqB,EAAE,CAAC;YAEnC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;gBACxB,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;gBAClB,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACzC,KAAK,CAAC,IAAI,CAAC;oBACT,EAAE,EAAE,GAAG;oBACP,KAAK,EAAE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;oBAC3D,GAAG;oBACH,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;oBAC/D,YAAY,EAAE,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;oBACjD,MAAM,EAAE,MAAM,CAAC,EAAE;iBAClB,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,MAAM,EAAE,MAAM,CAAC,EAAE;gBACjB,MAAM,EAAE,OAAO;gBACf,KAAK,EAAE,EAAE;gBACT,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}

package/dist/core/scrapers/json-scraper.d.ts

@@ -0,0 +1,4 @@
+import { BaseScraper, ScraperSource, ScraperResult } from '../scraper-types.js';
+export declare class JsonScraper extends BaseScraper {
+    scrape(source: ScraperSource): Promise<ScraperResult>;
+}

package/dist/core/scrapers/json-scraper.js

@@ -0,0 +1,56 @@
+import axios from 'axios';
+import { BaseScraper } from '../scraper-types.js';
+export class JsonScraper extends BaseScraper {
+    async scrape(source) {
+        try {
+            const response = await axios.get(source.url);
+            const data = response.data;
+            let items = [];
+            if (source.mapping) {
+                // Generic JSON array mapping
+                const list = Array.isArray(data) ? data : data[source.mapping.items || 'items'];
+                if (Array.isArray(list)) {
+                    items = list.map((item, index) => ({
+                        id: item.id || `json-${source.id}-${index}`,
+                        title: item[source.mapping.title || 'title'] || 'No Title',
+                        url: item[source.mapping.url || 'url'] || source.url,
+                        content: item[source.mapping.content || 'content'] || '',
+                        published_at: item[source.mapping.published_at || 'date'] || new Date().toISOString(),
+                        source: source.id,
+                    }));
+                }
+            }
+            else if (source.id.startsWith('reddit')) {
+                items = data.data.children.map((child) => {
+                    const post = child.data;
+                    return {
+                        id: post.name,
+                        title: post.title,
+                        url: post.url.startsWith('/') ? `https://reddit.com${post.url}` : post.url,
+                        content: post.selftext || '',
+                        published_at: new Date(post.created_utc * 1000).toISOString(),
+                        source: source.id,
+                    };
+                });
+            }
+            else {
+                return {
+                    source: source.id,
+                    status: 'error',
+                    items: [],
+                    error: 'Unsupported JSON source or missing mapping',
+                };
+            }
+            return { source: source.id, status: 'ok', items };
+        }
+        catch (error) {
+            return {
+                source: source.id,
+                status: 'error',
+                items: [],
+                error: error instanceof Error ? error.message : String(error),
+            };
+        }
+    }
+}
+//# sourceMappingURL=json-scraper.js.map

package/dist/core/scrapers/json-scraper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-scraper.js","sourceRoot":"","sources":["../../../src/core/scrapers/json-scraper.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,WAAW,EAAgD,MAAM,qBAAqB,CAAC;AAEhG,MAAM,OAAO,WAAY,SAAQ,WAAW;IAC1C,KAAK,CAAC,MAAM,CAAC,MAAqB;QAChC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC7C,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;YAC3B,IAAI,KAAK,GAAqB,EAAE,CAAC;YAEjC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBACnB,6BAA6B;gBAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,CAAC;gBAChF,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;oBACxB,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,IAAS,EAAE,KAAa,EAAE,EAAE,CAAC,CAAC;wBAC9C,EAAE,EAAE,IAAI,CAAC,EAAE,IAAI,QAAQ,MAAM,CAAC,EAAE,IAAI,KAAK,EAAE;wBAC3C,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,OAAQ,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,UAAU;wBAC3D,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,OAAQ,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI,MAAM,CAAC,GAAG;wBACrD,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,OAAQ,CAAC,OAAO,IAAI,SAAS,CAAC,IAAI,EAAE;wBACzD,YAAY,EAAE,IAAI,CAAC,MAAM,CAAC,OAAQ,CAAC,YAAY,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;wBACtF,MAAM,EAAE,MAAM,CAAC,EAAE;qBAClB,CAAC,CAAC,CAAC;gBACN,CAAC;YACH,CAAC;iBAAM,IAAI,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC1C,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAU,EAAE,EAAE;oBAC5C,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;oBACxB,OAAO;wBACL,EAAE,EAAE,IAAI,CAAC,IAAI;wBACb,KAAK,EAAE,IAAI,CAAC,KAAK;wBACjB,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG;wBAC1E,OAAO,EAAE,IAAI,CAAC,QAAQ,IAAI,EAAE;wBAC5B,YAAY,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE;wBAC7D,MAAM,EAAE,MAAM,CAAC,EAAE;qBAClB,CAAC;gBACJ,CAAC,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,OAAO;oBACL,MAAM,EAAE,MAAM,CAAC,EAAE;oBACjB,MAAM,EAAE,OAAO;oBACf,KAAK,EAAE,EAAE;oBACT,KAAK,EAAE,4CAA4C;iBACpD,CAAC;YACJ,CAAC;YACD,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;QACpD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO;gBACL,MAAM,EAAE,MAAM,CAAC,EAAE;gBACjB,MAAM,EAAE,OAAO;gBACf,KAAK,EAAE,EAAE;gBACT,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}

package/dist/core/scrapers/rss-scraper.d.ts

@@ -0,0 +1,6 @@
+import { BaseScraper, ScraperSource, ScraperResult } from '../scraper-types.js';
+export declare class RssScraper extends BaseScraper {
+    private parser;
+    constructor();
+    scrape(source: ScraperSource): Promise<ScraperResult>;
+}