alif-digest 1.0.1

package/package.json

@@ -0,0 +1,66 @@
+{
+  "name": "alif-digest",
+  "version": "1.0.1",
+  "description": "Autonomous AI Signal Digest CLI for tracking high-signal AI breakthroughs.",
+  "type": "module",
+  "bin": {
+    "alif": "./dist/cli/index.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "start": "node dist/cli/index.js",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "lint": "eslint .",
+    "format": "prettier --write .",
+    "dev": "tsx src/cli/index.ts",
+    "prepare": "husky"
+  },
+  "lint-staged": {
+    "*.{ts,js,mjs,json,md}": [
+      "prettier --write",
+      "eslint --fix"
+    ]
+  },
+  "keywords": [
+    "ai",
+    "cli",
+    "digest",
+    "newsletter",
+    "automation",
+    "ollama",
+    "anthropic",
+    "signal"
+  ],
+  "author": "qaribhaider",
+  "license": "MIT",
+  "dependencies": {
+    "axios": "^1.13.6",
+    "better-sqlite3": "^12.6.2",
+    "cheerio": "^1.2.0",
+    "commander": "^14.0.3",
+    "dotenv": "^17.3.1",
+    "openai": "^6.26.0",
+    "prompts": "^2.4.2",
+    "rss-parser": "^3.13.0",
+    "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "@eslint/js": "^10.0.1",
+    "@types/better-sqlite3": "^7.6.13",
+    "@types/node": "^25.3.3",
+    "@types/prompts": "^2.4.9",
+    "@typescript-eslint/eslint-plugin": "^8.56.1",
+    "@typescript-eslint/parser": "^8.56.1",
+    "eslint": "^10.0.2",
+    "globals": "^17.4.0",
+    "husky": "^9.1.7",
+    "lint-staged": "^16.3.2",
+    "prettier": "^3.8.1",
+    "ts-node": "^10.9.2",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3",
+    "typescript-eslint": "^8.56.1",
+    "vitest": "^4.0.18"
+  }
+}

package/src/cli/commands/init.ts

@@ -0,0 +1,94 @@
+import prompts from 'prompts';
+import path from 'path';
+import { ConfigManager } from '../../core/config-manager.js';
+import { Config } from '../../core/config-schema.js';
+
+export async function initCommand() {
+  const configManager = ConfigManager.getInstance();
+  const configDir = configManager.getConfigDir();
+
+  console.log('--- Alif Initialization ---');
+
+  const response = await prompts([
+    {
+      type: 'select',
+      name: 'llmProvider',
+      message: 'Which LLM provider would you like to use?',
+      choices: [
+        { title: 'Ollama (Local)', value: 'ollama' },
+        { title: 'Anthropic', value: 'anthropic' },
+        { title: 'OpenRouter', value: 'openrouter' },
+      ],
+    },
+    {
+      type: (prev) => (prev !== 'ollama' ? 'text' : null),
+      name: 'apiKey',
+      message: 'Enter your API Key:',
+    },
+    {
+      type: 'text',
+      name: 'model',
+      message: 'Enter the model name (e.g., llama3, claude-3-opus-20240229):',
+      initial: (prev, values) => {
+        if (values.llmProvider === 'ollama') return 'llama3';
+        if (values.llmProvider === 'anthropic') return 'claude-3-5-sonnet-20240620';
+        return 'meta-llama/llama-3-70b-instruct';
+      },
+    },
+    {
+      type: (prev, values) => (values.llmProvider === 'ollama' ? 'text' : null),
+      name: 'baseUrl',
+      message: 'Enter Ollama base URL:',
+      initial: 'http://localhost:11434',
+    },
+    {
+      type: 'multiselect',
+      name: 'deliveryProviders',
+      message: 'Where should we deliver the digest?',
+      choices: [
+        { title: 'Slack', value: 'slack' },
+        { title: 'Generic Webhook', value: 'webhook' },
+      ],
+      min: 1,
+    },
+  ]);
+
+  if (!response.llmProvider) {
+    console.log('Initialization cancelled.');
+    return;
+  }
+
+  const deliveryConfigs = [];
+  for (const provider of response.deliveryProviders) {
+    const { webhookUrl } = await prompts({
+      type: 'text',
+      name: 'webhookUrl',
+      message: `Enter ${provider} Webhook URL:`,
+    });
+    deliveryConfigs.push({ type: provider, webhookUrl });
+  }
+
+  const config: Config = {
+    llm: {
+      provider: response.llmProvider,
+      apiKey: response.apiKey,
+      model: response.model,
+      baseUrl: response.baseUrl,
+    },
+    delivery: deliveryConfigs,
+    preferences: {
+      signalThreshold: 60,
+      maxItemsPerCategory: 5,
+      sourceCooldownMinutes: 5,
+      customKeywords: {},
+    },
+    dbPath: path.join(configDir, 'alif.db'),
+    feedsPath: path.join(configDir, 'feeds.json'),
+  };
+
+  configManager.save(config);
+
+  console.log(`\nConfiguration saved to ${configManager.getConfigFile()}`);
+  console.log(`Database will be located at ${config.dbPath}`);
+  console.log('\nAlif is ready! Run "alif run" to start (after adding feeds).');
+}

package/src/cli/commands/run.ts

@@ -0,0 +1,52 @@
+import fs from 'fs';
+import { ConfigManager } from '../../core/config-manager.js';
+import { createDatabase } from '../../db/connection.js';
+import { runMigrations } from '../../db/migrate.js';
+import { Pipeline } from '../../core/pipeline.js';
+
+export async function runPipeline(config: any, db: any, force = false) {
+  const pipeline = new Pipeline(config, db);
+
+  // Load feeds
+  if (!fs.existsSync(config.feedsPath)) {
+    console.log(
+      `[Alif] Feeds file not found at ${config.feedsPath}. Initializing with default sources...`,
+    );
+    const { defaultFeeds } = await import('../../resources/index.js');
+    fs.writeFileSync(config.feedsPath, JSON.stringify(defaultFeeds, null, 2));
+    console.log(
+      `[Alif] Created default feeds.json at ${config.feedsPath} with ${defaultFeeds.length} sources.`,
+    );
+  }
+
+  const feeds = JSON.parse(fs.readFileSync(config.feedsPath, 'utf-8'));
+  await pipeline.run(feeds, force);
+}
+
+export async function runCommand(options: { force?: boolean } = {}) {
+  const configManager = ConfigManager.getInstance();
+
+  if (!configManager.exists()) {
+    console.error('Alif is not initialized. Run "alif init" first.');
+    process.exit(1);
+  }
+
+  let db;
+  try {
+    const config = configManager.load();
+    db = createDatabase(config.dbPath);
+    runMigrations(db);
+
+    await runPipeline(config, db, options.force);
+  } catch (error) {
+    if (error instanceof Error) {
+      console.error(`Error: ${error.message}`);
+    } else {
+      console.error('An unknown error occurred.');
+    }
+    process.exit(1);
+  } finally {
+    if (db) db.close();
+    process.exit(0);
+  }
+}

package/src/cli/commands/schedule.ts

@@ -0,0 +1,99 @@
+import prompts from 'prompts';
+import { ConfigManager } from '../../core/config-manager.js';
+import { createDatabase } from '../../db/connection.js';
+import { runMigrations } from '../../db/migrate.js';
+import { Scheduler } from '../../core/scheduler.js';
+import { runPipeline } from './run.js';
+
+export async function scheduleCommand(action: string) {
+  const configManager = ConfigManager.getInstance();
+  if (!configManager.exists()) {
+    console.error('Alif is not initialized. Run "alif init" first.');
+    return;
+  }
+
+  const config = configManager.load();
+  const db = createDatabase(config.dbPath);
+
+  try {
+    runMigrations(db);
+    const scheduler = new Scheduler(db);
+
+    if (action === 'add') {
+      const response = await prompts([
+        {
+          type: 'text',
+          name: 'name',
+          message: 'Name for this schedule:',
+          initial: 'Daily Digest',
+        },
+        {
+          type: 'text',
+          name: 'cron',
+          message: 'Enter frequency (e.g. daily, hourly):',
+          initial: 'daily',
+        },
+        {
+          type: (prev) => (prev === 'daily' ? 'text' : null),
+          name: 'time',
+          message: 'At what time? (HH:mm format, 24h):',
+          initial: '09:00',
+          validate: (val) =>
+            /^([0-1]?[0-9]|2[0-3]):[0-5][0-9]$/.test(val) ? true : 'Please enter valid HH:mm time',
+        },
+      ]);
+
+      if (response.name && response.cron) {
+        const id = await scheduler.add(response.name, response.cron, response.time);
+        console.log(
+          `Schedule added! ID: ${id} (Runs ${response.cron}${response.time ? ` at ${response.time}` : ''})`,
+        );
+      }
+    } else if (action === 'list') {
+      const schedules = scheduler.list();
+      if (schedules.length === 0) {
+        console.log('No schedules found.');
+      } else {
+        console.table(
+          schedules.map((s) => ({
+            ID: s.id,
+            Name: s.name,
+            Frequency: s.cron,
+            Time: s.scheduled_time || '-',
+            Active: s.active ? 'Yes' : 'No',
+            'Last Run': s.last_run || 'Never',
+          })),
+        );
+      }
+    } else if (action === 'delete') {
+      const schedules = scheduler.list();
+      if (schedules.length === 0) {
+        console.log('No schedules to delete.');
+        return;
+      }
+
+      const { id } = await prompts({
+        type: 'select',
+        name: 'id',
+        message: 'Select schedule to delete:',
+        choices: schedules.map((s) => ({ title: s.name, value: s.id })),
+      });
+
+      if (id) {
+        scheduler.remove(id);
+        console.log(`Schedule ${id} deleted.`);
+      }
+    } else if (action === 'check') {
+      console.log('[Scheduler] Checking for due tasks...');
+      await scheduler.checkAndRun(async () => {
+        await runPipeline(config, db);
+      });
+      console.log('[Scheduler] Check complete.');
+    } else {
+      console.error(`Unknown action: ${action}. Available: add, list, delete, check`);
+    }
+  } finally {
+    db.close();
+    process.exit(0);
+  }
+}

package/src/cli/index.ts

@@ -0,0 +1,34 @@
+#!/usr/bin/env node
+import { Command } from 'commander';
+import { initCommand } from './commands/init.js';
+import { runCommand } from './commands/run.js';
+import { scheduleCommand } from './commands/schedule.js';
+
+const program = new Command();
+
+program.name('alif').description('Alif - Daily AI Signal Digest CLI').version('1.0.0');
+
+program
+  .command('init')
+  .description('Initialize Alif configuration')
+  .action(async () => {
+    await initCommand();
+  });
+
+program
+  .command('run')
+  .description('Run the AI Signal Digest pipeline')
+  .option('-f, --force', 'Bypass source cooldown')
+  .action(async (options) => {
+    await runCommand(options);
+  });
+
+program
+  .command('schedule <action>')
+  .description('Manage digest schedules')
+  .addHelpText('after', '\nActions: add, list, delete, check')
+  .action(async (action) => {
+    await scheduleCommand(action);
+  });
+
+program.parse(process.argv);

package/src/core/config-manager.ts

@@ -0,0 +1,72 @@
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+import { Config, ConfigSchema } from './config-schema.js';
+
+export class ConfigManager {
+  private static instance: ConfigManager;
+  private config: Config | null = null;
+  private configDir: string;
+  private configFile: string;
+
+  private constructor() {
+    this.configDir = path.join(os.homedir(), '.config', 'alif');
+    this.configFile = path.join(this.configDir, 'config.json');
+  }
+
+  static getInstance(): ConfigManager {
+    if (!ConfigManager.instance) {
+      ConfigManager.instance = new ConfigManager();
+    }
+    return ConfigManager.instance;
+  }
+
+  getConfigDir(): string {
+    return this.configDir;
+  }
+
+  getConfigFile(): string {
+    return this.configFile;
+  }
+
+  load(): Config {
+    if (this.config) return this.config;
+
+    if (!fs.existsSync(this.configFile)) {
+      throw new Error(`Configuration file not found at ${this.configFile}. Run 'alif init' first.`);
+    }
+
+    try {
+      const raw = fs.readFileSync(this.configFile, 'utf-8');
+      const parsed = JSON.parse(raw);
+      this.config = ConfigSchema.parse(parsed);
+      return this.config;
+    } catch (error) {
+      if (error instanceof Error) {
+        throw new Error(`Failed to load configuration: ${error.message}`, { cause: error });
+      }
+      throw error;
+    }
+  }
+
+  save(config: Config): void {
+    if (!fs.existsSync(this.configDir)) {
+      fs.mkdirSync(this.configDir, { recursive: true });
+    }
+
+    try {
+      ConfigSchema.parse(config);
+      fs.writeFileSync(this.configFile, JSON.stringify(config, null, 2), 'utf-8');
+      this.config = config;
+    } catch (error) {
+      if (error instanceof Error) {
+        throw new Error(`Failed to save configuration: ${error.message}`, { cause: error });
+      }
+      throw error;
+    }
+  }
+
+  exists(): boolean {
+    return fs.existsSync(this.configFile);
+  }
+}

package/src/core/config-schema.ts

@@ -0,0 +1,31 @@
+import { z } from 'zod';
+
+export const LlmProviderType = z.enum(['ollama', 'anthropic', 'openrouter']);
+export const DeliveryProviderType = z.enum(['slack', 'webhook']);
+
+export const ConfigSchema = z.object({
+  llm: z.object({
+    provider: LlmProviderType,
+    apiKey: z.string().optional(),
+    model: z.string(),
+    baseUrl: z.string().url().optional(),
+  }),
+  delivery: z.array(
+    z.object({
+      type: DeliveryProviderType,
+      webhookUrl: z.string().url(),
+    }),
+  ),
+  preferences: z.object({
+    signalThreshold: z.number().min(0).max(100).default(60),
+    maxItemsPerCategory: z.number().min(1).default(5),
+    sourceCooldownMinutes: z.number().min(0).default(5),
+    customKeywords: z.record(z.string(), z.number()).default({}),
+  }),
+  dbPath: z.string(),
+  feedsPath: z.string(),
+});
+
+export type Config = z.infer<typeof ConfigSchema>;
+export type LlmProvider = z.infer<typeof LlmProviderType>;
+export type DeliveryProvider = z.infer<typeof DeliveryProviderType>;

package/src/core/default-keywords.ts

@@ -0,0 +1,9 @@
+export const BASE_KEYWORDS: Record<string, number> = {
+  breakthrough: 20,
+  'gpt-5': 30,
+  o1: 20,
+  deepseek: 25,
+  'open source': 15,
+  agi: 15,
+  agent: 15,
+};

package/src/core/filters/deduplicator.ts

@@ -0,0 +1,39 @@
+import { ScrapedArticle } from '../scraper-types.js';
+
+export interface DeduplicatorOptions {
+  similarityThreshold?: number;
+}
+
+export class Deduplicator {
+  constructor(private options: DeduplicatorOptions = {}) {}
+
+  process(articles: ScrapedArticle[]): ScrapedArticle[] {
+    const unique = new Map<string, ScrapedArticle>();
+
+    for (const article of articles) {
+      // Primary key: URL (simplified)
+      const urlKey = this.normalizeUrl(article.url);
+
+      if (!unique.has(urlKey)) {
+        unique.set(urlKey, article);
+      } else {
+        // If we have an existing one, keep the one with content if possible
+        const existing = unique.get(urlKey)!;
+        if (!existing.content && article.content) {
+          unique.set(urlKey, article);
+        }
+      }
+    }
+
+    return Array.from(unique.values());
+  }
+
+  private normalizeUrl(url: string): string {
+    try {
+      const u = new URL(url);
+      return u.origin + u.pathname.replace(/\/$/, '');
+    } catch {
+      return url;
+    }
+  }
+}

package/src/core/filters/keywords.ts

@@ -0,0 +1,18 @@
+import { ScrapedArticle } from '../scraper-types.js';
+
+export class KeywordScorer {
+  constructor(private keywords: Record<string, number>) {}
+
+  score(article: ScrapedArticle): number {
+    let score = 0;
+    const text = `${article.title} ${article.content || ''}`.toLowerCase();
+
+    for (const [keyword, weight] of Object.entries(this.keywords)) {
+      if (text.includes(keyword.toLowerCase())) {
+        score += weight;
+      }
+    }
+
+    return score;
+  }
+}

package/src/core/orchestrator.ts

@@ -0,0 +1,47 @@
+import { ScraperSource, ScraperResult, BaseScraper } from './scraper-types.js';
+import { RssScraper } from './scrapers/rss-scraper.js';
+import { ArxivScraper } from './scrapers/arxiv-scraper.js';
+import { ApiScraper } from './scrapers/api-scraper.js';
+import { ScrapeScraper } from './scrapers/scrape-scraper.js';
+import { JsonScraper } from './scrapers/json-scraper.js';
+
+export class ScraperOrchestrator {
+  private scrapers: Record<string, BaseScraper> = {};
+
+  constructor() {
+    this.scrapers.rss = new RssScraper();
+    this.scrapers.arxiv = new ArxivScraper();
+    this.scrapers.api = new ApiScraper();
+    this.scrapers.scrape = new ScrapeScraper();
+    this.scrapers.json = new JsonScraper();
+  }
+
+  async runAll(sources: ScraperSource[]): Promise<ScraperResult[]> {
+    const tasks = sources.map((source) => {
+      let scraper: BaseScraper | undefined;
+
+      // Priority 1: Match by ID for specialized logic in reference
+      if (source.id === 'hn') scraper = this.scrapers.api;
+      else if (source.id === 'github_trending') scraper = this.scrapers.scrape;
+      else if (source.id.startsWith('arxiv')) scraper = this.scrapers.arxiv;
+      else if (source.id.startsWith('reddit')) scraper = this.scrapers.json;
+
+      // Priority 2: Use explicitly defined type
+      if (!scraper) {
+        scraper = this.scrapers[source.type];
+      }
+
+      if (!scraper) {
+        return Promise.resolve({
+          source: source.id,
+          status: 'error',
+          items: [],
+          error: `No scraper found for type: ${source.type}`,
+        } as ScraperResult);
+      }
+      return scraper.scrape(source);
+    });
+
+    return Promise.all(tasks);
+  }
+}