domain-rank 0.1.1

package/src/export.ts

@@ -0,0 +1,98 @@
+import fs from 'fs/promises';
+import { createWriteStream } from 'fs';
+import path from 'path';
+
+type Format = 'json' | 'csv' | 'ndjson';
+
+const defaultFormats: Format[] = ['json', 'csv', 'ndjson'];
+
+async function loadData() {
+  const base = path.resolve(__dirname, '../data');
+  const mergedPath = path.join(base, 'domain-rank-merged.json');
+  const infoPath = path.join(base, 'domain-info.json');
+  const merged = JSON.parse(await fs.readFile(mergedPath, 'utf8'));
+  let info = {};
+  try {
+    info = JSON.parse(await fs.readFile(infoPath, 'utf8'));
+  } catch (e) {
+    // ignore
+  }
+  return { merged, info };
+}
+
+function toCSV(records: any[]) {
+  if (!records.length) return '';
+  const keys = Object.keys(records[0]);
+  const lines = [keys.join(',')];
+  for (const r of records) {
+    lines.push(keys.map(k => {
+      const v = r[k] ?? '';
+      const s = typeof v === 'string' ? v : JSON.stringify(v);
+      return '"' + s.replace(/"/g, '""') + '"';
+    }).join(','));
+  }
+  return lines.join('\n');
+}
+
+async function writeJSON(outDir: string, name: string, data: any) {
+  await fs.mkdir(outDir, { recursive: true });
+  await fs.writeFile(path.join(outDir, name + '.json'), JSON.stringify(data, null, 2), 'utf8');
+}
+
+async function writeNDJSON(outDir: string, name: string, arr: any[]) {
+  await fs.mkdir(outDir, { recursive: true });
+  const fp = path.join(outDir, name + '.ndjson');
+  const stream = createWriteStream(fp, { encoding: 'utf8' });
+  for (const item of arr) {
+    stream.write(JSON.stringify(item) + '\n');
+  }
+  stream.end();
+  await new Promise(resolve => stream.on('finish', resolve));
+}
+
+async function writeCSVFile(outDir: string, name: string, arr: any[]) {
+  await fs.mkdir(outDir, { recursive: true });
+  const csv = toCSV(arr);
+  await fs.writeFile(path.join(outDir, name + '.csv'), csv, 'utf8');
+}
+
+function makeRecords(merged: any[], info: Record<string, any>) {
+  return merged.map((m: any) => ({
+    rank: m.rank ?? null,
+    domain: m.domain ?? m.name ?? null,
+    source: m.source ?? null,
+    score: m.score ?? null,
+    info: info[m.domain] ?? null
+  }));
+}
+
+async function main() {
+  const args = process.argv.slice(2);
+  const formatsArg = args.find(a => a.startsWith('--formats='))?.split('=')[1];
+  const outArg = args.find(a => a.startsWith('--out='))?.split('=')[1];
+  const formats = formatsArg ? formatsArg.split(',') as Format[] : defaultFormats;
+  const outDir = outArg ? path.resolve(process.cwd(), outArg) : path.resolve(process.cwd(), 'exports');
+
+  const { merged, info } = await loadData();
+  const records = makeRecords(Array.isArray(merged) ? merged : Object.values(merged), info || {});
+
+  if (formats.includes('json')) {
+    await writeJSON(outDir, 'domain-rank', records);
+    console.log('Wrote JSON to', outDir);
+  }
+  if (formats.includes('ndjson')) {
+    await writeNDJSON(outDir, 'domain-rank', records);
+    console.log('Wrote NDJSON to', outDir);
+  }
+  if (formats.includes('csv')) {
+    await writeCSVFile(outDir, 'domain-rank', records);
+    console.log('Wrote CSV to', outDir);
+  }
+}
+
+if (require.main === module) {
+  main().catch(err => {
+    console.error(err);
+    process.exit(1);
+  });
+}

package/src/favicons.js

@@ -0,0 +1,213 @@
+import { Database } from "bun:sqlite";
+import domains from "../data/domains-1m.js";
+
+const db = new Database("favicons.sqlite");
+
+// Create table if it doesn't exist
+db.run(`
+  CREATE TABLE IF NOT EXISTS favicons (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    domain TEXT UNIQUE,
+    favicon BLOB,
+    size INTEGER,
+    created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+  )
+`);
+
+/**
+ * Get favicon for a single domain using Google's favicon API
+ * @param {string} domain - Domain name to fetch favicon for
+ * @param {number} [size=16] - Size of the favicon in pixels
+ * @returns {Promise<Buffer|null>} Buffer containing favicon data or null if failed
+ */
+async function getFaviconForSingleDomain(domain, size = 16) {
+  try {
+    // Create AbortController for timeout
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 10000); // 10 second timeout
+
+    const response = await fetch(
+      'https://www.google.com/s2/favicons?domain=' + encodeURIComponent(domain) + '&sz=' + size,
+      {
+        signal: controller.signal,
+        headers: {
+          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
+        }
+      }
+    );
+
+    clearTimeout(timeoutId);
+
+    if (!response.ok) {
+      return null;
+    }
+
+    const arrayBuffer = await response.arrayBuffer();
+    const buffer = Buffer.from(arrayBuffer);
+    
+    // Check if we got a valid image (not the default Google favicon)
+    if (buffer.length < 100) {
+      return null; // Too small, likely a default favicon
+    }
+
+    return buffer;
+  } catch (error) {
+    console.log(`Could not fetch favicon for ${domain}: ${error.message}`);
+    return null;
+  }
+}
+
+/**
+ * Fetch favicons and store them in SQLite database
+ * @param {object} options - Options for fetching favicons
+ * @param {number} options.startIndex - Start index in domains list (default: 0)
+ * @param {number} options.endIndex - End index in domains list (default: 1000)
+ * @param {number} options.size - Size of favicons in pixels (default: 16)
+ * @param {number} options.delay - Delay between requests in ms (default: 200)
+ * @returns {Promise<void>} Promise that resolves when all favicons are fetched and stored
+ */
+async function fetchAndStoreFavicons(options = {}) {
+  const {
+    startIndex = 0,
+    endIndex = 1000,
+    size = 16,
+    delay = 200,
+  } = options;
+
+  const domainsArray = domains.split(",");
+  const actualEndIndex = Math.min(endIndex, domainsArray.length);
+
+  // Prepare statements for better performance
+  const insertStmt = db.prepare(
+    "INSERT OR REPLACE INTO favicons (domain, favicon, size) VALUES (?, ?, ?)"
+  );
+  const existsStmt = db.prepare(
+    "SELECT id FROM favicons WHERE domain = ? AND size = ?"
+  );
+
+  // Get current count
+  const initialCount = db.query("SELECT COUNT(*) as count FROM favicons").get().count;
+  console.log(`Starting with ${initialCount} favicons in database`);
+  console.log(`Fetching favicons for domains ${startIndex} to ${actualEndIndex - 1}`);
+  console.log(`Target: ${actualEndIndex - startIndex} domains`);
+
+  let successCount = 0;
+  let skipCount = 0;
+  let failCount = 0;
+
+  for (let i = startIndex; i < actualEndIndex; i++) {
+    const domain = domainsArray[i];
+    
+    // Check if already exists with same size
+    const exists = existsStmt.get(domain, size);
+    if (exists) {
+      console.log(`Skipping ${i + 1}: ${domain} (already exists)`);
+      skipCount++;
+      continue;
+    }
+
+    console.log(`Fetching ${i + 1}: ${domain}`);
+    
+    const faviconBuffer = await getFaviconForSingleDomain(domain, size);
+    
+    if (faviconBuffer) {
+      insertStmt.run(domain, faviconBuffer, size);
+      successCount++;
+      console.log(`✓ Stored favicon for ${domain} (${faviconBuffer.length} bytes)`);
+      
+      // Log progress every 20 successful downloads
+      if (successCount % 20 === 0) {
+        const currentCount = db.query("SELECT COUNT(*) as count FROM favicons").get().count;
+        console.log(`Progress: ${successCount} new favicons downloaded, ${currentCount} total in database`);
+      }
+    } else {
+      failCount++;
+      console.log(`✗ Failed to download favicon for ${domain}`);
+    }
+
+    // Add delay to avoid overwhelming Google's API
+    if (delay > 0) {
+      await new Promise(resolve => setTimeout(resolve, delay));
+    }
+  }
+
+  const finalCount = db.query("SELECT COUNT(*) as count FROM favicons").get().count;
+  
+  console.log(`\nCompleted fetching favicons!`);
+  console.log(`Total favicons in database: ${finalCount}`);
+  console.log(`New favicons: ${successCount}`);
+  console.log(`Skipped: ${skipCount}`);
+  console.log(`Failed: ${failCount}`);
+}
+
+/**
+ * Get favicon from database by domain
+ * @param {string} domain - Domain to get favicon for
+ * @param {number} [size=16] - Size of favicon to retrieve
+ * @returns {Buffer|null} Favicon buffer or null if not found
+ */
+function getFaviconFromDatabase(domain, size = 16) {
+  const stmt = db.prepare("SELECT favicon FROM favicons WHERE domain = ? AND size = ?");
+  const result = stmt.get(domain, size);
+  return result ? result.favicon : null;
+}
+
+/**
+ * Get all domains that have favicons stored
+ * @param {number} [size=16] - Size filter
+ * @returns {string[]} Array of domain names
+ */
+function getDomainsWithFavicons(size = 16) {
+  const stmt = db.prepare("SELECT domain FROM favicons WHERE size = ? ORDER BY domain");
+  return stmt.all(size).map(row => row.domain);
+}
+
+/**
+ * Get database statistics
+ * @returns {object} Statistics about the favicon database
+ */
+function getDatabaseStats() {
+  const totalCount = db.query("SELECT COUNT(*) as count FROM favicons").get().count;
+  const sizeStats = db.query(`
+    SELECT size, COUNT(*) as count 
+    FROM favicons 
+    GROUP BY size 
+    ORDER BY size
+  `).all();
+  
+  const totalSize = db.query(`
+    SELECT SUM(LENGTH(favicon)) as totalBytes 
+    FROM favicons
+  `).get().totalBytes || 0;
+
+  return {
+    totalFavicons: totalCount,
+    totalSizeBytes: totalSize,
+    totalSizeMB: (totalSize / 1024 / 1024).toFixed(2),
+    sizeBreakdown: sizeStats
+  };
+}
+
+// Export functions for use as a module
+export {
+  fetchAndStoreFavicons,
+  getFaviconFromDatabase,
+  getDomainsWithFavicons,
+  getDatabaseStats,
+  getFaviconForSingleDomain
+};
+
+// Close database connection gracefully on exit
+process.on('SIGINT', () => {
+  console.log('\nClosing database connection...');
+  db.close();
+  process.exit(0);
+});
+
+// Run for top 1000 domains if this file is executed directly
+if (import.meta.main) {
+  console.log('Starting favicon fetcher...');
+  await fetchAndStoreFavicons({ startIndex: 0, endIndex: 1000, delay: 200 });
+  console.log('\nDatabase stats:');
+  console.log(getDatabaseStats());
+}

package/src/import-domains-1m.js

@@ -0,0 +1,170 @@
+import fs from "fs";
+import zlib from "zlib";
+import readline from "readline";
+import { writeFile } from 'fs/promises';
+import { pipeline } from 'stream/promises';
+import { Transform, Readable } from 'stream';
+import unzipper from 'unzipper';
+import grab from "grab-url";
+
+
+
+
+/**
+ * Domain Rank shows how trustworthy and influential a domain is based on links pointing to that
+ *  domain's pages across all 120+ million domains.
+ *
+ * @see [CommonCrawl](https://commoncrawl.org/web-graphs)
+ * CommonCrawl is a nonprofit for open source public dataset that crawls and downloads the entire
+ *  internet 100TB urls and html. CommonCrawl calculates domain rank for 100M domains, using
+ *  PageRank algorithm which randomly surfs links and counts travels to each page to find 
+ * probability of being at a domain, thus ranking influence among other reputable domains.
+ */
+export async function importDomainsPageRankCrawler(urlCommonCrawl = '', limit = 1000000) {
+  const url = urlCommonCrawl || await getDomainCrawlerUrl();
+
+  try {
+    if (!fs.existsSync("./data")) fs.mkdirSync("./data", { recursive: true });
+    try { fs.unlinkSync("./data/domains-1m.js"); } catch (_) {}
+
+    const writeStream = fs.createWriteStream("./data/domains-1m.js", { flags: "w" });
+    let lineNum = 0;
+    let headerProcessed = false;
+    writeStream.write("export default '");
+
+    const ticker = setInterval(() => {
+      console.log(`Processed lines: ${lineNum.toLocaleString()}`);
+      if (lineNum >= limit) clearInterval(ticker);
+    }, 5000);
+
+    await grab(url, {
+      onStream: async (body) => {
+        const decompressedStream = Readable.fromWeb(body).pipe(zlib.createGunzip());
+        const rl = readline.createInterface({ input: decompressedStream, crlfDelay: Infinity });
+        let streamClosed = false;
+
+        function closeStream() {
+          if (!streamClosed) {
+            streamClosed = true;
+            writeStream.write('\b');
+            writeStream.write("'");
+            writeStream.end();
+            clearInterval(ticker);
+            console.log(`Output file: ./data/domains-1m.js`);
+          }
+        }
+
+        rl.on('line', (line) => {
+          try {
+            if (streamClosed) return;
+            if (!headerProcessed) {
+              if (line.includes("#host_rev")) { headerProcessed = true; return; }
+            }
+            const parts = line.split("\t");
+            if (parts.length < 5) return;
+            const domain = parts[4];
+            if (!domain || domain === "#host_rev") return;
+            writeStream.write(`${domain.split(".").reverse().join(".")},`);
+            lineNum++;
+            if (lineNum >= limit) { rl.close(); closeStream(); }
+          } catch (error) {
+            console.error("Error processing line:", error);
+          }
+        });
+
+        await new Promise((resolve, reject) => {
+          rl.on('close', () => { closeStream(); resolve(); });
+          rl.on('error', (err) => { clearInterval(ticker); reject(err); });
+        });
+      }
+    });
+
+  } catch (error) {
+    console.error("Error:", error);
+  }
+}
+
+
+/**
+ * Scrapes Common Crawl web graphs page to find the domain-ranks.txt.gz URL
+ * from the first available date listing.
+ * 
+ * @returns {Promise<string>} The full URL to the domain-ranks.txt.gz file
+ * @throws {Error} When HTTP requests fail, date links are not found, or domain-ranks.txt.gz is not found
+ */
+export async function getDomainCrawlerUrl() {
+  const mainHtml = await grab('https://commoncrawl.org/web-graphs', { timeout: 10 });
+
+  const datePattern = /href="([^"]*\d{4}-\w+[^"]*)"/;
+  const dateMatch = mainHtml.match(datePattern);
+  if (!dateMatch) throw new Error('No date link found');
+
+  const dateUrl = dateMatch[1].startsWith('http')
+    ? dateMatch[1]
+    : `https://commoncrawl.org${dateMatch[1]}`;
+
+  const dateHtml = await grab(dateUrl, { timeout: 10 });
+
+  const rankPattern = /href="([^"]*domain-ranks\.txt\.gz[^"]*)"/;
+  const rankMatch = dateHtml.match(rankPattern);
+  if (!rankMatch) throw new Error('domain-ranks.txt.gz not found');
+
+  return rankMatch[1].startsWith('http')
+    ? rankMatch[1]
+    : `https://commoncrawl.org${rankMatch[1]}`;
+}
+
+
+
+/**
+ * Download and extract the current Tranco top-1M domain ranking.
+ *
+ * The Tranco project aggregates multiple ranking providers (Cisco Umbrella,
+ * Majestic, Farsight, Chrome UX Report, Cloudflare Radar) to generate
+ * manipulation-resistant popularity lists. The list is updated daily (UTC).
+ *
+ * Source: https://tranco-list.eu/
+ * Default dataset: https://tranco-list.eu/top-1m.csv.zip
+ */
+
+async function importDomainsOfficialList(limit = 10000) {
+  const url = 'https://tranco-list.eu/top-1m.csv.zip';
+  const output = './data/domains-official-100k.js';
+
+  console.log(`Streaming download and extraction (limit: ${limit.toLocaleString()})...`);
+
+  const domains = [];
+  let isFirstLine = true;
+  let done = false;
+
+  await grab(url, {
+    onStream: (body) => pipeline(
+      Readable.fromWeb(body),
+      unzipper.ParseOne(),
+      new Transform({
+        objectMode: false,
+        transform(chunk, encoding, callback) {
+          if (done) return callback();
+          for (const line of chunk.toString().split('\n')) {
+            if (isFirstLine) { isFirstLine = false; continue; }
+            const domain = line.split(',')[1]?.replace(/"/g, '').trim();
+            if (domain) domains.push(domain);
+            if (domains.length >= limit) { done = true; break; }
+          }
+          callback();
+        }
+      })
+    )
+  });
+
+  await writeFile(output, `export default '${domains.join(',')}';`);
+  console.log(`Saved ${domains.length} domains to ${output}`);
+}
+
+
+
+//if run directly
+if (import.meta.main) {
+  // importDomainsPageRankCrawler();
+  importDomainsOfficialList(10000);
+}

package/src/merge-domain-lists.ts

@@ -0,0 +1,109 @@
+import fs from "fs";
+
+type OldDomainEntry = [number, string]; // [rank, title] (existing source format)
+
+type OutputEntry = {
+  name?: string; // preferred display name (news title overrides)
+  domainRank?: number;
+  domainTitle?: string;
+  newsRank?: number;
+  newsTitle?: string;
+  langCode?: string;
+};
+
+type DomainMap = Record<string, OutputEntry>;
+
+/**
+ * Merge domain-info.json (general 1M list) with news-domain-rank.json
+ * (curated news sources). News entries override general entries for the same
+ * domain; news-only domains are appended after the general list.
+ *
+ * Ranks are reassigned sequentially in the merged output.
+ */
+export function mergeDomainLists(options: {
+  domainInfoPath?: string;
+  newsDomainRankPath?: string;
+  outputPath?: string;
+} = {}): DomainMap {
+  const {
+    domainInfoPath = "./data/domain-info.json",
+    newsDomainRankPath = "./data/news-domain-rank.json",
+    outputPath = "./data/domain-rank-merged.json",
+  } = options;
+
+  const domainInfoRaw: Record<string, OldDomainEntry> = fs.existsSync(domainInfoPath)
+    ? JSON.parse(fs.readFileSync(domainInfoPath, "utf8"))
+    : {};
+
+  const newsDomainRaw: Record<string, OldDomainEntry> = fs.existsSync(newsDomainRankPath)
+    ? JSON.parse(fs.readFileSync(newsDomainRankPath, "utf8"))
+    : {};
+
+  // Build merged map preserving both domain and news information
+  const merged: DomainMap = {};
+
+  // copy general list first
+  for (const [domain, entry] of Object.entries(domainInfoRaw)) {
+    const [rank, title] = entry || [undefined, undefined];
+    const newsEntry = newsDomainRaw[domain];
+    const newsRank = newsEntry ? newsEntry[0] : undefined;
+    const newsTitle = newsEntry ? newsEntry[1] : undefined;
+    const name = newsTitle || title || domain;
+    merged[domain] = {
+      name,
+      domainRank: typeof rank === 'number' ? rank : undefined,
+      domainTitle: title || undefined,
+      newsRank: typeof newsRank === 'number' ? newsRank : undefined,
+      newsTitle: newsTitle || undefined,
+      langCode: undefined,
+    };
+  }
+
+  // Determine next rank for news-only domains
+  const maxDomainRank = Object.values(merged).reduce((max, e) => Math.max(max, e.domainRank || 0), 0);
+  let nextRank = maxDomainRank + 1;
+
+  // append news-only domains
+  for (const [domain, entry] of Object.entries(newsDomainRaw)) {
+    if (merged[domain]) continue;
+    const [newsRank, newsTitle] = entry || [undefined, undefined];
+    merged[domain] = {
+      name: newsTitle || domain,
+      domainRank: nextRank++,
+      newsRank: typeof newsRank === 'number' ? newsRank : undefined,
+      newsTitle: newsTitle || undefined,
+      langCode: undefined,
+    };
+  }
+
+  // Prepare serializable output as fixed-order 6-element arrays:
+  // [name, domainRank, domainTitle, newsRank, newsTitle, langCode]
+  // Use empty strings (for text) and 0 (for ranks) for missing values.
+  const outputObj: Record<string, Array<string | number>> = {};
+  for (const [domain, entry] of Object.entries(merged)) {
+    // Avoid repeating the full name: if domainTitle or newsTitle exactly
+    // matches the chosen display name, store an empty string in that position.
+    const domainTitleField = entry.domainTitle !== undefined && entry.domainTitle !== entry.name ? entry.domainTitle : "";
+    const newsTitleField = entry.newsTitle !== undefined && entry.newsTitle !== entry.name ? entry.newsTitle : "";
+
+    const arr: Array<string | number> = [
+      entry.name !== undefined ? entry.name : "",
+      entry.domainRank !== undefined ? entry.domainRank : 0,
+      domainTitleField,
+      entry.newsRank !== undefined ? entry.newsRank : 0,
+      newsTitleField,
+      entry.langCode !== undefined ? entry.langCode : "",
+    ];
+    outputObj[domain] = arr;
+  }
+
+  fs.writeFileSync(outputPath, JSON.stringify(outputObj), "utf8");
+  console.log(
+    `Merged ${Object.keys(domainInfoRaw).length} general + ${Object.keys(newsDomainRaw).length} news entries → ${Object.keys(merged).length} total → ${outputPath}`
+  );
+
+  return merged;
+}
+
+// Run when called directly: bun src/merge-domain-lists.ts
+mergeDomainLists();

package/src/parse-domain-info.ts

@@ -0,0 +1,99 @@
+import domainsOfficial from "../data/domains-official-1m.js";
+import fs from "fs";
+import {
+  shouldRemoveDomain,
+  findMainDomain,
+  getTitleOverride,
+  formatDomainAsTitle,
+  cleanSourceTitle,
+  getSourceTitle,
+} from "./domain-name-formatter.js";
+
+type DomainMap = Record<string, [number, string]>;
+
+interface DomainInfoOptions {
+  startIndex?: number;
+  endIndex?: number;
+  resume?: boolean;
+}
+
+export async function domainInfo(options: DomainInfoOptions = {}): Promise<void> {
+  const { startIndex = 0, endIndex = 1000, resume = false } = options;
+
+  const domainsArray = domainsOfficial.split(",");
+  const actualEndIndex = Math.min(endIndex, domainsArray.length);
+  const dataPath = "./data/domain-info.json";
+
+  let domainResults: DomainMap = {};
+  let resumeFromIndex = startIndex;
+
+  if (fs.existsSync(dataPath)) {
+    domainResults = JSON.parse(fs.readFileSync(dataPath, "utf8"));
+
+    if (resume) {
+      const existing = Object.keys(domainResults);
+      if (existing.length > 0) {
+        const lastDomain = existing[existing.length - 1];
+        const lastIndex = domainsArray.indexOf(lastDomain);
+        if (lastIndex !== -1) {
+          resumeFromIndex = lastIndex + 1;
+          console.log(`Resuming from domain: ${lastDomain} (index ${resumeFromIndex})`);
+        }
+      }
+    }
+  } else {
+    fs.writeFileSync(dataPath, "{}", "utf8");
+  }
+
+  if (!resume && startIndex === 0) {
+    domainResults = {};
+    fs.writeFileSync(dataPath, "{}", "utf8");
+  }
+
+  console.log(`Processing domains from index ${resumeFromIndex} to ${actualEndIndex - 1}`);
+
+  let actualRank = Object.keys(domainResults).length;
+
+  for (let i = resumeFromIndex; i < actualEndIndex; i++) {
+    const domain = domainsArray[i];
+
+    if (shouldRemoveDomain(domain)) {
+      console.log(`Skipping ${i + 1}: ${domain} (marked for removal)`);
+      continue;
+    }
+
+    const mainDomain = findMainDomain(domain);
+    if (mainDomain) {
+      console.log(`Skipping ${i + 1}: ${domain} (alternative for ${mainDomain})`);
+      continue;
+    }
+
+    actualRank++;
+    console.log(`Processing ${actualRank}: ${domain}`);
+
+    let source: string | null = getTitleOverride(domain);
+
+    if (!source) {
+      source = formatDomainAsTitle(domain);
+    }
+
+    if (!getTitleOverride(domain)) {
+      const webTitle = await getSourceTitle(domain);
+      if (webTitle) {
+        const cleaned = cleanSourceTitle(webTitle)
+          ?.replace(/homepage/gi, "")
+          .replace(/home/gi, "")
+          .replace(".com", "");
+        if (cleaned && cleaned.length > 0) {
+          const wordCount = cleaned.split(/\s+/).filter((w) => w.length > 0).length;
+          if (wordCount < 3) source = cleaned;
+        }
+      }
+    }
+
+    domainResults[domain] = [actualRank, source || domain];
+    fs.writeFileSync(dataPath, JSON.stringify(domainResults), "utf8");
+  }
+}
+
+domainInfo({ startIndex: 0, endIndex: 1000000, resume: true });

package/test/domain.test.js

@@ -0,0 +1,13 @@
+import {test, expect} from 'vitest';
+import {importTopDomains} from '../src/download-1m'
+
+test('import top domains', async () => {
+
+    var result = await importTopDomains();
+
+    await new Promise(resolve => setTimeout(resolve, 40000));
+
+    console.log(result);
+    expect(result).toBeDefined();
+
+}, 40000)