@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/site-search.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Site-Aware Search URL Builders
+ *
+ * Provides URL templates for popular websites so AI agents can search them
+ * without needing to know site-specific URL structures.
+ *
+ * @module site-search
+ */
+export interface SiteSearchResult {
+    url: string;
+    site: string;
+    query: string;
+}
+export interface SiteTemplate {
+    name: string;
+    searchUrl: (query: string) => string;
+    category: 'shopping' | 'social' | 'jobs' | 'general' | 'tech' | 'real-estate' | 'food';
+}
+/**
+ * URL templates for popular sites, keyed by site ID.
+ * All query values are URL-encoded via encodeURIComponent.
+ */
+export declare const SITE_TEMPLATES: Record<string, SiteTemplate>;
+/**
+ * Build a search URL for a given site and query.
+ *
+ * @param site  Site ID (e.g. "ebay", "amazon") or alias (e.g. "x")
+ * @param query Search query string
+ * @throws Error if the site is not recognized
+ */
+export declare function buildSiteSearchUrl(site: string, query: string): SiteSearchResult;
+/**
+ * List all available site templates in a flat array.
+ */
+export declare function listSites(): Array<{
+    id: string;
+    name: string;
+    category: string;
+}>;
+/**
+ * Find which site ID a given URL belongs to (reverse lookup).
+ * Returns the canonical site ID, or null if the URL doesn't match any template.
+ */
+export declare function findSiteByUrl(url: string): string | null;

package/dist/core/site-search.js

@@ -0,0 +1,252 @@
+/**
+ * Site-Aware Search URL Builders
+ *
+ * Provides URL templates for popular websites so AI agents can search them
+ * without needing to know site-specific URL structures.
+ *
+ * @module site-search
+ */
+/**
+ * URL templates for popular sites, keyed by site ID.
+ * All query values are URL-encoded via encodeURIComponent.
+ */
+export const SITE_TEMPLATES = {
+    // ── Shopping ──────────────────────────────────────────────────────────────
+    ebay: {
+        name: 'eBay',
+        category: 'shopping',
+        searchUrl: (q) => `https://www.ebay.com/sch/i.html?_nkw=${encodeURIComponent(q)}`,
+    },
+    amazon: {
+        name: 'Amazon',
+        category: 'shopping',
+        searchUrl: (q) => `https://www.amazon.com/s?k=${encodeURIComponent(q)}`,
+    },
+    walmart: {
+        name: 'Walmart',
+        category: 'shopping',
+        searchUrl: (q) => `https://www.walmart.com/search?q=${encodeURIComponent(q)}`,
+    },
+    target: {
+        name: 'Target',
+        category: 'shopping',
+        searchUrl: (q) => `https://www.target.com/s?searchTerm=${encodeURIComponent(q)}`,
+    },
+    bestbuy: {
+        name: 'Best Buy',
+        category: 'shopping',
+        searchUrl: (q) => `https://www.bestbuy.com/site/searchpage.jsp?st=${encodeURIComponent(q)}`,
+    },
+    etsy: {
+        name: 'Etsy',
+        category: 'shopping',
+        searchUrl: (q) => `https://www.etsy.com/search?q=${encodeURIComponent(q)}`,
+    },
+    aliexpress: {
+        name: 'AliExpress',
+        category: 'shopping',
+        searchUrl: (q) => `https://www.aliexpress.com/wholesale?SearchText=${encodeURIComponent(q)}`,
+    },
+    newegg: {
+        name: 'Newegg',
+        category: 'shopping',
+        searchUrl: (q) => `https://www.newegg.com/p/pl?d=${encodeURIComponent(q)}`,
+    },
+    // ── General ───────────────────────────────────────────────────────────────
+    google: {
+        name: 'Google',
+        category: 'general',
+        searchUrl: (q) => `https://www.google.com/search?q=${encodeURIComponent(q)}`,
+    },
+    bing: {
+        name: 'Bing',
+        category: 'general',
+        searchUrl: (q) => `https://www.bing.com/search?q=${encodeURIComponent(q)}`,
+    },
+    duckduckgo: {
+        name: 'DuckDuckGo',
+        category: 'general',
+        searchUrl: (q) => `https://html.duckduckgo.com/html/?q=${encodeURIComponent(q)}`,
+    },
+    // ── Social / Content ──────────────────────────────────────────────────────
+    reddit: {
+        name: 'Reddit',
+        category: 'social',
+        searchUrl: (q) => `https://www.reddit.com/search/?q=${encodeURIComponent(q)}`,
+    },
+    youtube: {
+        name: 'YouTube',
+        category: 'social',
+        searchUrl: (q) => `https://www.youtube.com/results?search_query=${encodeURIComponent(q)}`,
+    },
+    twitter: {
+        name: 'X (Twitter)',
+        category: 'social',
+        searchUrl: (q) => `https://x.com/search?q=${encodeURIComponent(q)}`,
+    },
+    linkedin: {
+        name: 'LinkedIn',
+        category: 'social',
+        searchUrl: (q) => `https://www.linkedin.com/search/results/all/?keywords=${encodeURIComponent(q)}`,
+    },
+    // ── Tech ──────────────────────────────────────────────────────────────────
+    github: {
+        name: 'GitHub',
+        category: 'tech',
+        searchUrl: (q) => `https://github.com/search?q=${encodeURIComponent(q)}`,
+    },
+    stackoverflow: {
+        name: 'Stack Overflow',
+        category: 'tech',
+        searchUrl: (q) => `https://stackoverflow.com/search?q=${encodeURIComponent(q)}`,
+    },
+    npm: {
+        name: 'npm',
+        category: 'tech',
+        searchUrl: (q) => `https://www.npmjs.com/search?q=${encodeURIComponent(q)}`,
+    },
+    pypi: {
+        name: 'PyPI',
+        category: 'tech',
+        searchUrl: (q) => `https://pypi.org/search/?q=${encodeURIComponent(q)}`,
+    },
+    // ── Real Estate ───────────────────────────────────────────────────────────
+    zillow: {
+        name: 'Zillow',
+        category: 'real-estate',
+        searchUrl: (q) => `https://www.zillow.com/homes/${encodeURIComponent(q)}_rb/`,
+    },
+    realtor: {
+        name: 'Realtor.com',
+        category: 'real-estate',
+        searchUrl: (q) => `https://www.realtor.com/realestateandhomes-search/${encodeURIComponent(q)}`,
+    },
+    // ── Jobs ──────────────────────────────────────────────────────────────────
+    indeed: {
+        name: 'Indeed',
+        category: 'jobs',
+        searchUrl: (q) => `https://www.indeed.com/jobs?q=${encodeURIComponent(q)}`,
+    },
+    glassdoor: {
+        name: 'Glassdoor',
+        category: 'jobs',
+        searchUrl: (q) => `https://www.glassdoor.com/Job/jobs.htm?sc.keyword=${encodeURIComponent(q)}`,
+    },
+    'linkedin-jobs': {
+        name: 'LinkedIn Jobs',
+        category: 'jobs',
+        searchUrl: (q) => `https://www.linkedin.com/jobs/search/?keywords=${encodeURIComponent(q)}`,
+    },
+    // ── Food ──────────────────────────────────────────────────────────────────
+    yelp: {
+        name: 'Yelp',
+        category: 'food',
+        searchUrl: (q) => `https://www.yelp.com/search?find_desc=${encodeURIComponent(q)}`,
+    },
+    doordash: {
+        name: 'DoorDash',
+        category: 'food',
+        searchUrl: (q) => `https://www.doordash.com/search/store/${encodeURIComponent(q)}`,
+    },
+    ubereats: {
+        name: 'Uber Eats',
+        category: 'food',
+        searchUrl: (q) => `https://www.ubereats.com/search?q=${encodeURIComponent(q)}`,
+    },
+};
+/** Aliases that map to canonical site IDs */
+const SITE_ALIASES = {
+    x: 'twitter',
+    'best-buy': 'bestbuy',
+    'ali-express': 'aliexpress',
+    'stack-overflow': 'stackoverflow',
+    'duck-duck-go': 'duckduckgo',
+};
+/**
+ * Resolve a site ID (or alias) to its canonical key.
+ * Returns null if not found.
+ */
+function resolveSiteId(site) {
+    const lower = site.toLowerCase();
+    if (lower in SITE_TEMPLATES)
+        return lower;
+    if (lower in SITE_ALIASES)
+        return SITE_ALIASES[lower];
+    return null;
+}
+/**
+ * Build a search URL for a given site and query.
+ *
+ * @param site  Site ID (e.g. "ebay", "amazon") or alias (e.g. "x")
+ * @param query Search query string
+ * @throws Error if the site is not recognized
+ */
+export function buildSiteSearchUrl(site, query) {
+    const canonical = resolveSiteId(site);
+    if (!canonical) {
+        const available = Object.keys(SITE_TEMPLATES).join(', ');
+        throw new Error(`Unknown site: "${site}". Available sites: ${available}. ` +
+            `Run "webpeel sites" to list all supported sites.`);
+    }
+    const template = SITE_TEMPLATES[canonical];
+    return {
+        url: template.searchUrl(query),
+        site: canonical,
+        query,
+    };
+}
+/**
+ * List all available site templates in a flat array.
+ */
+export function listSites() {
+    return Object.entries(SITE_TEMPLATES).map(([id, template]) => ({
+        id,
+        name: template.name,
+        category: template.category,
+    }));
+}
+/**
+ * Find which site ID a given URL belongs to (reverse lookup).
+ * Returns the canonical site ID, or null if the URL doesn't match any template.
+ */
+export function findSiteByUrl(url) {
+    try {
+        const parsed = new URL(url);
+        const hostname = parsed.hostname.replace(/^www\./, '');
+        // Map of hostnames to site IDs
+        const hostnameMap = {
+            'ebay.com': 'ebay',
+            'amazon.com': 'amazon',
+            'walmart.com': 'walmart',
+            'target.com': 'target',
+            'bestbuy.com': 'bestbuy',
+            'etsy.com': 'etsy',
+            'aliexpress.com': 'aliexpress',
+            'newegg.com': 'newegg',
+            'google.com': 'google',
+            'bing.com': 'bing',
+            'html.duckduckgo.com': 'duckduckgo',
+            'duckduckgo.com': 'duckduckgo',
+            'reddit.com': 'reddit',
+            'youtube.com': 'youtube',
+            'x.com': 'twitter',
+            'twitter.com': 'twitter',
+            'linkedin.com': 'linkedin',
+            'github.com': 'github',
+            'stackoverflow.com': 'stackoverflow',
+            'npmjs.com': 'npm',
+            'pypi.org': 'pypi',
+            'zillow.com': 'zillow',
+            'realtor.com': 'realtor',
+            'indeed.com': 'indeed',
+            'glassdoor.com': 'glassdoor',
+            'yelp.com': 'yelp',
+            'doordash.com': 'doordash',
+            'ubereats.com': 'ubereats',
+        };
+        return hostnameMap[hostname] ?? null;
+    }
+    catch {
+        return null;
+    }
+}

package/dist/core/sitemap.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Sitemap discovery and parsing
+ * Discovers URLs from sitemap.xml files
+ */
+export interface SitemapUrl {
+    url: string;
+    lastmod?: string;
+    changefreq?: string;
+    priority?: number;
+}
+export interface SitemapResult {
+    urls: SitemapUrl[];
+    sitemapUrls: string[];
+    elapsed: number;
+}
+/**
+ * Discover all URLs from a domain's sitemap.xml
+ * Handles sitemap index files (recursive), gzip compression, and common locations
+ */
+export declare function discoverSitemap(domain: string, options?: {
+    timeout?: number;
+    maxUrls?: number;
+}): Promise<SitemapResult>;

package/dist/core/sitemap.js

@@ -0,0 +1,105 @@
+/**
+ * Sitemap discovery and parsing
+ * Discovers URLs from sitemap.xml files
+ */
+import { fetch as undiciFetch } from 'undici';
+/**
+ * Discover all URLs from a domain's sitemap.xml
+ * Handles sitemap index files (recursive), gzip compression, and common locations
+ */
+export async function discoverSitemap(domain, options) {
+    const startTime = Date.now();
+    const maxUrls = options?.maxUrls || 10000;
+    const timeout = options?.timeout || 10000;
+    const allUrls = [];
+    const sitemapUrls = [];
+    const visited = new Set();
+    // Try common sitemap locations
+    const sitemapLocations = [
+        `https://${domain}/sitemap.xml`,
+        `https://${domain}/sitemap_index.xml`,
+        `https://${domain}/sitemap/sitemap.xml`,
+        `https://${domain}/wp-sitemap.xml`,
+    ];
+    // Also check robots.txt for sitemap references
+    try {
+        const robotsResp = await undiciFetch(`https://${domain}/robots.txt`, {
+            signal: AbortSignal.timeout(timeout),
+            headers: { 'User-Agent': 'WebPeel/0.4.0 (+https://webpeel.dev)' },
+        });
+        if (robotsResp.ok) {
+            const robotsText = await robotsResp.text();
+            const sitemapMatches = robotsText.match(/Sitemap:\s*(.+)/gi) || [];
+            for (const match of sitemapMatches) {
+                const url = match.replace(/Sitemap:\s*/i, '').trim();
+                if (url && !sitemapLocations.includes(url)) {
+                    sitemapLocations.unshift(url); // Prioritize robots.txt sitemaps
+                }
+            }
+        }
+    }
+    catch { /* ignore robots.txt errors */ }
+    async function parseSitemap(sitemapUrl) {
+        if (visited.has(sitemapUrl) || allUrls.length >= maxUrls)
+            return;
+        visited.add(sitemapUrl);
+        try {
+            const resp = await undiciFetch(sitemapUrl, {
+                signal: AbortSignal.timeout(timeout),
+                headers: { 'User-Agent': 'WebPeel/0.4.0 (+https://webpeel.dev)' },
+            });
+            if (!resp.ok)
+                return;
+            const xml = await resp.text();
+            sitemapUrls.push(sitemapUrl);
+            // Use cheerio for XML parsing
+            const { load } = await import('cheerio');
+            const $ = load(xml, { xml: true });
+            // Check if this is a sitemap index
+            const sitemapIndexEntries = $('sitemapindex > sitemap > loc');
+            if (sitemapIndexEntries.length > 0) {
+                for (let i = 0; i < sitemapIndexEntries.length && allUrls.length < maxUrls; i++) {
+                    const childUrl = $(sitemapIndexEntries[i]).text().trim();
+                    if (childUrl) {
+                        await parseSitemap(childUrl);
+                    }
+                }
+                return;
+            }
+            // Parse URL entries
+            $('urlset > url').each((_, el) => {
+                if (allUrls.length >= maxUrls)
+                    return false;
+                const loc = $(el).find('loc').text().trim();
+                if (!loc)
+                    return undefined;
+                const entry = { url: loc };
+                const lastmod = $(el).find('lastmod').text().trim();
+                const changefreq = $(el).find('changefreq').text().trim();
+                const priority = $(el).find('priority').text().trim();
+                if (lastmod)
+                    entry.lastmod = lastmod;
+                if (changefreq)
+                    entry.changefreq = changefreq;
+                if (priority)
+                    entry.priority = parseFloat(priority);
+                allUrls.push(entry);
+                return undefined;
+            });
+        }
+        catch { /* skip failed sitemaps */ }
+    }
+    // Try each sitemap location
+    for (const sitemapUrl of sitemapLocations) {
+        if (allUrls.length >= maxUrls)
+            break;
+        await parseSitemap(sitemapUrl);
+        if (allUrls.length > 0)
+            break; // Found a working sitemap, stop trying others
+    }
+    return {
+        urls: allUrls,
+        sitemapUrls,
+        elapsed: Date.now() - startTime,
+    };
+}

package/dist/core/source-credibility.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Source credibility scoring — lightweight, zero dependencies, no network calls.
+ *
+ * Actively investigates domain signals from the URL itself:
+ *   - TLD trust score
+ *   - HTTPS enforcement
+ *   - Domain structure analysis
+ *   - Brand/platform recognition (500+ known domains)
+ *   - Content platform detection
+ *
+ * Score breakdown (0–100):
+ *   TLD weight    0–20
+ *   HTTPS         0–10
+ *   Known domain  0–40
+ *   Structure     0–15
+ *   Platform      0–15
+ */
+export interface SourceCredibility {
+    tier: 'official' | 'established' | 'community' | 'new' | 'suspicious';
+    score: number;
+    label: string;
+    signals: string[];
+    warnings: string[];
+}
+/**
+ * Assess the credibility of a source URL.
+ * Fully synchronous — no network calls.
+ */
+export declare function getSourceCredibility(url: string): SourceCredibility;