@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/local-search.d.ts

@@ -0,0 +1,60 @@
+/**
+ * local-search.ts — Local business search using Google Places API (Text Search)
+ *
+ * Primary: Google Places Text Search (New) API
+ * Secondary: Yelp Fusion API
+ * Fallback: peel() scraping Google Maps
+ */
+export interface LocalSearchOptions {
+    query: string;
+    location?: string;
+    country?: string;
+    language?: string;
+    radius?: number;
+    type?: string;
+    limit?: number;
+}
+export interface LocalSearchResult {
+    name: string;
+    address: string;
+    rating?: number;
+    reviewCount?: number;
+    priceLevel?: number;
+    categories?: string[];
+    phone?: string;
+    website?: string;
+    googleMapsUrl?: string;
+    isOpen?: boolean;
+    hours?: string[];
+    location?: {
+        lat: number;
+        lng: number;
+    };
+    photos?: string[];
+}
+export interface LocalSearchResponse {
+    results: LocalSearchResult[];
+    query: string;
+    location?: string;
+    source: 'google-places' | 'yelp' | 'fallback';
+}
+/**
+ * Search for local businesses/places using the best available source.
+ *
+ * Priority order:
+ * 1. Google Places Text Search (New) API — requires GOOGLE_PLACES_API_KEY
+ * 2. Yelp Fusion API — requires YELP_API_KEY (merged with Google if both available)
+ * 3. Google Maps scraping via peel() — no API key required, slowest
+ *
+ * @example
+ * ```typescript
+ * const results = await localSearch({
+ *   query: 'best sushi',
+ *   location: 'Shibuya, Tokyo',
+ *   language: 'ja',
+ *   country: 'JP',
+ * });
+ * console.log(results.results[0].name); // "寿司 さわ"
+ * ```
+ */
+export declare function localSearch(opts: LocalSearchOptions): Promise<LocalSearchResponse>;

package/dist/core/local-search.js

@@ -0,0 +1,308 @@
+/**
+ * local-search.ts — Local business search using Google Places API (Text Search)
+ *
+ * Primary: Google Places Text Search (New) API
+ * Secondary: Yelp Fusion API
+ * Fallback: peel() scraping Google Maps
+ */
+import { peel } from '../index.js';
+// ─── Google Places Text Search (New) API ─────────────────────────────────
+const GOOGLE_PLACES_FIELD_MASK = [
+    'places.displayName',
+    'places.formattedAddress',
+    'places.rating',
+    'places.userRatingCount',
+    'places.priceLevel',
+    'places.currentOpeningHours',
+    'places.googleMapsUri',
+    'places.websiteUri',
+    'places.internationalPhoneNumber',
+    'places.types',
+    'places.location',
+    'places.photos',
+].join(',');
+/**
+ * Convert Google Places priceLevel string to numeric 0-4 scale.
+ * Google New API returns strings like "PRICE_LEVEL_MODERATE".
+ */
+function parsePriceLevel(raw) {
+    if (typeof raw === 'number')
+        return raw;
+    if (!raw)
+        return undefined;
+    const MAP = {
+        PRICE_LEVEL_FREE: 0,
+        PRICE_LEVEL_INEXPENSIVE: 1,
+        PRICE_LEVEL_MODERATE: 2,
+        PRICE_LEVEL_EXPENSIVE: 3,
+        PRICE_LEVEL_VERY_EXPENSIVE: 4,
+    };
+    return MAP[String(raw)] ?? undefined;
+}
+async function searchGooglePlaces(opts) {
+    const key = process.env.GOOGLE_PLACES_API_KEY;
+    if (!key)
+        return null;
+    // Build the text query — append location if not already in query
+    let textQuery = opts.query;
+    if (opts.location && !opts.query.toLowerCase().includes(opts.location.toLowerCase())) {
+        textQuery = `${opts.query} in ${opts.location}`;
+    }
+    const body = {
+        textQuery,
+        maxResultCount: Math.min(opts.limit ?? 10, 20),
+    };
+    if (opts.language)
+        body.languageCode = opts.language;
+    if (opts.country)
+        body.regionCode = opts.country;
+    if (opts.type)
+        body.includedType = opts.type;
+    // If location is lat,lng coordinates, add location bias
+    if (opts.location) {
+        const latLngMatch = opts.location.match(/^(-?\d+\.?\d*),\s*(-?\d+\.?\d*)$/);
+        if (latLngMatch) {
+            body.locationBias = {
+                circle: {
+                    center: { latitude: parseFloat(latLngMatch[1]), longitude: parseFloat(latLngMatch[2]) },
+                    radius: opts.radius ?? 5000,
+                },
+            };
+        }
+    }
+    try {
+        const res = await fetch('https://places.googleapis.com/v1/places:searchText', {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'X-Goog-Api-Key': key,
+                'X-Goog-FieldMask': GOOGLE_PLACES_FIELD_MASK,
+            },
+            body: JSON.stringify(body),
+            signal: AbortSignal.timeout(8000),
+        });
+        if (!res.ok) {
+            console.warn(`[local-search] Google Places API ${res.status}: ${await res.text().catch(() => '')}`);
+            return null;
+        }
+        const data = await res.json();
+        if (!data.places?.length)
+            return [];
+        return data.places.map((p) => {
+            // Build photo URLs
+            const photos = [];
+            if (p.photos?.length) {
+                for (const photo of p.photos.slice(0, 3)) {
+                    if (photo.name) {
+                        photos.push(`https://places.googleapis.com/v1/${photo.name}/media?maxHeightPx=400&key=${key}`);
+                    }
+                }
+            }
+            // Parse hours
+            const hours = p.currentOpeningHours?.weekdayDescriptions ?? [];
+            return {
+                name: p.displayName?.text ?? '',
+                address: p.formattedAddress ?? '',
+                rating: p.rating,
+                reviewCount: p.userRatingCount,
+                priceLevel: parsePriceLevel(p.priceLevel),
+                categories: p.types?.filter((t) => !t.startsWith('point_of_interest') && !t.startsWith('establishment')),
+                phone: p.internationalPhoneNumber,
+                website: p.websiteUri,
+                googleMapsUrl: p.googleMapsUri,
+                isOpen: p.currentOpeningHours?.openNow,
+                hours,
+                location: p.location ? { lat: p.location.latitude, lng: p.location.longitude } : undefined,
+                photos,
+            };
+        });
+    }
+    catch (err) {
+        console.warn('[local-search] Google Places request failed:', err.message);
+        return null;
+    }
+}
+// ─── Yelp Fusion API ──────────────────────────────────────────────────────
+async function searchYelp(opts) {
+    const key = process.env.YELP_API_KEY;
+    if (!key)
+        return null;
+    try {
+        const params = new URLSearchParams({
+            term: opts.query,
+            limit: String(Math.min(opts.limit ?? 10, 50)),
+            sort_by: 'rating',
+        });
+        // Location can be an address string or lat,lng
+        if (opts.location) {
+            const latLngMatch = opts.location.match(/^(-?\d+\.?\d*),\s*(-?\d+\.?\d*)$/);
+            if (latLngMatch) {
+                params.set('latitude', latLngMatch[1]);
+                params.set('longitude', latLngMatch[2]);
+                params.set('radius', String(Math.min(opts.radius ?? 5000, 40000)));
+            }
+            else {
+                params.set('location', opts.location);
+            }
+        }
+        else {
+            params.set('location', 'New York, NY');
+        }
+        if (opts.language)
+            params.set('locale', opts.language + '_' + (opts.country ?? '').toUpperCase());
+        const res = await fetch(`https://api.yelp.com/v3/businesses/search?${params}`, {
+            headers: { Authorization: `Bearer ${key}` },
+            signal: AbortSignal.timeout(6000),
+        });
+        if (!res.ok)
+            return null;
+        const data = await res.json();
+        return (data.businesses || []).map((b) => ({
+            name: b.name,
+            address: b.location
+                ? [b.location.address1, b.location.city, b.location.state, b.location.country]
+                    .filter(Boolean)
+                    .join(', ')
+                : '',
+            rating: b.rating,
+            reviewCount: b.review_count,
+            priceLevel: b.price ? b.price.length : undefined, // "$" → 1, "$$" → 2, etc.
+            categories: (b.categories || []).map((c) => c.alias),
+            phone: b.display_phone,
+            website: b.url,
+            googleMapsUrl: undefined,
+            isOpen: b.is_closed === false,
+            hours: undefined,
+            location: b.coordinates
+                ? { lat: b.coordinates.latitude, lng: b.coordinates.longitude }
+                : undefined,
+            photos: b.image_url ? [b.image_url] : [],
+        }));
+    }
+    catch (err) {
+        console.warn('[local-search] Yelp request failed:', err.message);
+        return null;
+    }
+}
+// ─── Google Maps scraping fallback ────────────────────────────────────────
+async function searchGoogleMapsFallback(opts) {
+    try {
+        const query = opts.location
+            ? `${opts.query} in ${opts.location}`
+            : opts.query;
+        const url = `https://www.google.com/maps/search/${encodeURIComponent(query)}`;
+        const result = await peel(url, { render: true, timeout: 15000 });
+        // Extract what we can from the markdown content
+        const lines = result.content.split('\n').filter(Boolean);
+        const businesses = [];
+        // Simple heuristic: look for business name patterns followed by rating/address
+        let current = null;
+        for (const line of lines) {
+            // Rating line: "4.5 (1,234)"
+            const ratingMatch = line.match(/^(\d\.\d)\s*\((\d[\d,]*)\)/);
+            if (ratingMatch && current) {
+                current.rating = parseFloat(ratingMatch[1]);
+                current.reviewCount = parseInt(ratingMatch[2].replace(/,/g, ''));
+                continue;
+            }
+            // Push completed entry and start new one for non-empty lines that look like names
+            if (line.length > 3 && line.length < 100 && !line.startsWith('http') && !ratingMatch) {
+                if (current?.name) {
+                    businesses.push(current);
+                }
+                current = { name: line, address: '' };
+            }
+        }
+        if (current?.name)
+            businesses.push(current);
+        return businesses.slice(0, opts.limit ?? 10);
+    }
+    catch {
+        return [];
+    }
+}
+// ─── Dedup by name+address similarity ────────────────────────────────────
+function normalize(s) {
+    return s.toLowerCase().replace(/[^a-z0-9\s]/g, '').replace(/\s+/g, ' ').trim();
+}
+function isSimilar(a, b) {
+    const nameA = normalize(a.name);
+    const nameB = normalize(b.name);
+    if (nameA === nameB)
+        return true;
+    // Jaccard similarity on word tokens
+    const tokA = new Set(nameA.split(' '));
+    const tokB = new Set(nameB.split(' '));
+    const intersection = [...tokA].filter(t => tokB.has(t));
+    const union = new Set([...tokA, ...tokB]);
+    const jaccard = intersection.length / union.size;
+    return jaccard >= 0.7;
+}
+function dedupResults(primary, secondary) {
+    const merged = [...primary];
+    for (const item of secondary) {
+        if (!merged.some(existing => isSimilar(existing, item))) {
+            merged.push(item);
+        }
+    }
+    return merged;
+}
+// ─── Main export ──────────────────────────────────────────────────────────
+/**
+ * Search for local businesses/places using the best available source.
+ *
+ * Priority order:
+ * 1. Google Places Text Search (New) API — requires GOOGLE_PLACES_API_KEY
+ * 2. Yelp Fusion API — requires YELP_API_KEY (merged with Google if both available)
+ * 3. Google Maps scraping via peel() — no API key required, slowest
+ *
+ * @example
+ * ```typescript
+ * const results = await localSearch({
+ *   query: 'best sushi',
+ *   location: 'Shibuya, Tokyo',
+ *   language: 'ja',
+ *   country: 'JP',
+ * });
+ * console.log(results.results[0].name); // "寿司 さわ"
+ * ```
+ */
+export async function localSearch(opts) {
+    const limit = opts.limit ?? 10;
+    let source = 'fallback';
+    // Try Google Places first (best data quality)
+    const googleResults = await searchGooglePlaces(opts);
+    if (googleResults !== null) {
+        source = 'google-places';
+        let results = googleResults;
+        // Also fetch Yelp in parallel if key is available, merge for more coverage
+        const yelpResults = await searchYelp({ ...opts, limit: Math.ceil(limit / 2) });
+        if (yelpResults && yelpResults.length > 0) {
+            results = dedupResults(googleResults, yelpResults);
+        }
+        return {
+            results: results.slice(0, limit),
+            query: opts.query,
+            location: opts.location,
+            source,
+        };
+    }
+    // Try Yelp as primary if no Google key
+    const yelpResults = await searchYelp(opts);
+    if (yelpResults !== null && yelpResults.length > 0) {
+        return {
+            results: yelpResults.slice(0, limit),
+            query: opts.query,
+            location: opts.location,
+            source: 'yelp',
+        };
+    }
+    // Last resort: scrape Google Maps
+    const fallbackResults = await searchGoogleMapsFallback(opts);
+    return {
+        results: fallbackResults.slice(0, limit),
+        query: opts.query,
+        location: opts.location,
+        source: 'fallback',
+    };
+}

package/dist/core/logger.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Lightweight structured logger — no external dependencies.
+ *
+ * Production: JSON lines (structured, parseable by log aggregators)
+ * Development: Human-readable console output
+ *
+ * Supports child loggers for request-scoped context.
+ *
+ * Levels: debug < info < warn < error < silent
+ *
+ * Respects WEBPEEL_LOG_LEVEL env var.
+ * Defaults: production → 'info', development → 'debug'.
+ *
+ * All output goes to stderr so stdout stays clean for data/JSON (CLI piping).
+ */
+export type LogLevel = 'debug' | 'info' | 'warn' | 'error' | 'silent';
+interface LogContext {
+    [key: string]: any;
+}
+export interface Logger {
+    debug: (...args: any[]) => void;
+    info: (...args: any[]) => void;
+    warn: (...args: any[]) => void;
+    error: (...args: any[]) => void;
+    child: (context: LogContext) => Logger;
+}
+export declare function createLogger(module: string): Logger;
+export {};

package/dist/core/logger.js

@@ -0,0 +1,104 @@
+/**
+ * Lightweight structured logger — no external dependencies.
+ *
+ * Production: JSON lines (structured, parseable by log aggregators)
+ * Development: Human-readable console output
+ *
+ * Supports child loggers for request-scoped context.
+ *
+ * Levels: debug < info < warn < error < silent
+ *
+ * Respects WEBPEEL_LOG_LEVEL env var.
+ * Defaults: production → 'info', development → 'debug'.
+ *
+ * All output goes to stderr so stdout stays clean for data/JSON (CLI piping).
+ */
+const LEVELS = {
+    debug: 0,
+    info: 1,
+    warn: 2,
+    error: 3,
+    silent: 4,
+};
+const currentLevel = () => {
+    const env = process.env.WEBPEEL_LOG_LEVEL?.toLowerCase();
+    if (env && env in LEVELS)
+        return env;
+    return process.env.NODE_ENV === 'production' ? 'info' : 'debug';
+};
+const isProduction = process.env.NODE_ENV === 'production';
+function formatJson(level, module, args, context) {
+    // Extract message and data from args
+    let msg = '';
+    const data = {};
+    for (const arg of args) {
+        if (typeof arg === 'string') {
+            msg = msg ? `${msg} ${arg}` : arg;
+        }
+        else if (typeof arg === 'object' && arg !== null) {
+            // Merge objects into data
+            if (arg instanceof Error) {
+                data.error = { message: arg.message, stack: arg.stack, name: arg.name };
+                if ('code' in arg)
+                    data.error.code = arg.code;
+            }
+            else {
+                Object.assign(data, arg);
+            }
+        }
+        else {
+            msg = msg ? `${msg} ${String(arg)}` : String(arg);
+        }
+    }
+    const entry = {
+        level,
+        module,
+        msg,
+        timestamp: new Date().toISOString(),
+        ...context,
+    };
+    if (Object.keys(data).length > 0) {
+        entry.data = data;
+    }
+    try {
+        return JSON.stringify(entry);
+    }
+    catch {
+        return JSON.stringify({ level, module, msg: String(args), timestamp: new Date().toISOString() });
+    }
+}
+function createLoggerInternal(module, context = {}) {
+    const prefix = `[webpeel:${module}]`;
+    const shouldLog = (level) => LEVELS[currentLevel()] <= LEVELS[level];
+    const logFn = (level, ...args) => {
+        if (!shouldLog(level))
+            return;
+        if (isProduction) {
+            console.error(formatJson(level, module, args, context));
+        }
+        else {
+            // Human-readable for development
+            console.error(prefix, ...(level === 'warn' ? ['[WARN]'] : level === 'error' ? ['[ERROR]'] : []), ...args.map((a) => {
+                if (typeof a === 'object' && a !== null && !(a instanceof Error)) {
+                    try {
+                        return JSON.stringify(a);
+                    }
+                    catch {
+                        return String(a);
+                    }
+                }
+                return a;
+            }));
+        }
+    };
+    return {
+        debug: (...args) => logFn('debug', ...args),
+        info: (...args) => logFn('info', ...args),
+        warn: (...args) => logFn('warn', ...args),
+        error: (...args) => logFn('error', ...args),
+        child: (childContext) => createLoggerInternal(module, { ...context, ...childContext }),
+    };
+}
+export function createLogger(module) {
+    return createLoggerInternal(module);
+}

package/dist/core/map.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Domain URL mapping
+ * Combines sitemap discovery with link crawling to discover all URLs on a domain
+ */
+export interface MapOptions {
+    /** Include sitemap URLs (default: true) */
+    useSitemap?: boolean;
+    /** Crawl the homepage for additional links (default: true) */
+    crawlHomepage?: boolean;
+    /** Maximum URLs to discover (default: 5000) */
+    maxUrls?: number;
+    /** Timeout per request in ms (default: 10000) */
+    timeout?: number;
+    /** Include URL patterns matching these regexes only */
+    includePatterns?: string[];
+    /** Exclude URL patterns matching these regexes */
+    excludePatterns?: string[];
+    /** Filter URLs by relevance to this search query */
+    search?: string;
+    /** Only return URLs matching these content types */
+    contentTypeFilter?: string[];
+}
+export interface MapResult {
+    /** All discovered URLs (deduplicated) */
+    urls: string[];
+    /** Sitemap URLs used */
+    sitemapUrls: string[];
+    /** Total URLs discovered */
+    total: number;
+    /** Time elapsed in ms */
+    elapsed: number;
+}
+export declare function mapDomain(startUrl: string, options?: MapOptions): Promise<MapResult>;

package/dist/core/map.js

@@ -0,0 +1,127 @@
+/**
+ * Domain URL mapping
+ * Combines sitemap discovery with link crawling to discover all URLs on a domain
+ */
+import { discoverSitemap } from './sitemap.js';
+import { peel } from '../index.js';
+/** Safely compile a user-supplied regex pattern with length limit. */
+function safeRegex(pattern) {
+    if (pattern.length > 200) {
+        throw new Error(`Regex pattern too long (${pattern.length} chars, max 200)`);
+    }
+    try {
+        return new RegExp(pattern);
+    }
+    catch {
+        throw new Error(`Invalid regex pattern: ${pattern}`);
+    }
+}
+export async function mapDomain(startUrl, options = {}) {
+    const startTime = Date.now();
+    const { useSitemap = true, crawlHomepage = true, maxUrls = 5000, timeout = 10000, includePatterns = [], excludePatterns = [], search, contentTypeFilter = [], } = options;
+    const urlObj = new URL(startUrl);
+    const domain = urlObj.hostname;
+    const allUrls = new Map(); // URL -> relevance score
+    let sitemapUrls = [];
+    // Compile filter patterns
+    const includeRegexes = includePatterns.map(p => safeRegex(p));
+    const excludeRegexes = excludePatterns.map(p => safeRegex(p));
+    // Parse search terms
+    const searchTerms = search ? search.toLowerCase().split(/\s+/).filter(t => t.length > 0) : [];
+    /**
+     * Calculate relevance score for a URL based on search terms
+     * Scores based on matches in URL path and title/description if available
+     */
+    function calculateRelevance(url, title, description) {
+        if (searchTerms.length === 0)
+            return 1; // No search = all equal
+        let score = 0;
+        const urlLower = url.toLowerCase();
+        const titleLower = (title || '').toLowerCase();
+        const descLower = (description || '').toLowerCase();
+        for (const term of searchTerms) {
+            // URL path matches (highest weight)
+            if (urlLower.includes(term))
+                score += 3;
+            // Title matches
+            if (titleLower.includes(term))
+                score += 2;
+            // Description matches
+            if (descLower.includes(term))
+                score += 1;
+        }
+        return score;
+    }
+    /**
+     * Check if URL should be included based on patterns and content type
+     */
+    function shouldInclude(url) {
+        if (excludeRegexes.some(r => r.test(url)))
+            return false;
+        if (includeRegexes.length > 0 && !includeRegexes.some(r => r.test(url)))
+            return false;
+        // Content type filter (check file extension)
+        if (contentTypeFilter.length > 0) {
+            const ext = url.split('.').pop()?.toLowerCase() || '';
+            const hasMatch = contentTypeFilter.some(type => {
+                const typeExt = type.replace(/^\./, '').toLowerCase();
+                return ext === typeExt || url.toLowerCase().includes(`.${typeExt}`);
+            });
+            if (!hasMatch)
+                return false;
+        }
+        return true;
+    }
+    // Step 1: Sitemap discovery
+    if (useSitemap) {
+        const sitemap = await discoverSitemap(domain, { timeout, maxUrls });
+        sitemapUrls = sitemap.sitemapUrls;
+        for (const entry of sitemap.urls) {
+            if (allUrls.size >= maxUrls)
+                break;
+            if (shouldInclude(entry.url)) {
+                // Sitemap entries don't have title/description, just score by URL
+                const score = calculateRelevance(entry.url);
+                allUrls.set(entry.url, score);
+            }
+        }
+    }
+    // Step 2: Crawl homepage for additional links
+    if (crawlHomepage && allUrls.size < maxUrls) {
+        try {
+            const result = await peel(startUrl, { timeout });
+            for (const link of result.links) {
+                if (allUrls.size >= maxUrls)
+                    break;
+                try {
+                    const linkUrl = new URL(link);
+                    if (linkUrl.hostname === domain && shouldInclude(link)) {
+                        const score = calculateRelevance(link);
+                        if (!allUrls.has(link)) {
+                            allUrls.set(link, score);
+                        }
+                    }
+                }
+                catch { /* skip invalid URLs */ }
+            }
+        }
+        catch { /* skip homepage crawl errors */ }
+    }
+    // Sort URLs by relevance score (highest first), then alphabetically
+    const sortedUrls = Array.from(allUrls.entries())
+        .sort((a, b) => {
+        if (search) {
+            // Sort by score first if searching
+            if (b[1] !== a[1])
+                return b[1] - a[1];
+        }
+        return a[0].localeCompare(b[0]);
+    })
+        .map(([url]) => url);
+    return {
+        urls: sortedUrls,
+        sitemapUrls,
+        total: allUrls.size,
+        elapsed: Date.now() - startTime,
+    };
+}