@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/language-detect.js

@@ -0,0 +1,135 @@
+/**
+ * Lightweight language detection from URL hostname TLD.
+ * No external dependencies — pure TLD → language mapping.
+ */
+/** Map of country-code TLDs to primary BCP-47 language tags */
+const TLD_TO_LANGUAGE = {
+    jp: 'ja',
+    cn: 'zh',
+    de: 'de',
+    fr: 'fr',
+    kr: 'ko',
+    br: 'pt',
+    ru: 'ru',
+    es: 'es',
+    it: 'it',
+    nl: 'nl',
+    se: 'sv',
+    tw: 'zh-TW',
+    th: 'th',
+    vn: 'vi',
+    pl: 'pl',
+    in: 'hi',
+    id: 'id',
+    ar: 'es', // Argentina (.ar) — Spanish, not Arabic
+    // Additional common ccTLDs
+    pt: 'pt',
+    mx: 'es',
+    be: 'nl', // Belgium — defaults to Dutch; French also common
+    ch: 'de', // Switzerland — German most common
+    at: 'de', // Austria
+    dk: 'da', // Denmark
+    fi: 'fi', // Finland
+    no: 'nb', // Norway
+    hu: 'hu', // Hungary
+    cz: 'cs', // Czech Republic
+    sk: 'sk', // Slovakia
+    ro: 'ro', // Romania
+    bg: 'bg', // Bulgaria
+    hr: 'hr', // Croatia
+    gr: 'el', // Greece
+    tr: 'tr', // Turkey
+    ua: 'uk', // Ukraine
+    il: 'he', // Israel
+    sa: 'ar', // Saudi Arabia
+    ae: 'ar', // UAE
+    eg: 'ar', // Egypt
+};
+/**
+ * Detect the primary language from a URL based on the hostname TLD.
+ * Returns a BCP-47 language tag (e.g. 'ja', 'zh-TW') or null if unknown.
+ *
+ * Handles:
+ * - Simple ccTLDs: example.jp → 'ja'
+ * - Second-level ccTLDs: example.co.jp → 'ja'
+ * - No match for generic TLDs (.com, .org, .net, etc.): returns null
+ *
+ * @param url - The full URL string
+ * @returns BCP-47 language tag or null
+ */
+export function detectLanguageFromUrl(url) {
+    let hostname;
+    try {
+        hostname = new URL(url).hostname.toLowerCase();
+    }
+    catch {
+        return null;
+    }
+    // Remove www. and other common prefixes
+    hostname = hostname.replace(/^www\./, '');
+    // Split into parts: ['example', 'co', 'jp'] or ['tabelog', 'com']
+    const parts = hostname.split('.');
+    if (parts.length < 2)
+        return null;
+    const tld = parts[parts.length - 1];
+    // Check last part first (e.g., .jp, .fr)
+    if (TLD_TO_LANGUAGE[tld]) {
+        return TLD_TO_LANGUAGE[tld];
+    }
+    // For second-level domains like .co.jp, .com.br, check second-to-last too
+    // e.g., parts = ['example', 'co', 'jp'] → also already handled above
+    // This is already covered by checking the TLD (last part)
+    return null;
+}
+/**
+ * Build an Accept-Language header value from a list of language preferences.
+ * Adds quality weights (q=0.9, q=0.8, etc.) for secondary languages.
+ *
+ * @param languages - Array of BCP-47 language tags (e.g. ['ja', 'en'])
+ * @returns Accept-Language header string
+ */
+/** Map bare language codes to their most common region for Accept-Language headers */
+const LANG_TO_REGION = {
+    ja: 'JP', zh: 'CN', ko: 'KR', en: 'US', de: 'DE', fr: 'FR',
+    es: 'ES', pt: 'BR', ru: 'RU', it: 'IT', nl: 'NL', sv: 'SE',
+    da: 'DK', nb: 'NO', fi: 'FI', pl: 'PL', cs: 'CZ', hu: 'HU',
+    ro: 'RO', bg: 'BG', hr: 'HR', el: 'GR', tr: 'TR', uk: 'UA',
+    he: 'IL', ar: 'SA', th: 'TH', vi: 'VN', id: 'ID', hi: 'IN',
+    sk: 'SK',
+};
+export function buildAcceptLanguageHeader(languages) {
+    if (!languages || languages.length === 0)
+        return 'en-US,en;q=0.9';
+    const parts = [];
+    for (let i = 0; i < languages.length; i++) {
+        const lang = languages[i];
+        if (i === 0) {
+            // Primary language — no quality weight needed (implicitly q=1.0)
+            // Add region variant if it's a bare language code
+            if (!lang.includes('-') && lang.length === 2) {
+                const region = LANG_TO_REGION[lang] || lang.toUpperCase();
+                parts.push(`${lang}-${region}`);
+                parts.push(`${lang};q=0.9`);
+            }
+            else {
+                parts.push(lang);
+                // Add bare language code as fallback
+                const bare = lang.split('-')[0];
+                if (bare !== lang)
+                    parts.push(`${bare};q=0.9`);
+            }
+        }
+        else {
+            // Secondary languages get decreasing quality weights
+            const q = Math.max(0.1, 0.8 - (i - 1) * 0.1);
+            parts.push(`${lang};q=${q.toFixed(1)}`);
+        }
+    }
+    // Always add English as fallback unless already included
+    const hasEnglish = languages.some(l => l.startsWith('en'));
+    if (!hasEnglish) {
+        const q = Math.max(0.1, 0.8 - (languages.length - 1) * 0.1);
+        parts.push(`en;q=${q.toFixed(1)}`);
+    }
+    return parts.join(',');
+}

package/dist/core/links.d.ts

@@ -0,0 +1,10 @@
+export interface ExtractedLink {
+    url: string;
+    text: string;
+}
+/**
+ * Extract all links from an HTML string.
+ * Returns a deduplicated list of { url, text } pairs, excluding anchors,
+ * javascript: hrefs, mailto:, and tel: links.
+ */
+export declare function extractLinks(html: string, baseUrl?: string): ExtractedLink[];

package/dist/core/links.js

@@ -0,0 +1,44 @@
+/**
+ * Link extraction from HTML
+ * Extracts all <a href="..."> tags and returns deduplicated { url, text } pairs
+ */
+import { load } from 'cheerio';
+/**
+ * Extract all links from an HTML string.
+ * Returns a deduplicated list of { url, text } pairs, excluding anchors,
+ * javascript: hrefs, mailto:, and tel: links.
+ */
+export function extractLinks(html, baseUrl) {
+    if (!html)
+        return [];
+    const $ = load(html);
+    const seen = new Set();
+    const links = [];
+    $('a[href]').each((_i, el) => {
+        const href = $(el).attr('href') || '';
+        if (!href ||
+            href.startsWith('#') ||
+            href.startsWith('javascript:') ||
+            href.startsWith('mailto:') ||
+            href.startsWith('tel:') ||
+            href.startsWith('data:')) {
+            return;
+        }
+        let url = href;
+        // Resolve relative URLs when baseUrl is provided
+        if (baseUrl && !href.match(/^https?:\/\//)) {
+            try {
+                url = new URL(href, baseUrl).href;
+            }
+            catch {
+                return; // Skip unresolvable relative URLs
+            }
+        }
+        if (!seen.has(url)) {
+            seen.add(url);
+            const text = $(el).text().trim().replace(/\s+/g, ' ');
+            links.push({ url, text });
+        }
+    });
+    return links;
+}

package/dist/core/llm-extract.d.ts

@@ -0,0 +1,71 @@
+/**
+ * LLM-based extraction: sends markdown/text content to an LLM
+ * with instructions to extract structured data.
+ *
+ * Supports:
+ *   - OpenAI-compatible APIs (OpenAI, custom models via baseUrl)
+ *   - Anthropic (Claude Haiku, Sonnet, Opus)
+ *   - Google (Gemini Flash, Pro)
+ */
+export type LLMProvider = 'openai' | 'anthropic' | 'google';
+/** Default models per provider (cheapest/fastest) */
+export declare const DEFAULT_PROVIDER_MODELS: Record<LLMProvider, string>;
+export interface LLMExtractionOptions {
+    content: string;
+    instruction?: string;
+    schema?: object;
+    apiKey?: string;
+    baseUrl?: string;
+    model?: string;
+    maxTokens?: number;
+    url?: string;
+    prompt?: string;
+    llmProvider?: LLMProvider;
+    llmApiKey?: string;
+    llmModel?: string;
+}
+export interface LLMExtractionResult {
+    items: Array<Record<string, any>>;
+    tokensUsed: {
+        input: number;
+        output: number;
+    };
+    model: string;
+    cost?: number;
+    provider?: LLMProvider;
+}
+/**
+ * Detect if schema is a "full" JSON Schema (has type:"object" and properties).
+ */
+export declare function isFullJsonSchema(schema: object): boolean;
+/**
+ * Convert a simple example object to a proper JSON Schema.
+ *
+ * Supports:
+ *   - Primitive values: "" → { type: "string" }, 0 → { type: "number" }
+ *   - Arrays of objects: [{name:"", price:""}] → { type: "array", items: { type: "object", properties: {...} } }
+ *   - Nested objects
+ */
+export declare function convertSimpleToJsonSchema(example: object): object;
+/**
+ * Build the user message from content + optional instruction + optional schema.
+ */
+export declare function buildUserMessage(content: string, instruction?: string, schema?: object): string;
+/**
+ * Calculate estimated cost in USD for a given model and token counts.
+ */
+export declare function estimateCost(model: string, inputTokens: number, outputTokens: number): number | undefined;
+/**
+ * Parse the LLM response text into an items array.
+ * Handles both `{ "items": [...] }` and `[...]` formats.
+ * When a schema is provided, also handles single-object responses.
+ */
+export declare function parseItems(text: string, _schema?: object): Array<Record<string, any>>;
+/**
+ * Extract structured data from content using an LLM.
+ *
+ * Supports OpenAI (default), Anthropic, and Google providers.
+ * Pass `llmProvider` + `llmApiKey` to select a provider.
+ * Falls back to OpenAI-compatible path when no provider is specified.
+ */
+export declare function extractWithLLM(options: LLMExtractionOptions): Promise<LLMExtractionResult>;