@iflow-mcp/jakeliume-webpeel 0.22.0

package/dist/core/strategy-hooks.d.ts

@@ -0,0 +1,145 @@
+/**
+ * Strategy hooks — plugin interface for premium server-side optimizations.
+ *
+ * The base `smartFetch()` in strategies.ts provides solid simple→browser→stealth
+ * escalation.  Hooks allow the server (or any host) to layer on caching, domain
+ * intelligence, and parallel-race strategies *without* shipping that logic in
+ * the npm package.
+ *
+ * Register hooks once at startup via `registerStrategyHooks()`.
+ * All hook methods are optional — unset hooks are simply skipped.
+ */
+import type { FetchResult } from './fetcher.js';
+import type { DomainExtractResult } from '../ee/domain-extractors.js';
+export interface StrategyResult extends FetchResult {
+    method: 'simple' | 'browser' | 'stealth' | 'cached' | 'cloaked' | 'cycle' | 'peeltls' | 'cf-worker' | 'google-cache';
+    /**
+     * Set to true when the final response still appears to be a bot-challenge
+     * page after all escalation attempts have been exhausted.
+     * Consumers should warn the user when this is true.
+     */
+    challengeDetected?: boolean;
+}
+export interface DomainRecommendation {
+    mode: 'browser' | 'stealth';
+}
+export interface CacheCheckResult {
+    /** Cached response to serve immediately. */
+    value: StrategyResult;
+    /** When true the caller should trigger a background revalidation. */
+    stale: boolean;
+}
+export interface StrategyHooks {
+    /**
+     * Look up `url` in the cache.
+     * Return `null` for a cache miss.
+     * When `stale` is true the result can be served but should be refreshed.
+     */
+    checkCache?(url: string): CacheCheckResult | null;
+    /**
+     * Attempt to mark `url` as "currently revalidating" so that only one
+     * background refresh runs at a time.  Return `true` if this call won
+     * the race (caller should revalidate), `false` otherwise.
+     */
+    markRevalidating?(url: string): boolean;
+    /**
+     * Store a fresh result in the cache.
+     */
+    setCache?(url: string, result: StrategyResult): void;
+    /**
+     * Return a recommendation for how to fetch `url` based on historical
+     * success/failure data for the domain.  Return `null` to let the default
+     * escalation logic decide.
+     */
+    getDomainRecommendation?(url: string): DomainRecommendation | null;
+    /**
+     * Record the outcome of a fetch so the intelligence layer can learn.
+     */
+    recordDomainResult?(url: string, method: 'simple' | 'browser' | 'stealth', latencyMs: number): void;
+    /**
+     * Whether to use the parallel race strategy (fire browser after a short
+     * timeout if simple fetch hasn't resolved).  Default: false (no race).
+     */
+    shouldRace?(): boolean;
+    /**
+     * Timeout (ms) before the race starts a parallel browser fetch.
+     * Only called when `shouldRace()` returns true.  Default: 2000.
+     */
+    getRaceTimeoutMs?(): number;
+    /**
+     * Premium domain extraction hook — 55+ domain extractors.
+     * Return null to fall back to basic/no extraction.
+     */
+    extractDomainData?(html: string, url: string): Promise<DomainExtractResult | null>;
+    /**
+     * Returns a function that checks if a URL has a known domain extractor.
+     * Premium knows which domains have extractors; basic returns null for all.
+     */
+    getDomainExtractor?(url: string): ((html: string, url: string) => Promise<DomainExtractResult | null>) | null;
+    /**
+     * Premium SPA domain list — knows which sites require browser rendering.
+     * Basic: returns empty set (no SPA auto-detection).
+     */
+    getSPADomains?(): Set<string>;
+    /**
+     * Premium SPA URL patterns — matches specific paths needing render.
+     * Basic: returns empty array.
+     */
+    getSPAPatterns?(): RegExp[];
+    /**
+     * Premium CAPTCHA/challenge solving hook.
+     * Return null to fall back to default challenge handling.
+     */
+    solveChallenge?(page: any, url: string): Promise<{
+        solved: boolean;
+        html?: string;
+    } | null>;
+    /**
+     * Premium wait-for-stable content logic — smarter than waitForLoadState.
+     * Return null/undefined to fall back to default wait logic.
+     */
+    waitForContentStable?(page: any, options?: any): Promise<void>;
+}
+/**
+ * Register premium strategy hooks.  Should be called once at server startup.
+ * Calling again replaces the previous hooks entirely.
+ */
+export declare function registerStrategyHooks(hooks: StrategyHooks): void;
+/**
+ * Clear all registered hooks (useful in tests).
+ */
+export declare function clearStrategyHooks(): void;
+/**
+ * Retrieve the current hooks (internal — used by strategies.ts).
+ */
+export declare function getStrategyHooks(): Readonly<StrategyHooks>;
+/**
+ * Get the premium domain extraction hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export declare function getDomainExtractHook(): StrategyHooks['extractDomainData'];
+/**
+ * Get the premium domain extractor lookup hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export declare function getDomainExtractorHook(): StrategyHooks['getDomainExtractor'];
+/**
+ * Get the premium SPA domains hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export declare function getSPADomainsHook(): StrategyHooks['getSPADomains'];
+/**
+ * Get the premium SPA patterns hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export declare function getSPAPatternsHook(): StrategyHooks['getSPAPatterns'];
+/**
+ * Get the premium challenge solver hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export declare function getChallengeHook(): StrategyHooks['solveChallenge'];
+/**
+ * Get the premium content stability hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export declare function getStabilityHook(): StrategyHooks['waitForContentStable'];

package/dist/core/strategy-hooks.js

@@ -0,0 +1,74 @@
+/**
+ * Strategy hooks — plugin interface for premium server-side optimizations.
+ *
+ * The base `smartFetch()` in strategies.ts provides solid simple→browser→stealth
+ * escalation.  Hooks allow the server (or any host) to layer on caching, domain
+ * intelligence, and parallel-race strategies *without* shipping that logic in
+ * the npm package.
+ *
+ * Register hooks once at startup via `registerStrategyHooks()`.
+ * All hook methods are optional — unset hooks are simply skipped.
+ */
+/* ---------- singleton registry ------------------------------------------- */
+let registeredHooks = {};
+/**
+ * Register premium strategy hooks.  Should be called once at server startup.
+ * Calling again replaces the previous hooks entirely.
+ */
+export function registerStrategyHooks(hooks) {
+    registeredHooks = { ...hooks };
+}
+/**
+ * Clear all registered hooks (useful in tests).
+ */
+export function clearStrategyHooks() {
+    registeredHooks = {};
+}
+/**
+ * Retrieve the current hooks (internal — used by strategies.ts).
+ */
+export function getStrategyHooks() {
+    return registeredHooks;
+}
+/**
+ * Get the premium domain extraction hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export function getDomainExtractHook() {
+    return registeredHooks.extractDomainData;
+}
+/**
+ * Get the premium domain extractor lookup hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export function getDomainExtractorHook() {
+    return registeredHooks.getDomainExtractor;
+}
+/**
+ * Get the premium SPA domains hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export function getSPADomainsHook() {
+    return registeredHooks.getSPADomains;
+}
+/**
+ * Get the premium SPA patterns hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export function getSPAPatternsHook() {
+    return registeredHooks.getSPAPatterns;
+}
+/**
+ * Get the premium challenge solver hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export function getChallengeHook() {
+    return registeredHooks.solveChallenge;
+}
+/**
+ * Get the premium content stability hook, if registered.
+ * Returns undefined when no premium hooks are active (basic/npm mode).
+ */
+export function getStabilityHook() {
+    return registeredHooks.waitForContentStable;
+}

package/dist/core/structured-extract.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Structured JSON Extraction Engine
+ *
+ * Extracts structured data from markdown/text content using either:
+ *   1. LLM (via callLLM from llm-provider.ts) when an LLM config is provided
+ *   2. Heuristic regex/BM25-style extraction as a zero-key fallback
+ *
+ * Firecrawl-compatible: accepts a JSON schema, returns typed structured data.
+ */
+import { type LLMConfig } from './llm-provider.js';
+export interface ExtractionSchema {
+    type: 'object';
+    properties: Record<string, {
+        type: string;
+        description?: string;
+    }>;
+    required?: string[];
+}
+export interface ExtractionResult {
+    data: Record<string, unknown>;
+    /** Confidence score 0-1. LLM: 0.85-0.95. Heuristic: 0.3-0.5. */
+    confidence: number;
+    tokensUsed: number;
+}
+/**
+ * Extract structured data from markdown content using an LLM or heuristics.
+ *
+ * @param content   Markdown/text content to extract from
+ * @param schema    JSON schema describing what to extract
+ * @param llmConfig Optional LLM config (if omitted, uses heuristic fallback)
+ * @param prompt    Optional user guidance added to the LLM prompt
+ */
+export declare function extractStructured(content: string, schema: ExtractionSchema, llmConfig?: LLMConfig, prompt?: string, domainHints?: Record<string, unknown>): Promise<ExtractionResult>;
+/**
+ * Convert a shorthand schema `{ field: "string", active: "boolean" }` to a
+ * full ExtractionSchema.  Useful for CLI --extract flag.
+ */
+export declare function simpleToExtractionSchema(simple: Record<string, string>): ExtractionSchema;
+/**
+ * Check if a JSON object looks like a simple type-schema
+ * (`{ field: "string" | "boolean" | "number" }`) rather than CSS selectors.
+ */
+export declare function isTypeSchema(obj: Record<string, unknown>): boolean;