@graphext/cuery 0.9.0 → 0.9.2

package/esm/mod.d.ts

@@ -13,6 +13,8 @@ export * from './src/tools/personas.js';
 export * from './src/tools/search.js';
 export * from './src/tools/topics.js';
 export * from './src/helpers/utils.js';
+export * from './src/helpers/async.js';
+export * from './src/helpers/urls.js';
 export * from './src/tools/brands.js';
 export * from './src/tools/translate.js';
 export * from './src/tools/sentiment.js';
@@ -23,6 +25,8 @@ export * from './src/tools/prompts.js';
 export * from './src/tools/scorer.js';
 export * from './src/helpers/seedKeywords.js';
 export * from './src/tools/generic.js';
+export * from './src/apis/hasdata/index.js';
+export * from './src/apis/brightdata/index.js';
 export * from './src/apis/chatgptScraper/index.js';
 export * from './src/apis/googleAds/keywordPlanner.js';
 export * from './src/schemas/index.js';

package/esm/mod.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,cAAc,cAAc,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EACN,WAAW,EACX,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,SAAS,GACd,MAAM,0BAA0B,CAAC;AAGlC,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,wBAAwB,CAAC;AACvC,cAAc,oCAAoC,CAAC;AACnD,cAAc,wCAAwC,CAAC;AACvD,cAAc,wBAAwB,CAAC"}
+{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,cAAc,cAAc,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EACN,WAAW,EACX,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,SAAS,GACd,MAAM,0BAA0B,CAAC;AAGlC,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,wBAAwB,CAAC;AACvC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,oCAAoC,CAAC;AACnD,cAAc,wCAAwC,CAAC;AACvD,cAAc,wBAAwB,CAAC"}

package/esm/mod.js

@@ -15,6 +15,8 @@ export * from './src/tools/personas.js';
 export * from './src/tools/search.js';
 export * from './src/tools/topics.js';
 export * from './src/helpers/utils.js';
+export * from './src/helpers/async.js';
+export * from './src/helpers/urls.js';
 export * from './src/tools/brands.js';
 export * from './src/tools/translate.js';
 export * from './src/tools/sentiment.js';
@@ -25,6 +27,8 @@ export * from './src/tools/prompts.js';
 export * from './src/tools/scorer.js';
 export * from './src/helpers/seedKeywords.js';
 export * from './src/tools/generic.js';
+export * from './src/apis/hasdata/index.js';
+export * from './src/apis/brightdata/index.js';
 export * from './src/apis/chatgptScraper/index.js';
 export * from './src/apis/googleAds/keywordPlanner.js';
 export * from './src/schemas/index.js';

package/esm/src/apis/brightdata/index.d.ts

@@ -0,0 +1,2 @@
+export * from './scrape.js';
+//# sourceMappingURL=index.d.ts.map

package/esm/src/apis/brightdata/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/brightdata/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC"}

package/esm/src/apis/brightdata/index.js

@@ -0,0 +1 @@
+export * from './scrape.js';

package/esm/src/apis/brightdata/scrape.d.ts

@@ -0,0 +1,30 @@
+import { type RetryConfig } from '../../helpers/async.js';
+export interface BrightDataScrapeOptions {
+    /** BrightData zone name. Defaults to "web_unlocker". */
+    zone?: string;
+    /** HTTP method for the target request. Defaults to "GET". */
+    method?: 'GET' | 'POST';
+    /** Country code for geo-targeting (e.g. "us", "gb", "de"). */
+    country?: string;
+    /** Request body for POST requests to the target URL. */
+    body?: string;
+    /** Additional headers to send to the target URL. */
+    headers?: Record<string, string>;
+    /** Custom retry configuration. */
+    retryConfig?: RetryConfig;
+}
+export interface BrightDataScrapeResponse {
+    url: string;
+    html?: string;
+}
+/**
+ * Scrape a single URL using BrightData Web Unlocker API.
+ * Returns raw HTML content.
+ */
+export declare function scrapeBrightData(url: string, options?: BrightDataScrapeOptions): Promise<BrightDataScrapeResponse>;
+/**
+ * Scrape multiple URLs in parallel using BrightData Web Unlocker API.
+ * Uses mapParallel with configurable concurrency.
+ */
+export declare function scrapeBrightDataBatch(urls: Array<string>, options?: BrightDataScrapeOptions, maxConcurrency?: number): Promise<Array<BrightDataScrapeResponse>>;
+//# sourceMappingURL=scrape.d.ts.map

package/esm/src/apis/brightdata/scrape.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/brightdata/scrape.ts"],"names":[],"mappings":"AACA,OAAO,EAA4B,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAapF,MAAM,WAAW,uBAAuB;IACpC,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB,8DAA8D;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,kCAAkC;IAClC,WAAW,CAAC,EAAE,WAAW,CAAC;CAC7B;AAED,MAAM,WAAW,wBAAwB;IACrC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AA4ED;;;GAGG;AACH,wBAAsB,gBAAgB,CAClC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,uBAA4B,GACtC,OAAO,CAAC,wBAAwB,CAAC,CAYnC;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CACvC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EACnB,OAAO,GAAE,uBAA4B,EACrC,cAAc,GAAE,MAA+B,GAChD,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,CAQ1C"}

package/esm/src/apis/brightdata/scrape.js

@@ -0,0 +1,93 @@
+import * as dntShim from "../../../_dnt.shims.js";
+import { mapParallel, withRetries } from '../../helpers/async.js';
+const BRIGHTDATA_CONCURRENCY = 10;
+const BRIGHTDATA_RETRY_CONFIG = {
+    maxRetries: 3,
+    initialDelay: 1000,
+    maxDelay: 8000,
+    backoffMultiplier: 2,
+    statusCodes: [429, 500, 502, 503]
+};
+function getApiKey() {
+    const apiKey = dntShim.Deno.env.get('BRIGHTDATA_API_KEY');
+    if (apiKey == null) {
+        throw new Error('BRIGHTDATA_API_KEY environment variable is required');
+    }
+    return apiKey;
+}
+async function fetchBrightData(url, apiKey, options, retryConfig) {
+    const response = await withRetries(async () => {
+        const body = {
+            zone: options.zone ?? 'web_unlocker',
+            url,
+            format: 'raw',
+            method: options.method ?? 'GET'
+        };
+        if (options.country != null) {
+            body.country = options.country;
+        }
+        if (options.body != null) {
+            body.body = options.body;
+        }
+        const headers = {
+            'Authorization': `Bearer ${apiKey}`,
+            'Content-Type': 'application/json'
+        };
+        if (options.headers != null) {
+            // Forward custom headers as part of the BrightData request
+            body.headers = options.headers;
+        }
+        return fetch('https://api.brightdata.com/request', {
+            method: 'POST',
+            headers,
+            body: JSON.stringify(body),
+            signal: dntShim.dntGlobalThis.abortSignal
+        });
+    }, retryConfig);
+    if (!response.ok) {
+        const status = response.status;
+        let details = '';
+        try {
+            details = ` - ${await response.text()}`;
+        }
+        catch {
+            // ignore
+        }
+        const errorMessage = status === 401
+            ? 'BrightData API error (401): Invalid API key'
+            : status === 403
+                ? 'BrightData API error (403): Forbidden or credits exhausted'
+                : status === 429
+                    ? 'BrightData API error (429): Rate limit exceeded'
+                    : `BrightData API error: ${status} ${response.statusText}${details}`;
+        console.error(errorMessage);
+        throw new Error(errorMessage);
+    }
+    return response;
+}
+/**
+ * Scrape a single URL using BrightData Web Unlocker API.
+ * Returns raw HTML content.
+ */
+export async function scrapeBrightData(url, options = {}) {
+    const apiKey = getApiKey();
+    const retryConfig = options.retryConfig ?? BRIGHTDATA_RETRY_CONFIG;
+    try {
+        const response = await fetchBrightData(url, apiKey, options, retryConfig);
+        const html = await response.text();
+        return { url, html };
+    }
+    catch (error) {
+        console.error(`BrightData scrape error for ${url}:`, error);
+        return { url };
+    }
+}
+/**
+ * Scrape multiple URLs in parallel using BrightData Web Unlocker API.
+ * Uses mapParallel with configurable concurrency.
+ */
+export async function scrapeBrightDataBatch(urls, options = {}, maxConcurrency = BRIGHTDATA_CONCURRENCY) {
+    return mapParallel(urls, maxConcurrency, async (url) => {
+        return await scrapeBrightData(url, options);
+    });
+}

package/esm/src/apis/hasdata/aim.d.ts

@@ -0,0 +1,4 @@
+import type { AIOParsed } from './helpers.js';
+export declare function fetchAIM(prompt: string, country?: string | null, language?: string | null, location?: string | null): Promise<AIOParsed>;
+export declare function fetchAIMBatch(prompts: Array<string>, country?: string | null, language?: string | null, location?: string | null, maxConcurrency?: number): Promise<Array<AIOParsed>>;
+//# sourceMappingURL=aim.d.ts.map

package/esm/src/apis/hasdata/aim.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aim.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/aim.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EACX,SAAS,EACT,MAAM,cAAc,CAAC;AAOtB,wBAAsB,QAAQ,CAC7B,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,MAAM,GAAG,IAAW,EAC7B,QAAQ,GAAE,MAAM,GAAG,IAAW,EAC9B,QAAQ,GAAE,MAAM,GAAG,IAAW,GAC5B,OAAO,CAAC,SAAS,CAAC,CA4BpB;AAED,wBAAsB,aAAa,CAClC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,EACtB,OAAO,GAAE,MAAM,GAAG,IAAW,EAC7B,QAAQ,GAAE,MAAM,GAAG,IAAW,EAC9B,QAAQ,GAAE,MAAM,GAAG,IAAW,EAC9B,cAAc,GAAE,MAA4B,GAC1C,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAM3B"}

package/esm/src/apis/hasdata/aim.js

@@ -0,0 +1,32 @@
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+import { mapParallel } from '../../helpers/async.js';
+import { fetchHasDataWithRetry, HASDATA_CONCURRENCY, parseAIM } from './helpers.js';
+export async function fetchAIM(prompt, country = null, language = null, location = null) {
+    const aimEndpoint = 'https://api.hasdata.com/scrape/google/ai-mode';
+    const params = { q: prompt };
+    if (location) {
+        params.location = location;
+    }
+    if (country) {
+        params.gl = country.toLowerCase();
+    }
+    if (language) {
+        params.hl = language.toLowerCase();
+    }
+    const url = new URL(aimEndpoint);
+    for (const [key, value] of Object.entries(params)) {
+        url.searchParams.set(key, value);
+    }
+    try {
+        const response = await fetchHasDataWithRetry(url.toString());
+        const content = await response.json();
+        return parseAIM(content);
+    }
+    catch (error) {
+        console.error('HasData AI Mode API error:', error);
+        return { answer: '', sources: [] };
+    }
+}
+export async function fetchAIMBatch(prompts, country = null, language = null, location = null, maxConcurrency = HASDATA_CONCURRENCY) {
+    return mapParallel(prompts, maxConcurrency, async (prompt) => fetchAIM(prompt, country, language, location));
+}

package/esm/src/apis/hasdata/aio.d.ts

@@ -0,0 +1,4 @@
+import { type AIOParsed } from './helpers.js';
+export declare function fetchAIO(prompt: string, country?: string | null, language?: string | null): Promise<AIOParsed>;
+export declare function fetchAIOBatch(prompts: Array<string>, country?: string | null, language?: string | null, maxConcurrency?: number): Promise<Array<AIOParsed>>;
+//# sourceMappingURL=aio.d.ts.map

package/esm/src/apis/hasdata/aio.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"aio.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/aio.ts"],"names":[],"mappings":"AAGA,OAAO,EAIN,KAAK,SAAS,EAEd,MAAM,cAAc,CAAC;AAStB,wBAAsB,QAAQ,CAC7B,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,MAAM,GAAG,IAAW,EAC7B,QAAQ,GAAE,MAAM,GAAG,IAAW,GAC5B,OAAO,CAAC,SAAS,CAAC,CAiCpB;AAED,wBAAsB,aAAa,CAClC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,EACtB,OAAO,GAAE,MAAM,GAAG,IAAW,EAC7B,QAAQ,GAAE,MAAM,GAAG,IAAW,EAC9B,cAAc,GAAE,MAA4B,GAC1C,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAM3B"}

package/esm/src/apis/hasdata/aio.js

@@ -0,0 +1,42 @@
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+import { mapParallel } from '../../helpers/async.js';
+import { fetchHasDataWithRetry, HASDATA_CONCURRENCY, parseAIO } from './helpers.js';
+function aioRequestUrl(aio) {
+    if (aio.pageToken && aio.hasdataLink) {
+        return aio.hasdataLink;
+    }
+    return null;
+}
+export async function fetchAIO(prompt, country = null, language = null) {
+    const serpEndpoint = 'https://api.hasdata.com/scrape/google/serp';
+    const params = { q: prompt };
+    if (country) {
+        params.gl = country.toLowerCase();
+    }
+    if (language) {
+        params.hl = language.toLowerCase();
+    }
+    const url = new URL(serpEndpoint);
+    for (const [key, value] of Object.entries(params)) {
+        url.searchParams.set(key, value);
+    }
+    try {
+        let response = await fetchHasDataWithRetry(url.toString());
+        let content = await response.json();
+        let aio = content.aiOverview || {};
+        const aioUrlString = aioRequestUrl(aio);
+        if (aioUrlString) {
+            response = await fetchHasDataWithRetry(aioUrlString);
+            content = await response.json();
+            aio = content.aiOverview || {};
+        }
+        return parseAIO(aio);
+    }
+    catch (error) {
+        console.error('HasData API error:', error);
+        return { answer: '', sources: [] };
+    }
+}
+export async function fetchAIOBatch(prompts, country = null, language = null, maxConcurrency = HASDATA_CONCURRENCY) {
+    return mapParallel(prompts, maxConcurrency, async (prompt) => fetchAIO(prompt, country, language));
+}

package/esm/src/apis/hasdata/helpers.d.ts

@@ -0,0 +1,55 @@
+import { type RetryConfig } from '../../helpers/async.js';
+import type { Source } from '../../schemas/sources.schema.js';
+export declare const HASDATA_CONCURRENCY = 29;
+export declare const HASDATA_RETRY_CONFIG: RetryConfig;
+export declare function getHasDataApiKey(): string;
+export declare function fetchHasDataWithRetry(url: string, retryConfig?: RetryConfig): Promise<Response>;
+interface ListItem {
+    title?: string;
+    snippet?: string;
+    list?: Array<ListItem>;
+}
+interface TextBlock {
+    type?: string;
+    snippet?: string;
+    snippetHighlightedWords?: Array<string>;
+    referenceIndexes?: Array<number>;
+    list?: Array<ListItem>;
+    rows?: Array<Array<string>>;
+    thumbnail?: string;
+    language?: string;
+}
+interface Reference {
+    index?: number;
+    title?: string;
+    link?: string;
+    url?: string;
+    snippet?: string;
+    source?: string;
+}
+export interface AIOverview {
+    textBlocks?: Array<TextBlock>;
+    references?: Array<Reference>;
+    aiOverview?: AIOverview;
+    pageToken?: string;
+    hasdataLink?: string;
+}
+interface RequestMetadata {
+    id?: string;
+    status?: string;
+    html?: string;
+    url?: string;
+}
+export interface AIMode {
+    requestMetadata?: RequestMetadata;
+    textBlocks?: Array<TextBlock>;
+    references?: Array<Reference>;
+}
+export interface AIOParsed {
+    answer: string;
+    sources: Array<Source>;
+}
+export declare function parseAIO(aio: AIOverview): AIOParsed;
+export declare function parseAIM(aim: AIMode): AIOParsed;
+export {};
+//# sourceMappingURL=helpers.d.ts.map

package/esm/src/apis/hasdata/helpers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;AAwJD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}

package/esm/src/apis/hasdata/helpers.js

@@ -0,0 +1,182 @@
+import * as dntShim from "../../../_dnt.shims.js";
+import { withRetries } from '../../helpers/async.js';
+import { extractDomain } from '../../helpers/urls.js';
+export const HASDATA_CONCURRENCY = 29;
+export const HASDATA_RETRY_CONFIG = {
+    maxRetries: 3,
+    initialDelay: 1000,
+    maxDelay: 8000,
+    backoffMultiplier: 2,
+    statusCodes: [429, 500]
+};
+export function getHasDataApiKey() {
+    const apiKey = dntShim.Deno.env.get('HASDATA_API_KEY');
+    if (!apiKey) {
+        throw new Error('HASDATA_API_KEY environment variable is required');
+    }
+    return apiKey;
+}
+export async function fetchHasDataWithRetry(url, retryConfig = HASDATA_RETRY_CONFIG) {
+    const headers = {
+        'x-api-key': getHasDataApiKey()
+    };
+    const response = await withRetries(async () => fetch(url, {
+        headers,
+        signal: dntShim.dntGlobalThis.abortSignal
+    }), retryConfig);
+    if (!response.ok) {
+        const status = response.status;
+        let errorMessage;
+        if (status === 401) {
+            errorMessage = 'HasData API error (401): Invalid API key';
+        }
+        else if (status === 403) {
+            errorMessage = 'HasData API error (403): API credits exhausted';
+        }
+        else if (status === 404) {
+            errorMessage = 'HasData API error (404): Page not found';
+        }
+        else {
+            errorMessage = `HasData API error: ${status} ${response.statusText}`;
+        }
+        console.error(errorMessage);
+        throw new Error(errorMessage);
+    }
+    return response;
+}
+function removeCSSChunks(text) {
+    if (!text) {
+        return '';
+    }
+    // Remove CSS blocks that start with :root (anchored pattern - safe)
+    text = text.replace(/:root\{[^}]*\}(?:@supports[^}]*\{[^}]*\{[^}]*\}\})?(?:\.[a-zA-Z0-9_-]+\{[^}]*\})*\.?/g, '');
+    // Remove standalone @supports blocks (less common but safe anchor)
+    text = text.replace(/@supports[^\{]*\{(?:[^{}]|\{[^}]*\})*\}/g, '');
+    // Only remove class blocks if they appear in suspicious patterns (3+ consecutive)
+    text = text.replace(/(?:\.[a-zA-Z0-9_-]+\{[^}]*\}){3,}/g, '');
+    return text;
+}
+function cleanText(text) {
+    if (!text) {
+        return '';
+    }
+    text = removeCSSChunks(text);
+    text = text.replace(/\u00a0/g, ' ');
+    text = text.replace(/[ \t]+/g, ' ');
+    const lines = text.split('\n').map(line => line.trim());
+    const cleaned = [];
+    for (const line of lines) {
+        if (line || (cleaned.length > 0 && cleaned[cleaned.length - 1])) {
+            cleaned.push(line);
+        }
+    }
+    return cleaned.join('\n').trim();
+}
+function* iterListItems(items, indent = 0) {
+    const prefix = '  '.repeat(indent) + '- ';
+    for (const obj of items) {
+        const title = obj.title || '';
+        const snippet = obj.snippet || '';
+        let line;
+        if (title && snippet && title.endsWith(':')) {
+            line = `${title} ${snippet}`.trim();
+        }
+        else {
+            line = [title, snippet].filter(p => p).join(' ').trim();
+        }
+        if (line) {
+            yield prefix + cleanText(line);
+        }
+        if (obj.list && Array.isArray(obj.list)) {
+            yield* iterListItems(obj.list, indent + 1);
+        }
+    }
+}
+function formatTable(block) {
+    const rows = block.rows || [];
+    if (rows.length === 0) {
+        return '';
+    }
+    const out = [];
+    const header = rows[0].map(cell => removeCSSChunks(cell));
+    out.push('| ' + header.join(' | ') + ' |');
+    out.push('| ' + header.map(() => '---').join(' | ') + ' |');
+    for (let i = 1; i < rows.length; i++) {
+        const cleanedRow = rows[i].map(cell => removeCSSChunks(cell));
+        out.push('| ' + cleanedRow.join(' | ') + ' |');
+    }
+    return out.join('\n');
+}
+function formatCode(block) {
+    const lang = block.language || '';
+    const snippet = block.snippet || '';
+    if (!snippet) {
+        return '';
+    }
+    const header = `[Code${lang ? ': ' + lang : ''}]`;
+    return `${header}\n${snippet.trim()}`;
+}
+function parseAIResult(data, { allowNestedOverview = true } = {}) {
+    const textBlocks = data.textBlocks || (allowNestedOverview ? data.aiOverview?.textBlocks : []) || [];
+    const parts = [];
+    const handlers = {
+        paragraph: (b) => cleanText(b.snippet || ''),
+        list: (b) => Array.from(iterListItems(b.list || [])).join('\n'),
+        table: formatTable,
+        code: formatCode
+    };
+    for (const block of textBlocks) {
+        const btype = block.type || (block.snippet ? 'paragraph' : null);
+        if (!btype || btype === 'carousel') {
+            continue;
+        }
+        const handler = handlers[btype];
+        if (handler) {
+            const rendered = handler(block);
+            if (rendered) {
+                parts.push(rendered);
+            }
+        }
+        else {
+            const snippet = block.snippet || '';
+            if (snippet) {
+                parts.push(cleanText(snippet));
+            }
+        }
+    }
+    const deduped = [];
+    for (const p of parts) {
+        if (deduped.length === 0 || deduped[deduped.length - 1] !== p) {
+            deduped.push(p);
+        }
+    }
+    let answer = cleanText(deduped.join('\n\n'));
+    if (answer.length > 16000) {
+        console.warn('Warning: AI answer truncated to 16000 characters');
+        answer = answer.slice(0, 16000);
+    }
+    const refs = data.references || (allowNestedOverview ? data.aiOverview?.references : []) || [];
+    const sources = [];
+    for (const r of refs) {
+        const link = r.link || r.url;
+        const title = [r.title, r.source, r.snippet].filter(Boolean).join(' - ');
+        if (link) {
+            sources.push({
+                title,
+                url: link,
+                domain: extractDomain(link)
+            });
+        }
+    }
+    return { answer, sources };
+}
+export function parseAIO(aio) {
+    return parseAIResult(aio, {
+        allowNestedOverview: true
+    });
+}
+export function parseAIM(aim) {
+    return parseAIResult(aim, {
+        allowNestedOverview: false
+    });
+}

package/esm/src/apis/hasdata/index.d.ts

@@ -0,0 +1,6 @@
+export * from './helpers.js';
+export * from './aio.js';
+export * from './aim.js';
+export * from './serp.js';
+export * from './scrape.js';
+//# sourceMappingURL=index.d.ts.map

package/esm/src/apis/hasdata/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC"}

package/esm/src/apis/hasdata/index.js

@@ -0,0 +1,5 @@
+export * from './helpers.js';
+export * from './aio.js';
+export * from './aim.js';
+export * from './serp.js';
+export * from './scrape.js';

package/esm/src/apis/hasdata/scrape.d.ts

@@ -0,0 +1,73 @@
+type ProxyType = 'datacenter' | 'residential';
+type OutputFormat = 'markdown' | 'text' | 'html';
+interface JSScenarioAction {
+    click?: string;
+    fill?: [string, string];
+    wait?: number;
+    waitFor?: string;
+    scroll?: string;
+    evaluate?: string;
+}
+export interface ScrapeOptions {
+    formats: Array<OutputFormat>;
+    proxyType?: ProxyType;
+    proxyCountry?: string;
+    extractLinks?: boolean;
+    wait?: number;
+    waitFor?: string;
+    blockResources?: boolean;
+    blockAds?: boolean;
+    blockUrls?: Array<string>;
+    jsRendering?: boolean;
+    jsScenario?: Array<JSScenarioAction>;
+    headers?: Record<string, string>;
+}
+export interface ScrapeResponse {
+    url?: string;
+    markdown?: string;
+    text?: string;
+    html?: string;
+    links?: Array<string>;
+}
+export interface BatchJobResponse {
+    jobId: string;
+    status: string;
+}
+export interface BatchJobStatus {
+    jobId: string;
+    status: string;
+    data: {
+        status: string;
+        requestsCount: number;
+        responsesCount: number;
+    };
+}
+/**
+ * In batch jobs, results are only links to json files containing the actual scrape results.
+*/
+export interface BatchResultItem {
+    query: Record<string, unknown>;
+    result: {
+        id: string;
+        status: string;
+        json?: string;
+    };
+}
+export interface BatchResults {
+    page: number;
+    limit: number;
+    total: number;
+    results: Array<BatchResultItem>;
+}
+export declare function scrapeWeb(url: string, options: ScrapeOptions): Promise<ScrapeResponse>;
+export declare function scrapeWebBatch(urls: Array<string>, options: ScrapeOptions, maxConcurrency?: number): Promise<Array<ScrapeResponse>>;
+/** Submit a batch scrape job to HasData API.
+ * IMPORTANT: results are not returned in original order! You need to match them by jobId and query.url.
+*/
+export declare function submitBatchScrapeJob(urls: Array<string>, options: ScrapeOptions): Promise<BatchJobResponse>;
+export declare function getBatchJobStatus(jobId: string): Promise<BatchJobStatus>;
+export declare function waitForBatchCompletion(jobId: string, pollInterval?: number, maxWaitTime?: number): Promise<BatchJobStatus>;
+export declare function getBatchJobPage(jobId: string, page?: number, limit?: number): Promise<BatchResults>;
+export declare function runBatchScrape(urls: Array<string>, options: ScrapeOptions, pageSize?: number, pollInterval?: number, maxWaitTime?: number): Promise<Array<ScrapeResponse>>;
+export {};
+//# sourceMappingURL=scrape.d.ts.map

package/esm/src/apis/hasdata/scrape.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/scrape.ts"],"names":[],"mappings":"AAiBA,KAAK,SAAS,GAAG,YAAY,GAAG,aAAa,CAAC;AAE9C,KAAK,YAAY,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;AAEjD,UAAU,gBAAgB;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC7B,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC7B,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,SAAS,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,cAAc;IAC9B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE;QACL,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,EAAE,MAAM,CAAC;QACtB,cAAc,EAAE,MAAM,CAAC;KACvB,CAAA;CACD;AAED;;EAEE;AAEF,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,MAAM,EAAE;QACP,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,CAAC,EAAE,MAAM,CAAC;KACd,CAAC;CACF;AAED,MAAM,WAAW,YAAY;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;CAChC;AAkJD,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,CAAC,CAwC5F;AAED,wBAAsB,cAAc,CACnC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EACnB,OAAO,EAAE,aAAa,EACtB,cAAc,GAAE,MAA4B,GAC1C,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAQhC;AAED;;EAEE;AACF,wBAAsB,oBAAoB,CACzC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EACnB,OAAO,EAAE,aAAa,GACpB,OAAO,CAAC,gBAAgB,CAAC,CA8B3B;AAED,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAqB9E;AAED,wBAAsB,sBAAsB,CAC3C,KAAK,EAAE,MAAM,EACb,YAAY,GAAE,MAAa,EAC3B,WAAW,GAAE,MAAe,GAC1B,OAAO,CAAC,cAAc,CAAC,CAsBzB;AAED,wBAAsB,eAAe,CACpC,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,MAAU,EAChB,KAAK,GAAE,MAAY,GACjB,OAAO,CAAC,YAAY,CAAC,CAuBvB;AAED,wBAAsB,cAAc,CACnC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EACnB,OAAO,EAAE,aAAa,EACtB,QAAQ,GAAE,MAAY,EACtB,YAAY,GAAE,MAAa,EAC3B,WAAW,GAAE,MAAe,GAC1B,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CA8DhC"}