@graphext/cuery 0.7.0 → 0.8.2

package/esm/mod.d.ts

@@ -13,6 +13,8 @@ export * from './src/tools/personas.js';
 export * from './src/tools/search.js';
 export * from './src/tools/topics.js';
 export * from './src/helpers/utils.js';
+export * from './src/helpers/async.js';
+export * from './src/helpers/urls.js';
 export * from './src/tools/brands.js';
 export * from './src/tools/translate.js';
 export * from './src/tools/sentiment.js';
@@ -20,9 +22,10 @@ export * from './src/tools/summarize.js';
 export * from './src/tools/sources.js';
 export * from './src/tools/entities.js';
 export * from './src/tools/prompts.js';
+export * from './src/tools/scorer.js';
 export * from './src/helpers/seedKeywords.js';
 export * from './src/tools/generic.js';
-export * from './src/api.js';
+export * from './src/apis/hasdata/index.js';
 export * from './src/apis/chatgptScraper/index.js';
 export * from './src/apis/googleAds/keywordPlanner.js';
 export * from './src/schemas/index.js';

package/esm/mod.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,cAAc,cAAc,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EACN,WAAW,EACX,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,SAAS,GACd,MAAM,0BAA0B,CAAC;AAGlC,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,wBAAwB,CAAC;AACvC,cAAc,cAAc,CAAC;AAC7B,cAAc,oCAAoC,CAAC;AACnD,cAAc,wCAAwC,CAAC;AACvD,cAAc,wBAAwB,CAAC"}
+{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,cAAc,cAAc,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EACN,WAAW,EACX,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,SAAS,GACd,MAAM,0BAA0B,CAAC;AAGlC,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,wBAAwB,CAAC;AACvC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oCAAoC,CAAC;AACnD,cAAc,wCAAwC,CAAC;AACvD,cAAc,wBAAwB,CAAC"}

package/esm/mod.js

@@ -15,6 +15,8 @@ export * from './src/tools/personas.js';
 export * from './src/tools/search.js';
 export * from './src/tools/topics.js';
 export * from './src/helpers/utils.js';
+export * from './src/helpers/async.js';
+export * from './src/helpers/urls.js';
 export * from './src/tools/brands.js';
 export * from './src/tools/translate.js';
 export * from './src/tools/sentiment.js';
@@ -22,9 +24,10 @@ export * from './src/tools/summarize.js';
 export * from './src/tools/sources.js';
 export * from './src/tools/entities.js';
 export * from './src/tools/prompts.js';
+export * from './src/tools/scorer.js';
 export * from './src/helpers/seedKeywords.js';
 export * from './src/tools/generic.js';
-export * from './src/api.js';
+export * from './src/apis/hasdata/index.js';
 export * from './src/apis/chatgptScraper/index.js';
 export * from './src/apis/googleAds/keywordPlanner.js';
 export * from './src/schemas/index.js';

package/esm/src/apis/hasdata/index.d.ts

@@ -0,0 +1,6 @@
+export * from './helpers.js';
+export * from './aio.js';
+export * from './aim.js';
+export * from './serp.js';
+export * from './scrape.js';
+//# sourceMappingURL=index.d.ts.map

package/esm/src/apis/hasdata/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/index.ts"],"names":[],"mappings":"AAAA,cAAc,cAAc,CAAC;AAC7B,cAAc,UAAU,CAAC;AACzB,cAAc,UAAU,CAAC;AACzB,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC"}

package/esm/src/apis/hasdata/index.js

@@ -0,0 +1,5 @@
+export * from './helpers.js';
+export * from './aio.js';
+export * from './aim.js';
+export * from './serp.js';
+export * from './scrape.js';

package/esm/src/apis/hasdata/scrape.d.ts

@@ -0,0 +1,73 @@
+type ProxyType = 'datacenter' | 'residential';
+type OutputFormat = 'markdown' | 'text' | 'html';
+interface JSScenarioAction {
+    click?: string;
+    fill?: [string, string];
+    wait?: number;
+    waitFor?: string;
+    scroll?: string;
+    evaluate?: string;
+}
+export interface ScrapeOptions {
+    formats: Array<OutputFormat>;
+    proxyType?: ProxyType;
+    proxyCountry?: string;
+    extractLinks?: boolean;
+    wait?: number;
+    waitFor?: string;
+    blockResources?: boolean;
+    blockAds?: boolean;
+    blockUrls?: Array<string>;
+    jsRendering?: boolean;
+    jsScenario?: Array<JSScenarioAction>;
+    headers?: Record<string, string>;
+}
+export interface ScrapeResponse {
+    url?: string;
+    markdown?: string;
+    text?: string;
+    html?: string;
+    links?: Array<string>;
+}
+export interface BatchJobResponse {
+    jobId: string;
+    status: string;
+}
+export interface BatchJobStatus {
+    jobId: string;
+    status: string;
+    data: {
+        status: string;
+        requestsCount: number;
+        responsesCount: number;
+    };
+}
+/**
+ * In batch jobs, results are only links to json files containing the actual scrape results.
+*/
+export interface BatchResultItem {
+    query: Record<string, unknown>;
+    result: {
+        id: string;
+        status: string;
+        json?: string;
+    };
+}
+export interface BatchResults {
+    page: number;
+    limit: number;
+    total: number;
+    results: Array<BatchResultItem>;
+}
+export declare function scrapeWeb(url: string, options: ScrapeOptions): Promise<ScrapeResponse>;
+export declare function scrapeWebBatch(urls: Array<string>, options: ScrapeOptions, maxConcurrency?: number): Promise<Array<ScrapeResponse>>;
+/** Submit a batch scrape job to HasData API.
+ * IMPORTANT: results are not returned in original order! You need to match them by jobId and query.url.
+*/
+export declare function submitBatchScrapeJob(urls: Array<string>, options: ScrapeOptions): Promise<BatchJobResponse>;
+export declare function getBatchJobStatus(jobId: string): Promise<BatchJobStatus>;
+export declare function waitForBatchCompletion(jobId: string, pollInterval?: number, maxWaitTime?: number): Promise<BatchJobStatus>;
+export declare function getBatchJobPage(jobId: string, page?: number, limit?: number): Promise<BatchResults>;
+export declare function runBatchScrape(urls: Array<string>, options: ScrapeOptions, pageSize?: number, pollInterval?: number, maxWaitTime?: number): Promise<Array<ScrapeResponse>>;
+export {};
+//# sourceMappingURL=scrape.d.ts.map

package/esm/src/apis/hasdata/scrape.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/scrape.ts"],"names":[],"mappings":"AAiBA,KAAK,SAAS,GAAG,YAAY,GAAG,aAAa,CAAC;AAE9C,KAAK,YAAY,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,CAAC;AAEjD,UAAU,gBAAgB;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,aAAa;IAC7B,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC7B,SAAS,CAAC,EAAE,SAAS,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,SAAS,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC1B,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,UAAU,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,CAAC;IACrC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,cAAc;IAC9B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,cAAc;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE;QACL,MAAM,EAAE,MAAM,CAAC;QACf,aAAa,EAAE,MAAM,CAAC;QACtB,cAAc,EAAE,MAAM,CAAC;KACvB,CAAA;CACD;AAED;;EAEE;AAEF,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B,MAAM,EAAE;QACP,EAAE,EAAE,MAAM,CAAC;QACX,MAAM,EAAE,MAAM,CAAC;QACf,IAAI,CAAC,EAAE,MAAM,CAAC;KACd,CAAC;CACF;AAED,MAAM,WAAW,YAAY;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;CAChC;AAkJD,wBAAsB,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,cAAc,CAAC,CAwC5F;AAED,wBAAsB,cAAc,CACnC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EACnB,OAAO,EAAE,aAAa,EACtB,cAAc,GAAE,MAA4B,GAC1C,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAQhC;AAED;;EAEE;AACF,wBAAsB,oBAAoB,CACzC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EACnB,OAAO,EAAE,aAAa,GACpB,OAAO,CAAC,gBAAgB,CAAC,CA8B3B;AAED,wBAAsB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAqB9E;AAED,wBAAsB,sBAAsB,CAC3C,KAAK,EAAE,MAAM,EACb,YAAY,GAAE,MAAa,EAC3B,WAAW,GAAE,MAAe,GAC1B,OAAO,CAAC,cAAc,CAAC,CAsBzB;AAED,wBAAsB,eAAe,CACpC,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,MAAU,EAChB,KAAK,GAAE,MAAY,GACjB,OAAO,CAAC,YAAY,CAAC,CAuBvB;AAED,wBAAsB,cAAc,CACnC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EACnB,OAAO,EAAE,aAAa,EACtB,QAAQ,GAAE,MAAY,EACtB,YAAY,GAAE,MAAa,EAC3B,WAAW,GAAE,MAAe,GAC1B,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CA8DhC"}

package/esm/src/apis/hasdata/scrape.js

@@ -0,0 +1,310 @@
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+import * as dntShim from "../../../_dnt.shims.js";
+import { mapParallel, withRetries } from '../../helpers/async.js';
+const HASDATA_CONCURRENCY = 29;
+const HASDATA_RETRY_CONFIG = {
+    maxRetries: 3,
+    initialDelay: 1000,
+    maxDelay: 8000,
+    backoffMultiplier: 2,
+    statusCodes: [429, 500]
+};
+function cleanMarkdown(markdown, excludeImages = true) {
+    if (!markdown) {
+        return '';
+    }
+    if (excludeImages) {
+        // Remove markdown images: ![alt text](url)
+        markdown = markdown.replace(/!\[([^\]]*)\]\([^)]+\)/g, '');
+        // Remove standalone "Image" text between line breaks (from plain text format)
+        markdown = markdown.replace(/\n\s*Image\s*\n/g, '\n');
+        // Clean up multiple consecutive newlines
+        markdown = markdown.replace(/\n{3,}/g, '\n\n').trim();
+    }
+    markdown = markdown.replace(/\u00a0/g, ' ');
+    markdown = markdown.replace(/[ \t]+/g, ' ');
+    const lines = markdown.split('\n').map(line => line.trim());
+    const cleaned = [];
+    for (const line of lines) {
+        if (line || (cleaned.length > 0 && cleaned[cleaned.length - 1])) {
+            cleaned.push(line);
+        }
+    }
+    return cleaned.join('\n').trim();
+}
+async function fetchWithRetry(url, options, retryConfig = HASDATA_RETRY_CONFIG) {
+    const response = await withRetries(async () => fetch(url, {
+        ...options,
+        signal: dntShim.dntGlobalThis.abortSignal
+    }), retryConfig);
+    if (!response.ok) {
+        const status = response.status;
+        let errorMessage;
+        if (status === 400) {
+            let details = '';
+            try {
+                const body = await response.text();
+                details = ` - ${body}`;
+            }
+            catch {
+            }
+            errorMessage = `HasData API error (400): Bad Request${details}`;
+        }
+        else if (status === 401) {
+            errorMessage = 'HasData API error (401): Invalid API key';
+        }
+        else if (status === 403) {
+            errorMessage = 'HasData API error (403): API credits exhausted';
+        }
+        else if (status === 404) {
+            errorMessage = 'HasData API error (404): Resource not found';
+        }
+        else if (status === 422) {
+            let details = '';
+            try {
+                const body = await response.text();
+                details = ` - ${body}`;
+            }
+            catch {
+            }
+            errorMessage = `HasData API error (422): Unprocessable Entity${details}`;
+        }
+        else if (status === 429) {
+            errorMessage = 'HasData API error (429): Rate limit exceeded';
+        }
+        else {
+            errorMessage = `HasData API error: ${status} ${response.statusText}`;
+        }
+        console.error(errorMessage);
+        throw new Error(errorMessage);
+    }
+    return response;
+}
+function configureRequestBody(body, options) {
+    const formats = [...options.formats];
+    if (!formats.includes('json')) {
+        formats.push('json');
+    }
+    body.outputFormat = formats;
+    if (options.proxyType) {
+        body.proxyType = options.proxyType;
+    }
+    if (options.proxyCountry) {
+        body.proxyCountry = options.proxyCountry;
+    }
+    if (options.extractLinks != null) {
+        body.extractLinks = options.extractLinks;
+    }
+    if (options.wait != null) {
+        body.wait = options.wait;
+    }
+    if (options.waitFor) {
+        body.waitFor = options.waitFor;
+    }
+    if (options.blockResources != null) {
+        body.blockResources = options.blockResources;
+    }
+    if (options.blockAds != null) {
+        body.blockAds = options.blockAds;
+    }
+    if (options.blockUrls) {
+        body.blockUrls = options.blockUrls;
+    }
+    if (options.jsRendering != null) {
+        body.jsRendering = options.jsRendering;
+    }
+    if (options.jsScenario) {
+        body.jsScenario = options.jsScenario;
+    }
+    if (options.headers) {
+        body.headers = options.headers;
+    }
+    return body;
+}
+function getApiKey() {
+    const apiKey = dntShim.Deno.env.get('HASDATA_API_KEY');
+    if (!apiKey) {
+        throw new Error('HASDATA_API_KEY environment variable is required');
+    }
+    return apiKey;
+}
+export async function scrapeWeb(url, options) {
+    const apiKey = getApiKey();
+    const endpoint = 'https://api.hasdata.com/scrape/web';
+    let requestBody = { url: url };
+    requestBody = configureRequestBody(requestBody, options);
+    try {
+        const response = await fetchWithRetry(endpoint, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'x-api-key': apiKey
+            },
+            body: JSON.stringify(requestBody)
+        });
+        const responseJson = await response.json();
+        const result = { url: url };
+        if (responseJson.markdown) {
+            result.markdown = cleanMarkdown(responseJson.markdown);
+        }
+        if (responseJson.text) {
+            result.text = responseJson.text;
+        }
+        if (responseJson.content) {
+            result.html = responseJson.content;
+        }
+        if (options.extractLinks && responseJson.links) {
+            result.links = responseJson.links;
+        }
+        return result;
+    }
+    catch (error) {
+        console.error('HasData Web Scraping API error:', error);
+        return {}; // Return an empty object on error
+    }
+}
+export async function scrapeWebBatch(urls, options, maxConcurrency = HASDATA_CONCURRENCY) {
+    return mapParallel(urls, maxConcurrency, async (url) => {
+        return await scrapeWeb(url, options);
+    });
+}
+/** Submit a batch scrape job to HasData API.
+ * IMPORTANT: results are not returned in original order! You need to match them by jobId and query.url.
+*/
+export async function submitBatchScrapeJob(urls, options) {
+    const apiKey = getApiKey();
+    const endpoint = 'https://api.hasdata.com/scrape/batch/web/';
+    const requestPayloads = urls.map((url) => {
+        let payload = { url: url };
+        payload = configureRequestBody(payload, options);
+        return payload;
+    });
+    const requestBody = { requests: requestPayloads };
+    try {
+        const response = await fetchWithRetry(endpoint, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'x-api-key': apiKey
+            },
+            body: JSON.stringify(requestBody)
+        });
+        return await response.json();
+    }
+    catch (error) {
+        console.error('HasData Batch Scrape submission error:', error);
+        throw error;
+    }
+}
+export async function getBatchJobStatus(jobId) {
+    const apiKey = getApiKey();
+    const endpoint = `https://api.hasdata.com/scrape/batch/web/${jobId}`;
+    try {
+        const response = await fetchWithRetry(endpoint, {
+            method: 'GET',
+            headers: {
+                'x-api-key': apiKey
+            }
+        });
+        const status = await response.json();
+        return status;
+    }
+    catch (error) {
+        console.error('HasData Batch Job status error:', error);
+        throw error;
+    }
+}
+export async function waitForBatchCompletion(jobId, pollInterval = 5000, maxWaitTime = 300000) {
+    const startTime = Date.now();
+    while (true) {
+        const status = await getBatchJobStatus(jobId);
+        const endStates = ['done', 'stopped', 'finished', 'failed'];
+        if (endStates.includes(status.data.status)) {
+            return status;
+        }
+        else {
+            const total = status.data.requestsCount;
+            const completed = status.data.responsesCount;
+            console.log(`Batch job ${jobId} in progress: ${completed}/${total} completed.`);
+        }
+        const elapsed = Date.now() - startTime;
+        if (elapsed >= maxWaitTime) {
+            throw new Error(`Batch job ${jobId} did not complete within ${maxWaitTime}ms`);
+        }
+        await new Promise(resolve => setTimeout(resolve, pollInterval));
+    }
+}
+export async function getBatchJobPage(jobId, page = 0, limit = 100) {
+    const apiKey = getApiKey();
+    const url = new URL(`https://api.hasdata.com/scrape/batch/web/${jobId}/results`);
+    url.searchParams.set('page', page.toString());
+    url.searchParams.set('limit', limit.toString());
+    console.log(`Fetching batch job results from: ${url.toString()}`);
+    try {
+        const response = await fetchWithRetry(url.toString(), {
+            method: 'GET',
+            headers: {
+                'x-api-key': apiKey
+            }
+        });
+        return await response.json();
+    }
+    catch (error) {
+        console.error('HasData Batch Job results error:', error);
+        throw error;
+    }
+}
+export async function runBatchScrape(urls, options, pageSize = 100, pollInterval = 5000, maxWaitTime = 300000) {
+    const { jobId } = await submitBatchScrapeJob(urls, options);
+    const status = await waitForBatchCompletion(jobId, pollInterval, maxWaitTime);
+    if (status.data.status === 'done') {
+        console.log(`Batch job ${jobId} finished successfully.`);
+    }
+    else {
+        throw new Error(`Batch job failed with status:\n${JSON.stringify(status, null, 2)}`);
+    }
+    const aggregatedResults = [];
+    let currentPage = 0;
+    let hasMore = true;
+    while (hasMore) {
+        const pageResults = await getBatchJobPage(jobId, currentPage, pageSize);
+        console.log(`Fetched page ${pageResults.page} with ${pageResults.results.length} results.`);
+        const scrapeResponses = await mapParallel(pageResults.results, HASDATA_CONCURRENCY, async (item) => {
+            if (item.result.status === 'ok' && item.result.json) {
+                try {
+                    const response = await fetchWithRetry(item.result.json, { method: 'GET' });
+                    const fullResponse = await response.json();
+                    const scrapeResponse = {
+                        url: item.query.url
+                    };
+                    if (options.formats.includes('markdown') && fullResponse.markdown) {
+                        scrapeResponse.markdown = cleanMarkdown(fullResponse.markdown);
+                    }
+                    if (options.formats.includes('text') && fullResponse.text) {
+                        scrapeResponse.text = fullResponse.text;
+                    }
+                    if (options.formats.includes('html') && fullResponse.content) {
+                        scrapeResponse.html = fullResponse.content;
+                    }
+                    if (options.extractLinks && fullResponse.links) {
+                        scrapeResponse.links = fullResponse.links;
+                    }
+                    return scrapeResponse;
+                }
+                catch (error) {
+                    console.error(`Failed to fetch result for ${item.query.url}:`, error);
+                    return {};
+                }
+            }
+            return {};
+        });
+        aggregatedResults.push(...scrapeResponses);
+        if (pageResults.results.length < pageSize || (pageResults.page + 1) * pageResults.limit >= pageResults.total) {
+            hasMore = false;
+        }
+        else {
+            currentPage += 1;
+        }
+    }
+    return aggregatedResults;
+}

package/esm/src/apis/hasdata/serp.d.ts

@@ -0,0 +1,152 @@
+import { type AIOverview, type AIOParsed } from './helpers.js';
+type SerpSearchType = 'all' | 'images' | 'videos' | 'news' | 'shopping' | 'local';
+export interface SerpRequestOptions {
+    location: string;
+    country: string;
+    language: string;
+    contentLanguage?: string;
+    domain?: string;
+    filters?: string | Array<string>;
+    safeSearch?: 'active' | 'off' | boolean;
+    filterResults?: boolean;
+    preventAutoCorrect?: boolean;
+    offset?: number;
+    resultsPerPage?: number;
+    type?: SerpSearchType;
+    device?: 'desktop' | 'mobile' | 'tablet';
+    placeId?: string;
+    lsig?: string;
+    entityId?: string;
+    encodedLocation?: string;
+    searchId?: string;
+}
+interface SerpInlineSiteLink {
+    title?: string;
+    link?: string;
+}
+interface SerpListSiteLink {
+    title?: string;
+    link?: string;
+    snippet?: string;
+}
+interface SerpRichSnippetTop {
+    extensions?: Array<string>;
+    detectedExtensions?: Record<string, string | number>;
+}
+interface SerpRichSnippet {
+    top?: SerpRichSnippetTop;
+}
+interface SerpSiteLinks {
+    inline?: Array<SerpInlineSiteLink>;
+    list?: Array<SerpListSiteLink>;
+}
+export interface SerpOrganicResult {
+    position?: number;
+    title?: string;
+    link?: string;
+    url?: string;
+    displayedLink?: string;
+    source?: string;
+    snippet?: string;
+    snippetHighlitedWords?: Array<string>;
+    images?: Array<string>;
+    richSnippet?: SerpRichSnippet;
+    sitelinks?: SerpSiteLinks;
+}
+export interface SerpRequestMetadata {
+    id?: string;
+    status?: string;
+    html?: string;
+    url?: string;
+}
+export interface SerpSearchInformation {
+    totalResults?: string;
+    formattedTotalResults?: string;
+    timeTaken?: number;
+    searchTime?: number;
+}
+export interface SerpLocalPlace {
+    position?: number;
+    title?: string;
+    rating?: number;
+    reviews?: number;
+    reviewsOriginal?: string;
+    address?: string;
+    hours?: string;
+    placeId?: string;
+    description?: string;
+}
+export interface SerpLocalResults {
+    places?: Array<SerpLocalPlace>;
+    moreLocationsLink?: string;
+}
+export interface SerpRelatedSearch {
+    query?: string;
+    link?: string;
+}
+export interface SerpRelatedQuestion {
+    question?: string;
+    snippet?: string;
+    link?: string;
+    title?: string;
+    displayedLink?: string;
+    date?: string;
+    list?: Array<string>;
+    table?: Array<Array<string>>;
+    aiOverview?: AIOverview;
+}
+export interface SerpPerspective {
+    index?: number;
+    author?: string;
+    source?: string;
+    duration?: string;
+    extensions?: Array<string>;
+    thumbnail?: string;
+    title?: string;
+    link?: string;
+    date?: string;
+    snippet?: string;
+}
+export interface SerpImmersiveProduct {
+    position?: number;
+    category?: string;
+    title?: string;
+    productId?: string;
+    productLink?: string;
+    price?: string;
+    extractedPrice?: number;
+    source?: string;
+    reviews?: number;
+    rating?: number;
+    delivery?: string;
+    extensions?: Array<string>;
+    thumbnail?: string;
+}
+export interface SerpPagination {
+    next?: string;
+    pages?: Array<Record<string, string>>;
+}
+export interface SerpResponse {
+    requestMetadata?: SerpRequestMetadata;
+    searchMetadata?: Record<string, unknown>;
+    searchParameters?: Record<string, unknown>;
+    searchInformation?: SerpSearchInformation;
+    organicResults?: Array<SerpOrganicResult>;
+    adsResults?: Array<Record<string, unknown>>;
+    localResults?: SerpLocalResults;
+    knowledgeGraph?: Record<string, unknown>;
+    relatedSearches?: Array<SerpRelatedSearch>;
+    topStories?: Array<Record<string, unknown>>;
+    peopleAlsoAsk?: Array<Record<string, unknown>>;
+    relatedQuestions?: Array<SerpRelatedQuestion>;
+    imagesResults?: Array<Record<string, unknown>>;
+    videosResults?: Array<Record<string, unknown>>;
+    perspectives?: Array<SerpPerspective>;
+    immersiveProducts?: Array<SerpImmersiveProduct>;
+    pagination?: SerpPagination;
+    aiOverview?: AIOParsed;
+}
+export declare function fetchSerp(query: string, options: SerpRequestOptions): Promise<SerpResponse>;
+export declare function fetchSerpBatch(queries: Array<string>, options: SerpRequestOptions, maxConcurrency?: number): Promise<Array<SerpResponse>>;
+export {};
+//# sourceMappingURL=serp.d.ts.map

package/esm/src/apis/hasdata/serp.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"serp.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/serp.ts"],"names":[],"mappings":"AAGA,OAAO,EAIN,KAAK,UAAU,EACf,KAAK,SAAS,EACd,MAAM,cAAc,CAAC;AAEtB,KAAK,cAAc,GAAG,KAAK,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,GAAG,UAAU,GAAG,OAAO,CAAC;AAUlF,MAAM,WAAW,kBAAkB;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,UAAU,CAAC,EAAE,QAAQ,GAAG,KAAK,GAAG,OAAO,CAAC;IACxC,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,IAAI,CAAC,EAAE,cAAc,CAAC;IACtB,MAAM,CAAC,EAAE,SAAS,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,kBAAkB;IAC3B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;CACd;AAED,UAAU,gBAAgB;IACzB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,UAAU,kBAAkB;IAC3B,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3B,kBAAkB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;CACrD;AAED,UAAU,eAAe;IACxB,GAAG,CAAC,EAAE,kBAAkB,CAAC;CACzB;AAED,UAAU,aAAa;IACtB,MAAM,CAAC,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC;IACnC,IAAI,CAAC,EAAE,KAAK,CAAC,gBAAgB,CAAC,CAAC;CAC/B;AAED,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qBAAqB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACtC,MAAM,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,WAAW,CAAC,EAAE,eAAe,CAAC;IAC9B,SAAS,CAAC,EAAE,aAAa,CAAC;CAC1B;AAED,MAAM,WAAW,mBAAmB;IACnC,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,qBAAqB;IACrC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAChC,MAAM,CAAC,EAAE,KAAK,CAAC,cAAc,CAAC,CAAC;IAC/B,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,iBAAiB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,mBAAmB;IACnC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACrB,KAAK,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC7B,UAAU,CAAC,EAAE,UAAU,CAAC;CACxB;AAED,MAAM,WAAW,eAAe;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CACtC;AAED,MAAM,WAAW,YAAY;IAC5B,eAAe,CAAC,EAAE,mBAAmB,CAAC;IACtC,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACzC,gBAAgB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3C,iBAAiB,CAAC,EAAE,qBAAqB,CAAC;IAC1C,cAAc,CAAC,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAC1C,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC5C,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACzC,eAAe,CAAC,EAAE,KAAK,CAAC,iBAAiB,CAAC,CAAC;IAC3C,UAAU,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC5C,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC/C,gBAAgB,CAAC,EAAE,KAAK,CAAC,mBAAmB,CAAC,CAAC;IAC9C,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC/C,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC/C,YAAY,CAAC,EAAE,KAAK,CAAC,eAAe,CAAC,CAAC;IACtC,iBAAiB,CAAC,EAAE,KAAK,CAAC,oBAAoB,CAAC,CAAC;IAChD,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B,UAAU,CAAC,EAAE,SAAS,CAAC;CACvB;AA0ID,wBAAsB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,kBAAkB,GAAG,OAAO,CAAC,YAAY,CAAC,CAMjG;AAED,wBAAsB,cAAc,CACnC,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,EACtB,OAAO,EAAE,kBAAkB,EAC3B,cAAc,GAAE,MAA4B,GAC1C,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAY9B"}