@graphext/cuery 0.2.0

package/esm/src/api.js

@@ -0,0 +1,161 @@
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+import { fetchAIMBatch } from './apis/hasdata/aim.js';
+import { fetchAIOBatch } from './apis/hasdata/aio.js';
+import { downloadGPTSnapshots, scrapeGPTBatch } from './apis/chatgptScraper/index.js';
+import { classifyBatch, labelBatch } from './tools/classifier.js';
+import { extractEntitiesBatch } from './tools/entities.js';
+import { funnelToTopics } from './tools/funnel.js';
+import { ModelId } from './models.js';
+import { scoreBatch } from './tools/scorer.js';
+import { extractTopics, assignTopics } from './tools/topics.js';
+import { dedent } from './utils.js';
+export async function queryModel(prompts, model, searchCountry = null) {
+    const modelId = model instanceof ModelId ? model : new ModelId(model);
+    let searchResults;
+    if (modelId.equals('google/ai-overview')) {
+        searchResults = await fetchAIOBatch(prompts, searchCountry, null);
+    }
+    else if (modelId.equals('google/ai-mode')) {
+        searchResults = await fetchAIMBatch(prompts, searchCountry, null);
+    }
+    else if (modelId.provider == 'openai' || modelId.name.includes('gpt-') || modelId.name.includes('chatgpt')) {
+        // JobIds are alphanumeric strings without spaces
+        // (e.g., "7420410504197219329" for Oxylabs, "s_xxxxx" for Brightdata)
+        // Prompts are natural language text with spaces
+        const firstPrompt = prompts[0];
+        const isJobIdArray = prompts.length === 0 || firstPrompt == null || !firstPrompt.includes(' ');
+        if (isJobIdArray) {
+            searchResults = await downloadGPTSnapshots(prompts);
+        }
+        else {
+            searchResults = await scrapeGPTBatch({
+                prompts: prompts,
+                countryISOCode: searchCountry
+            });
+        }
+    }
+    else {
+        throw new Error(`Unsupported model: ${modelId}`);
+    }
+    const responses = searchResults.map((sr, i) => ({
+        answer: sr.answer,
+        sources: sr.sources,
+        prompt: sr.prompt ?? (typeof prompts[i] === 'string' ? prompts[i] : ''),
+        model: modelId.columnName(),
+        searchQueries: sr.searchQueries ?? [],
+        searchSources: sr.searchSources ?? []
+    }));
+    return responses;
+}
+const intentLabels = {
+    'informational': 'The user is seeking information or answers to questions about a topic.',
+    'navigational': 'The user is trying to find a specific website or page.',
+    'transactional': 'The user intends to complete a transaction, such as making a purchase or signing up for a service.'
+};
+export async function classifyIntent(texts) {
+    console.log('Classifying prompt intents...');
+    const textRecords = texts.map(text => ({ text }));
+    const intents = await classifyBatch(textRecords, intentLabels, 'Classify the search query into one of the following intents: informational, navigational, transactional.', 'gpt-4.1-mini');
+    return intents;
+}
+export async function extractAndAssignTopics(texts) {
+    console.log('Identifying topics...');
+    const textRecords = texts.map(text => ({ text }));
+    const taxonomy = await extractTopics({
+        records: textRecords,
+        maxSamples: 500
+    });
+    console.log('Assigning topics...');
+    const topicLabels = await assignTopics(texts, taxonomy);
+    return topicLabels.map(label => ({
+        topic: label?.topic ?? null,
+        subtopic: label?.subtopic ?? null
+    }));
+}
+export async function assignFunnelStages(texts, funnel) {
+    console.log('Assigning funnel stages...');
+    const funnelTopics = funnelToTopics(funnel);
+    const funnelLabels = await assignTopics(texts, funnelTopics);
+    return funnelLabels.map(label => ({
+        funnelStage: label?.topic ?? null,
+        funnelCategory: label?.subtopic ?? null
+    }));
+}
+export async function classifyIntoPersonas(texts, personas) {
+    console.log('Classifying prompts into personas...');
+    const personaLabels = {};
+    personas.forEach(persona => {
+        personaLabels[persona.name] = persona.description;
+    });
+    const textRecords = texts.map(text => ({ text }));
+    const personaAssignments = await labelBatch(textRecords, personaLabels, 'Classify the search query into one or more customer personas based on the language, intent, and context.', 'gpt-4.1-mini');
+    return personaAssignments;
+}
+export async function classifyBrandedNonBranded(texts) {
+    console.log('Classifying prompts into branded/non-branded...');
+    const brandedLabels = {
+        'branded': 'The query explicitly mentions a specific brand name, company name, product name, or trademark. It shows clear brand awareness and intent to find information about that particular brand.',
+        'non-branded': 'The query is generic and does not mention any specific brand names. It focuses on product categories, features, problems, or general information without brand specificity.'
+    };
+    const textRecords = texts.map(text => ({ text }));
+    const brandedClassifications = await classifyBatch(textRecords, brandedLabels, 'Classify whether the search query mentions specific brands or is generic/category-based.', 'gpt-4.1-mini');
+    return brandedClassifications;
+}
+export async function extractEntities(texts) {
+    const entityDefinitions = `
+		- brands: Any brand or companies mentioned
+		- products: Any products or services mentioned
+		- features: Specific features or attributes of products/services
+		- issues: Problems or issues mentioned
+	`;
+    return extractEntitiesBatch(texts, entityDefinitions, '', 'gpt-4.1-mini');
+}
+export async function scorePurchaseProbability(texts, numDays = 30) {
+    console.log('Scoring purchase probability...');
+    const records = texts.map(text => ({ text }));
+    const description = dedent(`
+		A score from 0 to 100 indicating the likelihood that the user intends to make a purchase
+		in the next ${numDays} days. The input record is a text query representing a search
+		that a user may perform using a traditional search engine or LLM chat.
+	`);
+    const scores = await scoreBatch(records, 'Purchase Probability', description, 'integer', 0, 100, 'gpt-4.1-mini');
+    return scores;
+}
+export async function scoreRelevance(prompts, context) {
+    console.log('Scoring prompt relevance...');
+    const contextParts = [];
+    if (context.shortName) {
+        contextParts.push(`Brand: ${context.shortName}`);
+    }
+    if (context.sector) {
+        contextParts.push(`Sector: ${context.sector}`);
+    }
+    if (context.country) {
+        contextParts.push(`Market: ${context.country}`);
+    }
+    if (context.briefing) {
+        contextParts.push(`Brief: ${context.briefing}`);
+    }
+    const contextDescription = contextParts.length > 0
+        ? `Analysis context:\n${contextParts.join('\n')}`
+        : '';
+    const records = prompts.map(prompt => ({ prompt }));
+    const description = dedent(`
+		A score from 0.0 to 1.0 measuring how relevant this search keyword/prompt is
+		for the specified SEO analysis context. Consider whether it is specific enough
+		for the following brand, sector, market and research brief.
+
+		${contextDescription}
+
+		A score of 0.0 means the keyword/prompt is completely irrelevant to the context,
+		e.g. "cheap shoes" for a brand selling enterprise software. A low score can mean
+		the keyword/prompt is too generic, not related to the sector/market, or targets
+		a different audience. E.g. "data analysis" might be too broad in the context of
+		a social network scheduler tool. A medium or high score means the keyword is relevant,
+		e.g. "social media data analysis" for the same brand and analysis brief. A score of
+		1.0 means the keyword/prompt is highly relevant and specific to the context, e.g.
+		"best CRM software for small business" for a brand selling CRM software.
+	`);
+    const scores = await scoreBatch(records, 'Prompt Relevance', description, 'number', 0.0, 1.0, 'gpt-4.1-mini');
+    return scores;
+}

package/esm/src/apis/chatgptScraper/brightdata.d.ts

@@ -0,0 +1,3 @@
+import { type ProviderFunctions } from './scraper.js';
+export declare const brightdataProvider: ProviderFunctions;
+//# sourceMappingURL=brightdata.d.ts.map

package/esm/src/apis/chatgptScraper/brightdata.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"brightdata.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/chatgptScraper/brightdata.ts"],"names":[],"mappings":"AAeA,OAAO,EACN,KAAK,iBAAiB,EAKtB,MAAM,cAAc,CAAC;AAmNtB,eAAO,MAAM,kBAAkB,EAAE,iBAQhC,CAAC"}

package/esm/src/apis/chatgptScraper/brightdata.js

@@ -0,0 +1,172 @@
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+/**
+ * Brightdata GPT Scraper Provider.
+ *
+ * API Flow:
+ * 1. Trigger: POST to /datasets/v3/trigger → returns snapshot_id
+ * 2. Monitor: GET /datasets/v3/progress/{snapshot_id} until ready
+ * 3. Download: GET /datasets/v3/snapshot/{snapshot_id}
+ */
+import * as dntShim from "../../../_dnt.shims.js";
+import { withRetries, sleep } from '../../async.js';
+import { getAbortSignal, cleanAnswer, buildSources, buildSearchSources } from './scraper.js';
+// ============================================================================
+// Constants
+// ============================================================================
+const API_BASE = 'https://api.brightdata.com';
+const DATASET_ID = 'gd_m7aof0k82r803d5bjm';
+const OUTPUT_FIELDS = 'url|prompt|answer_text|answer_text_markdown|citations|links_attached|search_sources|country|model|web_search_triggered|web_search_query|index';
+const TRIGGER_RETRY = {
+    maxRetries: 3,
+    initialDelay: 0,
+    statusCodes: [429, 500, 502, 503, 504]
+};
+const DOWNLOAD_RETRY = {
+    maxRetries: 5,
+    initialDelay: 2000,
+    statusCodes: [202, 500, 502, 503, 504]
+};
+const MONITOR_RETRY = {
+    maxRetries: 4,
+    initialDelay: 1000,
+    statusCodes: [408, 425, 429, 500, 502, 503, 504]
+};
+const MONITOR_RETRIABLE = new Set(MONITOR_RETRY.statusCodes ?? []);
+const MAX_WAIT_MS = 600_000; // 10 minutes
+const POLL_INTERVAL_MS = 5_000;
+// ============================================================================
+// API Key
+// ============================================================================
+function getApiKey() {
+    const apiKey = dntShim.Deno.env.get('BRIGHTDATA_API_KEY');
+    if (!apiKey) {
+        throw new Error('BRIGHTDATA_API_KEY environment variable is required');
+    }
+    return apiKey;
+}
+// ============================================================================
+// Provider Functions
+// ============================================================================
+async function triggerJob(prompt, useSearch, countryISOCode) {
+    const apiKey = getApiKey();
+    const url = `${API_BASE}/datasets/v3/trigger?dataset_id=${DATASET_ID}&include_errors=true`;
+    const body = {
+        custom_output_fields: OUTPUT_FIELDS,
+        input: [{
+                url: 'http://chatgpt.com/',
+                prompt,
+                web_search: useSearch,
+                country: countryISOCode || '',
+                index: 0
+            }]
+    };
+    try {
+        const response = await withRetries(() => fetch(url, {
+            method: 'POST',
+            headers: {
+                'Authorization': `Bearer ${apiKey}`,
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify(body),
+            signal: getAbortSignal()
+        }), TRIGGER_RETRY);
+        if (!response.ok) {
+            console.error(`[Brightdata] Trigger error: ${response.status}`);
+            return null;
+        }
+        const data = await response.json();
+        return data?.snapshot_id || null;
+    }
+    catch (error) {
+        console.error('[Brightdata] Trigger failed:', error);
+        return null;
+    }
+}
+async function monitorJob(snapshotId) {
+    const apiKey = getApiKey();
+    const url = `${API_BASE}/datasets/v3/progress/${snapshotId}`;
+    const startTime = Date.now();
+    const abortSignal = getAbortSignal();
+    while (Date.now() - startTime < MAX_WAIT_MS) {
+        if (abortSignal?.aborted)
+            return false;
+        try {
+            const response = await withRetries(() => fetch(url, {
+                headers: { 'Authorization': `Bearer ${apiKey}` },
+                signal: abortSignal
+            }), MONITOR_RETRY);
+            if (!response.ok) {
+                if (!MONITOR_RETRIABLE.has(response.status))
+                    return false;
+            }
+            else {
+                const status = await response.json();
+                if (status.status === 'ready' || status.status === 'complete')
+                    return true;
+                if (status.status === 'failed' || status.status === 'error')
+                    return false;
+            }
+        }
+        catch (error) {
+            console.error('[Brightdata] Monitor error:', error);
+        }
+        await sleep(POLL_INTERVAL_MS, abortSignal);
+    }
+    console.error(`[Brightdata] Monitor timeout after ${MAX_WAIT_MS / 1000}s`);
+    return false;
+}
+async function downloadJob(snapshotId) {
+    const apiKey = getApiKey();
+    const url = `${API_BASE}/datasets/v3/snapshot/${snapshotId}?format=json`;
+    try {
+        const response = await withRetries(() => fetch(url, {
+            headers: { 'Authorization': `Bearer ${apiKey}` },
+            signal: getAbortSignal()
+        }), DOWNLOAD_RETRY);
+        if (!response.ok) {
+            console.error(`[Brightdata] Download error: ${response.status}`);
+            return null;
+        }
+        const data = await response.json();
+        return Array.isArray(data) ? data : null;
+    }
+    catch (error) {
+        console.error('[Brightdata] Download failed:', error);
+        return null;
+    }
+}
+function transformResponse(raw) {
+    const responses = raw;
+    if (!responses || responses.length === 0)
+        return null;
+    const response = responses[0];
+    let answer = response.answer_text_markdown || response.answer_text || '';
+    answer = cleanAnswer(answer);
+    // Build link positions map
+    const linkPositions = {};
+    for (const link of response.links_attached ?? []) {
+        if (link.url && link.position != null) {
+            linkPositions[link.url] ??= [];
+            linkPositions[link.url].push(link.position);
+        }
+    }
+    return {
+        prompt: response.prompt,
+        answer,
+        sources: buildSources(response.citations ?? [], linkPositions),
+        searchQueries: response.web_search_query || [],
+        searchSources: buildSearchSources(response.search_sources ?? [])
+    };
+}
+// ============================================================================
+// Export
+// ============================================================================
+export const brightdataProvider = {
+    name: 'Brightdata',
+    maxConcurrency: 50,
+    maxPromptsPerRequest: 1,
+    triggerJob,
+    monitorJob,
+    downloadJob,
+    transformResponse
+};

package/esm/src/apis/chatgptScraper/index.d.ts

@@ -0,0 +1,10 @@
+import type { ModelResult } from '../../schemas/models.schema.js';
+import { type BatchOptions } from './scraper.js';
+export type { BatchOptions };
+export type JobId = string | null;
+export declare function getMaxConcurrency(): number;
+export declare function getMaxPromptsPerRequest(): number;
+export declare function scrapeGPTBatch(options: BatchOptions): Promise<Array<ModelResult>>;
+export declare function triggerGPTBatch(options: BatchOptions): Promise<Array<string | null>>;
+export declare function downloadGPTSnapshots(jobIds: Array<string | null>): Promise<Array<ModelResult>>;
+//# sourceMappingURL=index.d.ts.map

package/esm/src/apis/chatgptScraper/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/chatgptScraper/index.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,KAAK,YAAY,EAAkC,MAAM,cAAc,CAAC;AAKjF,YAAY,EAAE,YAAY,EAAE,CAAC;AAC7B,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC;AAqBlC,wBAAgB,iBAAiB,IAAI,MAAM,CAE1C;AAED,wBAAgB,uBAAuB,IAAI,MAAM,CAEhD;AAED,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG"}

package/esm/src/apis/chatgptScraper/index.js

@@ -0,0 +1,41 @@
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+/**
+ * GPT Scraper - Public API
+ *
+ * Selects between Brightdata and Oxylabs based on CHATGPT_SCRAPER_PROVIDER env var.
+ * Default: oxylabs
+ */
+import * as dntShim from "../../../_dnt.shims.js";
+import { createScraper } from './scraper.js';
+import { brightdataProvider } from './brightdata.js';
+import { oxylabsProvider } from './oxy.js';
+// ============================================================================
+// Scraper Instance (lazy singleton)
+// ============================================================================
+let scraper = null;
+function getScraper() {
+    if (!scraper) {
+        const providerName = dntShim.Deno.env.get('CHATGPT_SCRAPER_PROVIDER')?.toLowerCase();
+        const provider = providerName === 'brightdata' ? brightdataProvider : oxylabsProvider;
+        scraper = createScraper(provider);
+    }
+    return scraper;
+}
+// ============================================================================
+// Public API
+// ============================================================================
+export function getMaxConcurrency() {
+    return getScraper().maxConcurrency;
+}
+export function getMaxPromptsPerRequest() {
+    return getScraper().maxPromptsPerRequest;
+}
+export async function scrapeGPTBatch(options) {
+    return getScraper().scrapeGPTBatch(options);
+}
+export async function triggerGPTBatch(options) {
+    return getScraper().triggerGPTBatch(options);
+}
+export async function downloadGPTSnapshots(jobIds) {
+    return getScraper().downloadGPTSnapshots(jobIds);
+}

package/esm/src/apis/chatgptScraper/oxy.d.ts

@@ -0,0 +1,3 @@
+import { type ProviderFunctions } from './scraper.js';
+export declare const oxylabsProvider: ProviderFunctions;
+//# sourceMappingURL=oxy.d.ts.map

package/esm/src/apis/chatgptScraper/oxy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"oxy.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/chatgptScraper/oxy.ts"],"names":[],"mappings":"AAeA,OAAO,EACN,KAAK,iBAAiB,EAItB,MAAM,cAAc,CAAC;AA6LtB,eAAO,MAAM,eAAe,EAAE,iBAQ7B,CAAC"}

package/esm/src/apis/chatgptScraper/oxy.js

@@ -0,0 +1,156 @@
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+/**
+ * Oxylabs GPT Scraper Provider.
+ *
+ * API Flow (Async Push-Pull):
+ * 1. Trigger: POST to /v1/queries → returns job id
+ * 2. Monitor: GET /v1/queries/{id} until status is 'done'
+ * 3. Download: GET /v1/queries/{id}/results
+ */
+import * as dntShim from "../../../_dnt.shims.js";
+import { withRetries, sleep } from '../../async.js';
+import { getAbortSignal, cleanAnswer, buildSources } from './scraper.js';
+// ============================================================================
+// Constants
+// ============================================================================
+const API_BASE = 'https://data.oxylabs.io/v1';
+const RETRY_CONFIG = {
+    maxRetries: 3,
+    initialDelay: 1000,
+    statusCodes: [429, 500, 502, 503, 504, 524, 612, 613]
+};
+const MAX_WAIT_MS = 600_000; // 10 minutes
+const POLL_INTERVAL_MS = 5_000;
+// ============================================================================
+// Auth
+// ============================================================================
+function getAuthHeader() {
+    const username = dntShim.Deno.env.get('OXYLABS_USERNAME');
+    const password = dntShim.Deno.env.get('OXYLABS_PASSWORD');
+    if (!username || !password) {
+        throw new Error('OXYLABS_USERNAME and OXYLABS_PASSWORD environment variables are required');
+    }
+    return `Basic ${btoa(`${username}:${password}`)}`;
+}
+// ============================================================================
+// Provider Functions
+// ============================================================================
+async function triggerJob(prompt, useSearch, countryISOCode) {
+    const authHeader = getAuthHeader();
+    const url = `${API_BASE}/queries`;
+    const body = {
+        source: 'chatgpt',
+        prompt,
+        parse: true,
+        search: true // Oxylabs requires search: true (cannot be false or blank)
+    };
+    if (countryISOCode) {
+        body.geo_location = countryISOCode;
+    }
+    try {
+        const response = await withRetries(() => fetch(url, {
+            method: 'POST',
+            headers: {
+                'Authorization': authHeader,
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify(body),
+            signal: getAbortSignal()
+        }), RETRY_CONFIG);
+        if (!response.ok) {
+            console.error(`[Oxylabs] Trigger error: ${response.status}`);
+            return null;
+        }
+        const data = await response.json();
+        return data?.id || null;
+    }
+    catch (error) {
+        console.error('[Oxylabs] Trigger failed:', error);
+        return null;
+    }
+}
+async function monitorJob(jobId) {
+    const authHeader = getAuthHeader();
+    const url = `${API_BASE}/queries/${jobId}`;
+    const startTime = Date.now();
+    const abortSignal = getAbortSignal();
+    while (Date.now() - startTime < MAX_WAIT_MS) {
+        if (abortSignal?.aborted)
+            return false;
+        try {
+            const response = await fetch(url, {
+                headers: { 'Authorization': authHeader },
+                signal: abortSignal
+            });
+            // 204 = job not completed yet, continue polling
+            if (response.status === 204) {
+                await sleep(POLL_INTERVAL_MS, abortSignal);
+                continue;
+            }
+            if (response.ok) {
+                const status = await response.json();
+                if (status.status === 'done')
+                    return true;
+                if (status.status === 'faulted' || status.status === 'failed')
+                    return false;
+            }
+        }
+        catch (error) {
+            console.error('[Oxylabs] Monitor error:', error);
+        }
+        await sleep(POLL_INTERVAL_MS, abortSignal);
+    }
+    console.error(`[Oxylabs] Monitor timeout after ${MAX_WAIT_MS / 1000}s`);
+    return false;
+}
+async function downloadJob(jobId) {
+    const authHeader = getAuthHeader();
+    const url = `${API_BASE}/queries/${jobId}/results`;
+    try {
+        const response = await withRetries(() => fetch(url, {
+            headers: { 'Authorization': authHeader },
+            signal: getAbortSignal()
+        }), RETRY_CONFIG);
+        if (!response.ok) {
+            console.error(`[Oxylabs] Download error: ${response.status}`);
+            return null;
+        }
+        return await response.json();
+    }
+    catch (error) {
+        console.error('[Oxylabs] Download failed:', error);
+        return null;
+    }
+}
+function transformResponse(raw) {
+    const response = raw;
+    const content = response?.results?.[0]?.content;
+    if (!content)
+        return null;
+    let answer = content.markdown_text || content.response_text || '';
+    answer = cleanAnswer(answer);
+    // Map section='citations' to cited=true (like Brightdata's cited field)
+    const citations = (content.citations ?? []).map(c => ({
+        ...c,
+        cited: c.section === 'citations'
+    }));
+    return {
+        prompt: content.prompt || '',
+        answer,
+        sources: buildSources(citations),
+        searchQueries: [],
+        searchSources: []
+    };
+}
+// ============================================================================
+// Export
+// ============================================================================
+export const oxylabsProvider = {
+    name: 'Oxylabs',
+    maxConcurrency: 10,
+    maxPromptsPerRequest: 1,
+    triggerJob,
+    monitorJob,
+    downloadJob,
+    transformResponse
+};

package/esm/src/apis/chatgptScraper/scraper.d.ts

@@ -0,0 +1,46 @@
+import type { ModelResult } from '../../schemas/models.schema.js';
+import type { Source, SearchSource } from '../../schemas/sources.schema.js';
+export interface BatchOptions {
+    prompts: Array<string>;
+    useSearch?: boolean;
+    countryISOCode?: string | null;
+}
+export interface ProviderFunctions {
+    name: string;
+    maxConcurrency: number;
+    maxPromptsPerRequest: number;
+    triggerJob: (prompt: string, useSearch: boolean, countryISOCode: string | null) => Promise<string | null>;
+    monitorJob: (jobId: string) => Promise<boolean>;
+    downloadJob: (jobId: string) => Promise<unknown>;
+    transformResponse: (raw: unknown) => ModelResult | null;
+}
+export interface GPTScraper {
+    maxConcurrency: number;
+    maxPromptsPerRequest: number;
+    scrapeGPTBatch: (options: BatchOptions) => Promise<Array<ModelResult>>;
+    triggerGPTBatch: (options: BatchOptions) => Promise<Array<string | null>>;
+    downloadGPTSnapshots: (jobIds: Array<string | null>) => Promise<Array<ModelResult>>;
+}
+export declare function getAbortSignal(): AbortSignal | undefined;
+export declare function cleanAnswer(answer: string): string;
+export declare function buildSources(citations: Array<{
+    url: string;
+    title?: string;
+    description?: string;
+    text?: string;
+    cited?: boolean;
+}>, linkPositions?: Record<string, Array<number>>): Array<Source>;
+export declare function buildSearchSources(sources: Array<{
+    url?: string;
+    title?: string;
+    snippet?: string;
+    rank?: number;
+    date_published?: string;
+}>): Array<SearchSource>;
+/**
+ * Creates an empty model result for failed jobs.
+ * This ensures we always return the same number of rows as input.
+ */
+export declare function emptyModelResult(providerName: string, errorMessage?: string, context?: unknown): ModelResult;
+export declare function createScraper(provider: ProviderFunctions): GPTScraper;
+//# sourceMappingURL=scraper.d.ts.map

package/esm/src/apis/chatgptScraper/scraper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/chatgptScraper/scraper.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAO5E,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1G,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAChD,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACjD,iBAAiB,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,WAAW,GAAG,IAAI,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,cAAc,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;IACvE,eAAe,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;IAC1E,oBAAoB,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;CACpF;AAMD,wBAAgB,cAAc,IAAI,WAAW,GAAG,SAAS,CAExD;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAMlD;AAED,wBAAgB,YAAY,CAC3B,SAAS,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,EACvG,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAC3C,KAAK,CAAC,MAAM,CAAC,CAQf;AAED,wBAAgB,kBAAkB,CACjC,OAAO,EAAE,KAAK,CAAC;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GACxG,KAAK,CAAC,YAAY,CAAC,CAQrB;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,GAAG,WAAW,CAS5G;AAMD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,iBAAiB,GAAG,UAAU,CAkErE"}